From f1b41aa20024f93de4d3f35351c27b94ba2f230e Mon Sep 17 00:00:00 2001 From: Sparsh Agarwal Date: Sat, 12 Feb 2022 00:45:09 +0530 Subject: [PATCH 1/2] Created using Colaboratory --- _notebooks/2022-01-07-ncf.ipynb | 16712 +++++++++++++++++++++++++++++- 1 file changed, 16711 insertions(+), 1 deletion(-) diff --git a/_notebooks/2022-01-07-ncf.ipynb b/_notebooks/2022-01-07-ncf.ipynb index 6214e3e..6ce85d7 100644 --- a/_notebooks/2022-01-07-ncf.ipynb +++ b/_notebooks/2022-01-07-ncf.ipynb @@ -1 +1,16711 @@ -{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-07-ncf.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/NCF.ipynb","timestamp":1644444357896}],"collapsed_sections":[],"toc_visible":true,"authorship_tag":"ABX9TyM8P9CeW0IV3JQC20K0/Y2L"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"iVkUysixCyk6"},"source":["# Neural Collaborative Filtering Recommenders"]},{"cell_type":"code","metadata":{"id":"-xzeGn4mvtvd"},"source":["!pip install -q pytorch-lightning"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xNgD2QXOkq4g"},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","import pandas as pd\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","import torch.optim as optim\n","from torch.utils.data import Dataset, DataLoader\n","from torch.utils.data import TensorDataset\n","from tqdm.notebook import tqdm\n","import pytorch_lightning as pl\n","\n","np.random.seed(123)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"8fqqbAgmrxoT"},"source":["## NCF with PyTorch Lightning on ML-25m\n","\n","In this section, we will build a simple yet accurate model using movielens-25m dataset and pytorch lightning library. This will be a retrieval model where the objective is to maximize recall over precision."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MrlkpJITv8Rw","executionInfo":{"elapsed":93221,"status":"ok","timestamp":1633618213038,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"4fd97487-1f04-4563-8adf-5aa306c13d2b"},"source":["!wget -q --show-progress https://files.grouplens.org/datasets/movielens/ml-25m.zip\n","!unzip ml-25m.zip"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["ml-25m.zip.1 100%[===================>] 249.84M 45.7MB/s in 5.9s \n","Archive: ml-25m.zip\n","replace ml-25m/tags.csv? [y]es, [n]o, [A]ll, [N]one, [r]ename: N\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"5-juuyKOwCmL","executionInfo":{"elapsed":18456,"status":"ok","timestamp":1633618231482,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"93ff3c63-a959-4e2d-fdff-c5e72a475160"},"source":["ratings = pd.read_csv('ml-25m/ratings.csv', infer_datetime_format=True)\n","ratings.head()"],"execution_count":null,"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdratingtimestamp
012965.01147880044
113063.51147868817
213075.01147868828
316655.01147878820
418993.51147868510
\n","
"],"text/plain":[" userId movieId rating timestamp\n","0 1 296 5.0 1147880044\n","1 1 306 3.5 1147868817\n","2 1 307 5.0 1147868828\n","3 1 665 5.0 1147878820\n","4 1 899 3.5 1147868510"]},"execution_count":30,"metadata":{},"output_type":"execute_result"}]},{"cell_type":"markdown","metadata":{"id":"A2bKYi9WwGyP"},"source":["### Subset\n","\n","In order to keep memory usage manageable, we will only use data from 20% of the users in this dataset. Let's randomly select 30% of the users and only use data from the selected users."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HsYAMEXqwZoH","executionInfo":{"elapsed":2345,"status":"ok","timestamp":1633618233821,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"434b4209-8c53-4474-edd8-6c5d109c3184"},"source":["rand_userIds = np.random.choice(ratings['userId'].unique(), \n"," size=int(len(ratings['userId'].unique())*0.2), \n"," replace=False)\n","\n","ratings = ratings.loc[ratings['userId'].isin(rand_userIds)]\n","\n","print('There are {} rows of data from {} users'.format(len(ratings), len(rand_userIds)))"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["There are 5015129 rows of data from 32508 users\n"]}]},{"cell_type":"markdown","metadata":{"id":"w7OS6UqDpXdi"},"source":["### Train/Test Split\n","**Chronological Leave-One-Out Split**"]},{"cell_type":"markdown","metadata":{"id":"ZaqAMrH-gn_i"},"source":["Along with the rating, there is also a timestamp column that shows the date and time the review was submitted. Using the timestamp column, we will implement our train-test split strategy using the leave-one-out methodology. For each user, the most recent review is used as the test set (i.e. leave one out), while the rest will be used as training data ."]},{"cell_type":"markdown","metadata":{"id":"A4COa9yVguUO"},"source":["> Note: Doing a random split would not be fair, as we could potentially be using a user's recent reviews for training and earlier reviews for testing. This introduces data leakage with a look-ahead bias, and the performance of the trained model would not be generalizable to real-world performance."]},{"cell_type":"code","metadata":{"id":"WtdtS0FMgTez"},"source":["ratings['rank_latest'] = ratings.groupby(['userId'])['timestamp'] \\\n"," .rank(method='first', ascending=False)\n","\n","train_ratings = ratings[ratings['rank_latest'] != 1]\n","test_ratings = ratings[ratings['rank_latest'] == 1]\n","\n","# drop columns that we no longer need\n","train_ratings = train_ratings[['userId', 'movieId', 'rating']]\n","test_ratings = test_ratings[['userId', 'movieId', 'rating']]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"XFoYAbhqpnPD"},"source":["### Implicit Conversion"]},{"cell_type":"markdown","metadata":{"id":"HwYvXqJ6hz2u"},"source":["We will train a recommender system using implicit feedback. However, the MovieLens dataset that we're using is based on explicit feedback. To convert this dataset into an implicit feedback dataset, we'll simply binarize the ratings such that they are are '1' (i.e. positive class). The value of '1' represents that the user has interacted with the item.\n","\n","> Note: Using implicit feedback reframes the problem that our recommender is trying to solve. Instead of trying to predict movie ratings (when using explicit feedback), we are trying to predict whether the user will interact (i.e. click/buy/watch) with each movie, with the aim of presenting to users the movies with the highest interaction likelihood.\n","\n","> Tip: This setting is suitable at retrieval stage where the objective is to maximize recall by identifying items that user will at least interact with."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"sQYuW1Otg_Cg","executionInfo":{"elapsed":751,"status":"ok","timestamp":1633618236164,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"096aec13-d21e-4690-e88d-258fe9a681a0"},"source":["train_ratings.loc[:, 'rating'] = 1\n","\n","train_ratings.sample(5)"],"execution_count":null,"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdrating
98655406404320191
1764897511439826711
190457581235279861
312501220593864871
45403492980345711
\n","
"],"text/plain":[" userId movieId rating\n","9865540 64043 2019 1\n","17648975 114398 2671 1\n","19045758 123527 986 1\n","3125012 20593 86487 1\n","4540349 29803 4571 1"]},"execution_count":33,"metadata":{},"output_type":"execute_result"}]},{"cell_type":"markdown","metadata":{"id":"uhwZiaBPpsQl"},"source":["### Negative Sampling"]},{"cell_type":"markdown","metadata":{"id":"ngZdyoMjizlw"},"source":["We do have a problem now though. After binarizing our dataset, we see that every sample in the dataset now belongs to the positive class. However we also require negative samples to train our models, to indicate movies that the user has not interacted with. We assume that such movies are those that the user are not interested in - even though this is a sweeping assumption that may not be true, it usually works out rather well in practice.\n","\n","The code below generates 4 negative samples for each row of data. In other words, the ratio of negative to positive samples is 4:1. This ratio is chosen arbitrarily but I found that it works rather well (feel free to find the best ratio yourself!)"]},{"cell_type":"code","metadata":{"id":"4T0_UVhTizVn"},"source":["# Get a list of all movie IDs\n","all_movieIds = ratings['movieId'].unique()\n","\n","# Placeholders that will hold the training data\n","users, items, labels = [], [], []\n","\n","# This is the set of items that each user has interaction with\n","user_item_set = set(zip(train_ratings['userId'], train_ratings['movieId']))\n","\n","# 4:1 ratio of negative to positive samples\n","num_negatives = 4\n","\n","for (u, i) in tqdm(user_item_set):\n"," users.append(u)\n"," items.append(i)\n"," labels.append(1) # items that the user has interacted with are positive\n"," for _ in range(num_negatives):\n"," # randomly select an item\n"," negative_item = np.random.choice(all_movieIds) \n"," # check that the user has not interacted with this item\n"," while (u, negative_item) in user_item_set:\n"," negative_item = np.random.choice(all_movieIds)\n"," users.append(u)\n"," items.append(negative_item)\n"," labels.append(0) # items not interacted with are negative"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"9brxnZqlpvXD"},"source":["### PyTorch Dataset"]},{"cell_type":"markdown","metadata":{"id":"Br2u5nn5jAy1"},"source":["Great! We now have the data in the format required by our model. Before we move on, let's define a PyTorch Dataset to facilitate training. The class below simply encapsulates the code we have written above into a PyTorch Dataset class."]},{"cell_type":"code","metadata":{"id":"pCn0M346i6Z8"},"source":["class MovieLensTrainDataset(Dataset):\n"," \"\"\"MovieLens PyTorch Dataset for Training\n"," \n"," Args:\n"," ratings (pd.DataFrame): Dataframe containing the movie ratings\n"," all_movieIds (list): List containing all movieIds\n"," \n"," \"\"\"\n","\n"," def __init__(self, ratings, all_movieIds):\n"," self.users, self.items, self.labels = self.get_dataset(ratings, all_movieIds)\n","\n"," def __len__(self):\n"," return len(self.users)\n"," \n"," def __getitem__(self, idx):\n"," return self.users[idx], self.items[idx], self.labels[idx]\n","\n"," def get_dataset(self, ratings, all_movieIds):\n"," users, items, labels = [], [], []\n"," user_item_set = set(zip(ratings['userId'], ratings['movieId']))\n","\n"," num_negatives = 4\n"," for u, i in user_item_set:\n"," users.append(u)\n"," items.append(i)\n"," labels.append(1)\n"," for _ in range(num_negatives):\n"," negative_item = np.random.choice(all_movieIds)\n"," while (u, negative_item) in user_item_set:\n"," negative_item = np.random.choice(all_movieIds)\n"," users.append(u)\n"," items.append(negative_item)\n"," labels.append(0)\n","\n"," return torch.tensor(users), torch.tensor(items), torch.tensor(labels)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"SqMDcEYRjOZN"},"source":["### Model\n","\n","While there are many deep learning based architecture for recommendation systems, I find that the framework proposed by He et al. is the most straightforward and it is simple enough to be implemented in a tutorial such as this."]},{"cell_type":"code","metadata":{"id":"xwlBJpqljJvS"},"source":["class NCF(pl.LightningModule):\n"," \"\"\" Neural Collaborative Filtering (NCF)\n"," \n"," Args:\n"," num_users (int): Number of unique users\n"," num_items (int): Number of unique items\n"," ratings (pd.DataFrame): Dataframe containing the movie ratings for training\n"," all_movieIds (list): List containing all movieIds (train + test)\n"," \"\"\"\n"," \n"," def __init__(self, num_users, num_items, ratings, all_movieIds):\n"," super().__init__()\n"," self.user_embedding = nn.Embedding(num_embeddings=num_users, embedding_dim=8)\n"," self.item_embedding = nn.Embedding(num_embeddings=num_items, embedding_dim=8)\n"," self.fc1 = nn.Linear(in_features=16, out_features=64)\n"," self.fc2 = nn.Linear(in_features=64, out_features=32)\n"," self.output = nn.Linear(in_features=32, out_features=1)\n"," self.ratings = ratings\n"," self.all_movieIds = all_movieIds\n"," \n"," def forward(self, user_input, item_input):\n"," \n"," # Pass through embedding layers\n"," user_embedded = self.user_embedding(user_input)\n"," item_embedded = self.item_embedding(item_input)\n","\n"," # Concat the two embedding layers\n"," vector = torch.cat([user_embedded, item_embedded], dim=-1)\n","\n"," # Pass through dense layer\n"," vector = nn.ReLU()(self.fc1(vector))\n"," vector = nn.ReLU()(self.fc2(vector))\n","\n"," # Output layer\n"," pred = nn.Sigmoid()(self.output(vector))\n","\n"," return pred\n"," \n"," def training_step(self, batch, batch_idx):\n"," user_input, item_input, labels = batch\n"," predicted_labels = self(user_input, item_input)\n"," loss = nn.BCELoss()(predicted_labels, labels.view(-1, 1).float())\n"," return loss\n","\n"," def configure_optimizers(self):\n"," return torch.optim.Adam(self.parameters())\n","\n"," def train_dataloader(self):\n"," return DataLoader(MovieLensTrainDataset(self.ratings, self.all_movieIds),\n"," batch_size=512, num_workers=2)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"I8wT1WK9jzeJ"},"source":["We instantiate the NCF model using the class that we have defined above."]},{"cell_type":"code","metadata":{"id":"F3Bh9dorjww7"},"source":["num_users = ratings['userId'].max()+1\n","num_items = ratings['movieId'].max()+1\n","\n","all_movieIds = ratings['movieId'].unique()\n","\n","model = NCF(num_users, num_items, train_ratings, all_movieIds)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"K4Mw8CdVp5lF"},"source":["### Model Training"]},{"cell_type":"markdown","metadata":{"id":"xnYHNWe3kRRD"},"source":["> Note: One advantage of PyTorch Lightning over vanilla PyTorch is that you don't need to write your own boiler plate training code. Notice how the Trainer class allows us to train our model with just a few lines of code.\n","\n","Let's train our NCF model for 5 epochs using the GPU. "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":375,"referenced_widgets":["68bcd7bfc32f4d9ebaba5c08437bca28"]},"id":"0JganCIMj2EW","outputId":"6fa64b89-c835-4f39-d6ad-bac6f2369c64"},"source":["trainer = pl.Trainer(max_epochs=5, gpus=1, reload_dataloaders_every_epoch=True,\n"," progress_bar_refresh_rate=50, logger=False, checkpoint_callback=False)\n","\n","trainer.fit(model)"],"execution_count":null,"outputs":[{"name":"stderr","output_type":"stream","text":["GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","---------------------------------------------\n","0 | user_embedding | Embedding | 1.3 M \n","1 | item_embedding | Embedding | 1.7 M \n","2 | fc1 | Linear | 1.1 K \n","3 | fc2 | Linear | 2.1 K \n","4 | output | Linear | 33 \n","---------------------------------------------\n","3.0 M Trainable params\n","0 Non-trainable params\n","3.0 M Total params\n","11.907 Total estimated model params size (MB)\n","/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," cpuset_checked))\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"68bcd7bfc32f4d9ebaba5c08437bca28","version_major":2,"version_minor":0},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"]},"metadata":{"tags":[]},"output_type":"display_data"}]},{"cell_type":"markdown","metadata":{"id":"R6V1Tiw1kIxk"},"source":["> Note: We are using the argument reload_dataloaders_every_epoch=True. This creates a new randomly chosen set of negative samples for each epoch, which ensures that our model is not biased by the selection of negative samples."]},{"cell_type":"markdown","metadata":{"id":"I3BXx1YzlAUq"},"source":["### Evaluating our Recommender System\n","\n","Now that our model is trained, we are ready to evaluate it using the test data. In traditional Machine Learning projects, we evaluate our models using metrics such as Accuracy (for classification problems) and RMSE (for regression problems). However, such metrics are too simplistic for evaluating recommender systems.\n","\n","The key here is that we don't need the user to interact on every single item in the list of recommendations. Instead, we just need the user to interact with at least one item on the list - as long as the user does that, the recommendations have worked.\n","\n","To simulate this, let's run the following evaluation protocol to generate a list of 10 recommended items for each user.\n","- For each user, randomly select 99 items that the user has not interacted with\n","- Combine these 99 items with the test item (the actual item that the user interacted with). We now have 100 items.\n","- Run the model on these 100 items, and rank them according to their predicted probabilities\n","- Select the top 10 items from the list of 100 items. If the test item is present within the top 10 items, then we say that this is a hit.\n","- Repeat the process for all users. The Hit Ratio is then the average hits."]},{"cell_type":"markdown","metadata":{"id":"B2PVVpUflN34"},"source":["> Note: This evaluation protocol is known as Hit Ratio @ 10, and it is commonly used to evaluate recommender systems."]},{"cell_type":"code","metadata":{"id":"uSLTYZuhlNEV"},"source":["# User-item pairs for testing\n","test_user_item_set = set(zip(test_ratings['userId'], test_ratings['movieId']))\n","\n","# Dict of all items that are interacted with by each user\n","user_interacted_items = ratings.groupby('userId')['movieId'].apply(list).to_dict()\n","\n","hits = []\n","for (u,i) in tqdm(test_user_item_set):\n"," interacted_items = user_interacted_items[u]\n"," not_interacted_items = set(all_movieIds) - set(interacted_items)\n"," selected_not_interacted = list(np.random.choice(list(not_interacted_items), 99))\n"," test_items = selected_not_interacted + [i]\n"," \n"," predicted_labels = np.squeeze(model(torch.tensor([u]*100), \n"," torch.tensor(test_items)).detach().numpy())\n"," \n"," top10_items = [test_items[i] for i in np.argsort(predicted_labels)[::-1][0:10].tolist()]\n"," \n"," if i in top10_items:\n"," hits.append(1)\n"," else:\n"," hits.append(0)\n"," \n","print(\"The Hit Ratio @ 10 is {:.2f}\".format(np.average(hits)))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"s1XtzBFsllfN"},"source":["We got a pretty good Hit Ratio @ 10 score! To put this into context, what this means is that 86% of the users were recommended the actual item (among a list of 10 items) that they eventually interacted with. Not bad!"]},{"cell_type":"markdown","metadata":{"id":"_xofdqRI29zl"},"source":["## NMF with PyTorch on ML-1m"]},{"cell_type":"code","metadata":{"id":"3wo7aehx3AyG"},"source":["import os\n","import time\n","import random\n","import argparse\n","import numpy as np \n","import pandas as pd \n","import torch\n","import torch.nn as nn\n","import torch.optim as optim\n","import torch.utils.data as data\n","from torch.utils.tensorboard import SummaryWriter"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"y1iNipgl3JhO","executionInfo":{"elapsed":682,"status":"ok","timestamp":1633618725401,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"363cf5f9-b062-4930-b122-d7573d824ab0"},"source":["DATA_URL = \"https://raw.githubusercontent.com/sparsh-ai/rec-data-public/master/ml-1m-dat/ratings.dat\"\n","MAIN_PATH = '/content/'\n","DATA_PATH = MAIN_PATH + 'ratings.dat'\n","MODEL_PATH = MAIN_PATH + 'models/'\n","MODEL = 'ml-1m_Neu_MF'\n","\n","!wget -q --show-progress https://raw.githubusercontent.com/sparsh-ai/rec-data-public/master/ml-1m-dat/ratings.dat"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["\rratings.dat 0%[ ] 0 --.-KB/s \rratings.dat 100%[===================>] 23.45M 128MB/s in 0.2s \n"]}]},{"cell_type":"code","metadata":{"id":"EXFnsMFy3YTE"},"source":["def seed_everything(seed):\n"," random.seed(seed)\n"," os.environ['PYTHONHASHSEED'] = str(seed)\n"," np.random.seed(seed)\n"," torch.manual_seed(seed)\n"," torch.cuda.manual_seed(seed)\n"," torch.backends.cudnn.deterministic = True\n"," torch.backends.cudnn.benchmark = True"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"KvTX81Z23bFs"},"source":["### Dataset"]},{"cell_type":"code","metadata":{"id":"NN5GjJCf3rI8"},"source":["class Rating_Datset(torch.utils.data.Dataset):\n","\tdef __init__(self, user_list, item_list, rating_list):\n","\t\tsuper(Rating_Datset, self).__init__()\n","\t\tself.user_list = user_list\n","\t\tself.item_list = item_list\n","\t\tself.rating_list = rating_list\n","\n","\tdef __len__(self):\n","\t\treturn len(self.user_list)\n","\n","\tdef __getitem__(self, idx):\n","\t\tuser = self.user_list[idx]\n","\t\titem = self.item_list[idx]\n","\t\trating = self.rating_list[idx]\n","\t\t\n","\t\treturn (\n","\t\t\ttorch.tensor(user, dtype=torch.long),\n","\t\t\ttorch.tensor(item, dtype=torch.long),\n","\t\t\ttorch.tensor(rating, dtype=torch.float)\n","\t\t\t)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"d4xgxyBsfoJM"},"source":["- *_reindex*: process dataset to reindex userID and itemID, also set rating as binary feedback\n","- *_leave_one_out*: leave-one-out evaluation protocol in paper https://www.comp.nus.edu.sg/~xiangnan/papers/ncf.pdf\n","- *negative_sampling*: randomly selects n negative examples for each positive one"]},{"cell_type":"code","metadata":{"id":"HggfgX_8Oqmq"},"source":["class NCF_Data(object):\n","\t\"\"\"\n","\tConstruct Dataset for NCF\n","\t\"\"\"\n","\tdef __init__(self, args, ratings):\n","\t\tself.ratings = ratings\n","\t\tself.num_ng = args.num_ng\n","\t\tself.num_ng_test = args.num_ng_test\n","\t\tself.batch_size = args.batch_size\n","\n","\t\tself.preprocess_ratings = self._reindex(self.ratings)\n","\n","\t\tself.user_pool = set(self.ratings['user_id'].unique())\n","\t\tself.item_pool = set(self.ratings['item_id'].unique())\n","\n","\t\tself.train_ratings, self.test_ratings = self._leave_one_out(self.preprocess_ratings)\n","\t\tself.negatives = self._negative_sampling(self.preprocess_ratings)\n","\t\trandom.seed(args.seed)\n","\t\n","\tdef _reindex(self, ratings):\n","\t\t\"\"\"\n","\t\tProcess dataset to reindex userID and itemID, also set rating as binary feedback\n","\t\t\"\"\"\n","\t\tuser_list = list(ratings['user_id'].drop_duplicates())\n","\t\tuser2id = {w: i for i, w in enumerate(user_list)}\n","\n","\t\titem_list = list(ratings['item_id'].drop_duplicates())\n","\t\titem2id = {w: i for i, w in enumerate(item_list)}\n","\n","\t\tratings['user_id'] = ratings['user_id'].apply(lambda x: user2id[x])\n","\t\tratings['item_id'] = ratings['item_id'].apply(lambda x: item2id[x])\n","\t\tratings['rating'] = ratings['rating'].apply(lambda x: float(x > 0))\n","\t\treturn ratings\n","\n","\tdef _leave_one_out(self, ratings):\n","\t\t\"\"\"\n","\t\tleave-one-out evaluation protocol in paper https://www.comp.nus.edu.sg/~xiangnan/papers/ncf.pdf\n","\t\t\"\"\"\n","\t\tratings['rank_latest'] = ratings.groupby(['user_id'])['timestamp'].rank(method='first', ascending=False)\n","\t\ttest = ratings.loc[ratings['rank_latest'] == 1]\n","\t\ttrain = ratings.loc[ratings['rank_latest'] > 1]\n","\t\tassert train['user_id'].nunique()==test['user_id'].nunique(), 'Not Match Train User with Test User'\n","\t\treturn train[['user_id', 'item_id', 'rating']], test[['user_id', 'item_id', 'rating']]\n","\n","\tdef _negative_sampling(self, ratings):\n","\t\tinteract_status = (\n","\t\t\tratings.groupby('user_id')['item_id']\n","\t\t\t.apply(set)\n","\t\t\t.reset_index()\n","\t\t\t.rename(columns={'item_id': 'interacted_items'}))\n","\t\tinteract_status['negative_items'] = interact_status['interacted_items'].apply(lambda x: self.item_pool - x)\n","\t\tinteract_status['negative_samples'] = interact_status['negative_items'].apply(lambda x: random.sample(x, self.num_ng_test))\n","\t\treturn interact_status[['user_id', 'negative_items', 'negative_samples']]\n","\n","\tdef get_train_instance(self):\n","\t\tusers, items, ratings = [], [], []\n","\t\ttrain_ratings = pd.merge(self.train_ratings, self.negatives[['user_id', 'negative_items']], on='user_id')\n","\t\ttrain_ratings['negatives'] = train_ratings['negative_items'].apply(lambda x: random.sample(x, self.num_ng))\n","\t\tfor row in train_ratings.itertuples():\n","\t\t\tusers.append(int(row.user_id))\n","\t\t\titems.append(int(row.item_id))\n","\t\t\tratings.append(float(row.rating))\n","\t\t\tfor i in range(self.num_ng):\n","\t\t\t\tusers.append(int(row.user_id))\n","\t\t\t\titems.append(int(row.negatives[i]))\n","\t\t\t\tratings.append(float(0)) # negative samples get 0 rating\n","\t\tdataset = Rating_Datset(\n","\t\t\tuser_list=users,\n","\t\t\titem_list=items,\n","\t\t\trating_list=ratings)\n","\t\treturn torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=True, num_workers=2)\n","\n","\tdef get_test_instance(self):\n","\t\tusers, items, ratings = [], [], []\n","\t\ttest_ratings = pd.merge(self.test_ratings, self.negatives[['user_id', 'negative_samples']], on='user_id')\n","\t\tfor row in test_ratings.itertuples():\n","\t\t\tusers.append(int(row.user_id))\n","\t\t\titems.append(int(row.item_id))\n","\t\t\tratings.append(float(row.rating))\n","\t\t\tfor i in getattr(row, 'negative_samples'):\n","\t\t\t\tusers.append(int(row.user_id))\n","\t\t\t\titems.append(int(i))\n","\t\t\t\tratings.append(float(0))\n","\t\tdataset = Rating_Datset(\n","\t\t\tuser_list=users,\n","\t\t\titem_list=items,\n","\t\t\trating_list=ratings)\n","\t\treturn torch.utils.data.DataLoader(dataset, batch_size=self.num_ng_test+1, shuffle=False, num_workers=2)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"hfXyLsgVOsBs"},"source":["### Metrics\n","Using Hit Rate and NDCG as our evaluation metrics"]},{"cell_type":"code","metadata":{"id":"KM4B7r12OvnS"},"source":["def hit(ng_item, pred_items):\n","\tif ng_item in pred_items:\n","\t\treturn 1\n","\treturn 0\n","\n","\n","def ndcg(ng_item, pred_items):\n","\tif ng_item in pred_items:\n","\t\tindex = pred_items.index(ng_item)\n","\t\treturn np.reciprocal(np.log2(index+2))\n","\treturn 0\n","\n","\n","def metrics(model, test_loader, top_k, device):\n","\tHR, NDCG = [], []\n","\n","\tfor user, item, label in test_loader:\n","\t\tuser = user.to(device)\n","\t\titem = item.to(device)\n","\n","\t\tpredictions = model(user, item)\n","\t\t_, indices = torch.topk(predictions, top_k)\n","\t\trecommends = torch.take(\n","\t\t\t\titem, indices).cpu().numpy().tolist()\n","\n","\t\tng_item = item[0].item() # leave one-out evaluation has only one item per user\n","\t\tHR.append(hit(ng_item, recommends))\n","\t\tNDCG.append(ndcg(ng_item, recommends))\n","\n","\treturn np.mean(HR), np.mean(NDCG)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"HWyCD7pLOxjq"},"source":["### Models\n","- Generalized Matrix Factorization\n","- Multi Layer Perceptron\n","- Neural Matrix Factorization"]},{"cell_type":"code","metadata":{"id":"aTQaitu7d1R3"},"source":["class Generalized_Matrix_Factorization(nn.Module):\n"," def __init__(self, args, num_users, num_items):\n"," super(Generalized_Matrix_Factorization, self).__init__()\n"," self.num_users = num_users\n"," self.num_items = num_items\n"," self.factor_num = args.factor_num\n","\n"," self.embedding_user = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num)\n"," self.embedding_item = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num)\n","\n"," self.affine_output = nn.Linear(in_features=self.factor_num, out_features=1)\n"," self.logistic = nn.Sigmoid()\n","\n"," def forward(self, user_indices, item_indices):\n"," user_embedding = self.embedding_user(user_indices)\n"," item_embedding = self.embedding_item(item_indices)\n"," element_product = torch.mul(user_embedding, item_embedding)\n"," logits = self.affine_output(element_product)\n"," rating = self.logistic(logits)\n"," return rating\n","\n"," def init_weight(self):\n"," pass"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7kSFzPlNd50f"},"source":["class Multi_Layer_Perceptron(nn.Module):\n"," def __init__(self, args, num_users, num_items):\n"," super(Multi_Layer_Perceptron, self).__init__()\n"," self.num_users = num_users\n"," self.num_items = num_items\n"," self.factor_num = args.factor_num\n"," self.layers = args.layers\n","\n"," self.embedding_user = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num)\n"," self.embedding_item = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num)\n","\n"," self.fc_layers = nn.ModuleList()\n"," for idx, (in_size, out_size) in enumerate(zip(self.layers[:-1], self.layers[1:])):\n"," self.fc_layers.append(nn.Linear(in_size, out_size))\n","\n"," self.affine_output = nn.Linear(in_features=self.layers[-1], out_features=1)\n"," self.logistic = nn.Sigmoid()\n","\n"," def forward(self, user_indices, item_indices):\n"," user_embedding = self.embedding_user(user_indices)\n"," item_embedding = self.embedding_item(item_indices)\n"," vector = torch.cat([user_embedding, item_embedding], dim=-1) # the concat latent vector\n"," for idx, _ in enumerate(range(len(self.fc_layers))):\n"," vector = self.fc_layers[idx](vector)\n"," vector = nn.ReLU()(vector)\n"," # vector = nn.BatchNorm1d()(vector)\n"," # vector = nn.Dropout(p=0.5)(vector)\n"," logits = self.affine_output(vector)\n"," rating = self.logistic(logits)\n"," return rating\n","\n"," def init_weight(self):\n"," pass"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7DQpVuaV9cF0"},"source":["class NeuMF(nn.Module):\n"," def __init__(self, args, num_users, num_items):\n"," super(NeuMF, self).__init__()\n"," self.num_users = num_users\n"," self.num_items = num_items\n"," self.factor_num_mf = args.factor_num\n"," self.factor_num_mlp = int(args.layers[0]/2)\n"," self.layers = args.layers\n"," self.dropout = args.dropout\n","\n"," self.embedding_user_mlp = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num_mlp)\n"," self.embedding_item_mlp = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num_mlp)\n","\n"," self.embedding_user_mf = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num_mf)\n"," self.embedding_item_mf = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num_mf)\n","\n"," self.fc_layers = nn.ModuleList()\n"," for idx, (in_size, out_size) in enumerate(zip(args.layers[:-1], args.layers[1:])):\n"," self.fc_layers.append(torch.nn.Linear(in_size, out_size))\n"," self.fc_layers.append(nn.ReLU())\n","\n"," self.affine_output = nn.Linear(in_features=args.layers[-1] + self.factor_num_mf, out_features=1)\n"," self.logistic = nn.Sigmoid()\n"," self.init_weight()\n","\n"," def init_weight(self):\n"," nn.init.normal_(self.embedding_user_mlp.weight, std=0.01)\n"," nn.init.normal_(self.embedding_item_mlp.weight, std=0.01)\n"," nn.init.normal_(self.embedding_user_mf.weight, std=0.01)\n"," nn.init.normal_(self.embedding_item_mf.weight, std=0.01)\n"," \n"," for m in self.fc_layers:\n"," if isinstance(m, nn.Linear):\n"," nn.init.xavier_uniform_(m.weight)\n"," \n"," nn.init.xavier_uniform_(self.affine_output.weight)\n","\n"," for m in self.modules():\n"," if isinstance(m, nn.Linear) and m.bias is not None:\n"," m.bias.data.zero_()\n","\n"," def forward(self, user_indices, item_indices):\n"," user_embedding_mlp = self.embedding_user_mlp(user_indices)\n"," item_embedding_mlp = self.embedding_item_mlp(item_indices)\n","\n"," user_embedding_mf = self.embedding_user_mf(user_indices)\n"," item_embedding_mf = self.embedding_item_mf(item_indices)\n","\n"," mlp_vector = torch.cat([user_embedding_mlp, item_embedding_mlp], dim=-1)\n"," mf_vector =torch.mul(user_embedding_mf, item_embedding_mf)\n","\n"," for idx, _ in enumerate(range(len(self.fc_layers))):\n"," mlp_vector = self.fc_layers[idx](mlp_vector)\n","\n"," vector = torch.cat([mlp_vector, mf_vector], dim=-1)\n"," logits = self.affine_output(vector)\n"," rating = self.logistic(logits)\n"," return rating.squeeze()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"tpBX6rqNfSc9"},"source":["### Setting Arguments\n","\n","Here is the brief description of important ones:\n","- Learning rate is 0.001\n","- Dropout rate is 0.2\n","- Running for 10 epochs\n","- HitRate@10 and NDCG@10\n","- 4 negative samples for each positive one"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Bc5Vg1Ik_gnF","executionInfo":{"elapsed":11,"status":"ok","timestamp":1633618730970,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"072e970d-c6d2-413c-d6f4-2f25e13ee4bf"},"source":["parser = argparse.ArgumentParser()\n","parser.add_argument(\"--seed\", \n","\ttype=int, \n","\tdefault=42, \n","\thelp=\"Seed\")\n","parser.add_argument(\"--lr\", \n","\ttype=float, \n","\tdefault=0.001, \n","\thelp=\"learning rate\")\n","parser.add_argument(\"--dropout\", \n","\ttype=float,\n","\tdefault=0.2, \n","\thelp=\"dropout rate\")\n","parser.add_argument(\"--batch_size\", \n","\ttype=int, \n","\tdefault=256, \n","\thelp=\"batch size for training\")\n","parser.add_argument(\"--epochs\", \n","\ttype=int,\n","\tdefault=10, \n","\thelp=\"training epoches\")\n","parser.add_argument(\"--top_k\", \n","\ttype=int, \n","\tdefault=10, \n","\thelp=\"compute metrics@top_k\")\n","parser.add_argument(\"--factor_num\", \n","\ttype=int,\n","\tdefault=32, \n","\thelp=\"predictive factors numbers in the model\")\n","parser.add_argument(\"--layers\",\n"," nargs='+', \n"," default=[64,32,16,8],\n"," help=\"MLP layers. Note that the first layer is the concatenation of user \\\n"," and item embeddings. So layers[0]/2 is the embedding size.\")\n","parser.add_argument(\"--num_ng\", \n","\ttype=int,\n","\tdefault=4, \n","\thelp=\"Number of negative samples for training set\")\n","parser.add_argument(\"--num_ng_test\", \n","\ttype=int,\n","\tdefault=100, \n","\thelp=\"Number of negative samples for test set\")\n","parser.add_argument(\"--out\", \n","\tdefault=True,\n","\thelp=\"save model or not\")"],"execution_count":null,"outputs":[{"data":{"text/plain":["_StoreAction(option_strings=['--out'], dest='out', nargs=None, const=None, default=True, type=None, choices=None, help='save model or not', metavar=None)"]},"execution_count":10,"metadata":{},"output_type":"execute_result"}]},{"cell_type":"markdown","metadata":{"id":"RnaRWy2gg_Nw"},"source":["### Training"]},{"cell_type":"code","metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"VyWquJG893CV","executionInfo":{"elapsed":2230853,"status":"ok","timestamp":1633621278281,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"61938a61-f7f5-4885-85d1-1e3e2d2a06f6"},"source":["# set device and parameters\n","args = parser.parse_args(args={})\n","device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","writer = SummaryWriter()\n","\n","# seed for Reproducibility\n","seed_everything(args.seed)\n","\n","# load data\n","ml_1m = pd.read_csv(\n","\tDATA_PATH, \n","\tsep=\"::\", \n","\tnames = ['user_id', 'item_id', 'rating', 'timestamp'], \n","\tengine='python')\n","\n","# set the num_users, items\n","num_users = ml_1m['user_id'].nunique()+1\n","num_items = ml_1m['item_id'].nunique()+1\n","\n","# construct the train and test datasets\n","data = NCF_Data(args, ml_1m)\n","train_loader = data.get_train_instance()\n","test_loader = data.get_test_instance()\n","\n","# set model and loss, optimizer\n","model = NeuMF(args, num_users, num_items)\n","model = model.to(device)\n","loss_function = nn.BCELoss()\n","optimizer = optim.Adam(model.parameters(), lr=args.lr)\n","\n","# train, evaluation\n","best_hr = 0\n","for epoch in range(1, args.epochs+1):\n","\tmodel.train() # Enable dropout (if have).\n","\tstart_time = time.time()\n","\n","\tfor user, item, label in train_loader:\n","\t\tuser = user.to(device)\n","\t\titem = item.to(device)\n","\t\tlabel = label.to(device)\n","\n","\t\toptimizer.zero_grad()\n","\t\tprediction = model(user, item)\n","\t\tloss = loss_function(prediction, label)\n","\t\tloss.backward()\n","\t\toptimizer.step()\n","\t\twriter.add_scalar('loss/Train_loss', loss.item(), epoch)\n","\n","\tmodel.eval()\n","\tHR, NDCG = metrics(model, test_loader, args.top_k, device)\n","\twriter.add_scalar('Perfomance/HR@10', HR, epoch)\n","\twriter.add_scalar('Perfomance/NDCG@10', NDCG, epoch)\n","\n","\telapsed_time = time.time() - start_time\n","\tprint(\"The time elapse of epoch {:03d}\".format(epoch) + \" is: \" + \n","\t\t\ttime.strftime(\"%H: %M: %S\", time.gmtime(elapsed_time)))\n","\tprint(\"HR: {:.3f}\\tNDCG: {:.3f}\".format(np.mean(HR), np.mean(NDCG)))\n","\n","\tif HR > best_hr:\n","\t\tbest_hr, best_ndcg, best_epoch = HR, NDCG, epoch\n","\t\tif args.out:\n","\t\t\tif not os.path.exists(MODEL_PATH):\n","\t\t\t\tos.mkdir(MODEL_PATH)\n","\t\t\ttorch.save(model, \n","\t\t\t\t'{}{}.pth'.format(MODEL_PATH, MODEL))\n","\n","writer.close()"],"execution_count":null,"outputs":[{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," cpuset_checked))\n"]},{"name":"stdout","output_type":"stream","text":["The time elapse of epoch 001 is: 00: 05: 41\n","HR: 0.626\tNDCG: 0.359\n","The time elapse of epoch 002 is: 00: 05: 42\n","HR: 0.658\tNDCG: 0.389\n","The time elapse of epoch 003 is: 00: 05: 47\n","HR: 0.664\tNDCG: 0.396\n","The time elapse of epoch 004 is: 00: 05: 34\n","HR: 0.669\tNDCG: 0.400\n","The time elapse of epoch 005 is: 00: 05: 44\n","HR: 0.671\tNDCG: 0.401\n","The time elapse of epoch 006 is: 00: 05: 44\n","HR: 0.672\tNDCG: 0.402\n","The time elapse of epoch 007 is: 00: 05: 39\n","HR: 0.668\tNDCG: 0.396\n","The time elapse of epoch 008 is: 00: 05: 34\n","HR: 0.667\tNDCG: 0.396\n","The time elapse of epoch 009 is: 00: 05: 41\n","HR: 0.668\tNDCG: 0.397\n","The time elapse of epoch 010 is: 00: 05: 37\n","HR: 0.664\tNDCG: 0.395\n"]}]},{"cell_type":"code","metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"fkiRJWeD_trR","executionInfo":{"elapsed":93,"status":"ok","timestamp":1633621278282,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"},"user_tz":-330},"outputId":"d3efcab5-fa0b-4938-d5ff-7967f38dab4d"},"source":["print(\"Best epoch {:03d}: HR = {:.3f}, NDCG = {:.3f}\".format(\n","\t\t\t\t\t\t\t\t\tbest_epoch, best_hr, best_ndcg))"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["Best epoch 006: HR = 0.672, NDCG = 0.402\n"]}]},{"cell_type":"markdown","metadata":{"id":"WOTaSMGnPoAG"},"source":["## MF with PyTorch on ML-100k\n","\n","Training Pytorch MLP model on movielens-100k dataset and visualizing factors by decomposing using PCA"]},{"cell_type":"code","metadata":{"id":"I2f_R0Yo6BUp"},"source":["!pip install -U -q git+https://github.com/sparsh-ai/recochef.git"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KXT07lHDBzAQ"},"source":["import torch\n","from torch import nn\n","from torch import optim\n","from torch.nn import functional as F \n","from torch.optim.lr_scheduler import _LRScheduler\n","\n","from recochef.datasets.movielens import MovieLens\n","from recochef.preprocessing.encode import label_encode\n","from recochef.utils.iterators import batch_generator\n","from recochef.models.embedding import EmbeddingNet\n","\n","import math\n","import copy\n","import pickle\n","import numpy as np\n","import pandas as pd\n","from textwrap import wrap\n","from sklearn.decomposition import PCA\n","from sklearn.model_selection import train_test_split\n","\n","import matplotlib.pyplot as plt\n","plt.style.use('ggplot')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"NINhOhYAxt5n"},"source":["### Data loading and preprocessing"]},{"cell_type":"code","metadata":{"id":"3Z4R3bXNjaNP"},"source":["data = MovieLens()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"A2Xgw-sXk7Ac","executionInfo":{"status":"ok","timestamp":1633622511935,"user_tz":-330,"elapsed":637,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"399404ea-51bd-476f-f74e-0dc4807ebaa9"},"source":["ratings_df = data.load_interactions()\n","ratings_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","0 196 242 3.0 881250949\n","1 186 302 3.0 891717742\n","2 22 377 1.0 878887116\n","3 244 51 2.0 880606923\n","4 166 346 1.0 886397596"]},"metadata":{},"execution_count":16}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":343},"id":"wIUc-Ba_6xBK","executionInfo":{"status":"ok","timestamp":1633622512590,"user_tz":-330,"elapsed":666,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"3f1382b8-d132-48b8-f85f-fae21140922d"},"source":["movies_df = data.load_items()\n","movies_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
01Toy Story (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Toy%20Story%2...0001110000000000000
12GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
23Four Rooms (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Four%20Rooms%...0000000000000000100
34Get Shorty (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Get%20Shorty%...0100010010000000000
45Copycat (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Copycat%20(1995)0000001010000000100
\n","
"],"text/plain":[" ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n","0 1 Toy Story (1995) 01-Jan-1995 ... 0 0 0\n","1 2 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n","2 3 Four Rooms (1995) 01-Jan-1995 ... 1 0 0\n","3 4 Get Shorty (1995) 01-Jan-1995 ... 0 0 0\n","4 5 Copycat (1995) 01-Jan-1995 ... 1 0 0\n","\n","[5 rows x 24 columns]"]},"metadata":{},"execution_count":17}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"X-IcaBzgmOrN","executionInfo":{"status":"ok","timestamp":1633623011681,"user_tz":-330,"elapsed":742,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"7a6afd50-b93a-498b-f9ea-96d7db363795"},"source":["ratings_df, umap = label_encode(ratings_df, 'USERID')\n","ratings_df, imap = label_encode(ratings_df, 'ITEMID')\n","ratings_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
0003.0881250949
1113.0891717742
2221.0878887116
3332.0880606923
4441.0886397596
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","0 0 0 3.0 881250949\n","1 1 1 3.0 891717742\n","2 2 2 1.0 878887116\n","3 3 3 2.0 880606923\n","4 4 4 1.0 886397596"]},"metadata":{},"execution_count":50}]},{"cell_type":"code","metadata":{"id":"36dsiSqWwNRz"},"source":["X = ratings_df[['USERID','ITEMID']]\n","y = ratings_df[['RATING']]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Mp-OqA2jyNAS","executionInfo":{"status":"ok","timestamp":1633622720665,"user_tz":-330,"elapsed":21,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"969eca89-0f0e-464d-d98e-70aeb215b99b"},"source":["for _x_batch, _y_batch in batch_generator(X, y, bs=4):\n"," print(_x_batch)\n"," print(_y_batch)\n"," break"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["tensor([[873, 377],\n"," [808, 601],\n"," [ 90, 354],\n"," [409, 570]])\n","tensor([[4.],\n"," [3.],\n"," [4.],\n"," [2.]])\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oQlnTmST0cx6","executionInfo":{"status":"ok","timestamp":1633622723064,"user_tz":-330,"elapsed":15,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"96db7cc7-9614-4215-f0e8-be69eb17e4e0"},"source":["_x_batch[:, 1]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([377, 601, 354, 570])"]},"metadata":{},"execution_count":24}]},{"cell_type":"markdown","metadata":{"id":"mVavggU2_WUY"},"source":["### Embedding Net"]},{"cell_type":"markdown","metadata":{"id":"D39WDxT5_f3l"},"source":["The PyTorch is a framework that allows to build various computational graphs (not only neural networks) and run them on GPU. The conception of tensors, neural networks, and computational graphs is outside the scope of this article but briefly speaking, one could treat the library as a set of tools to create highly computationally efficient and flexible machine learning models. In our case, we want to create a neural network that could help us to infer the similarities between users and predict their ratings based on available data."]},{"cell_type":"markdown","metadata":{"id":"9Gve8w7f_l8i"},"source":["The picture above schematically shows the model we're going to build. At the very beginning, we put our embeddings matrices, or look-ups, which convert integer IDs into arrays of floating-point numbers. Next, we put a bunch of fully-connected layers with dropouts. Finally, we need to return a list of predicted ratings. For this purpose, we use a layer with sigmoid activation function and rescale it to the original range of values (in case of MovieLens dataset, it is usually from 1 to 5)."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9zDzhH2c0Cv-","executionInfo":{"status":"ok","timestamp":1633622725269,"user_tz":-330,"elapsed":33,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"134739d6-e9ff-4cd7-a0b5-0aac999500fa"},"source":["netx = EmbeddingNet(\n"," n_users=50, n_items=20, \n"," n_factors=10, hidden=[500], \n"," embedding_dropout=0.05, dropouts=[0.5])\n","netx"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["EmbeddingNet(\n"," (u): Embedding(50, 10)\n"," (m): Embedding(20, 10)\n"," (drop): Dropout(p=0.05, inplace=False)\n"," (hidden): Sequential(\n"," (0): Linear(in_features=20, out_features=500, bias=True)\n"," (1): ReLU()\n"," (2): Dropout(p=0.5, inplace=False)\n"," )\n"," (fc): Linear(in_features=500, out_features=1, bias=True)\n",")"]},"metadata":{},"execution_count":25}]},{"cell_type":"markdown","metadata":{"id":"o4vQFZ6iwiyM"},"source":["### Cyclical Learning Rate (CLR)"]},{"cell_type":"markdown","metadata":{"id":"6RIaav5rwk66"},"source":["One of the `fastai` library features is the cyclical learning rate scheduler. We can implement something similar inheriting the `_LRScheduler` class from the `torch` library. Following the [original paper's](https://arxiv.org/abs/1506.01186) pseudocode, this [CLR Keras callback implementation](https://github.com/bckenstler/CLR), and making a couple of adjustments to support [cosine annealing](https://pytorch.org/docs/stable/optim.html#torch.optim.lr_scheduler.CosineAnnealingLR) with restarts, let's create our own CLR scheduler.\n","\n","The implementation of this idea is quite simple. The [base PyTorch scheduler class](https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html) has the `get_lr()` method that is invoked each time when we call the `step()` method. The method should return a list of learning rates depending on the current training epoch. In our case, we have the same learning rate for all of the layers, and therefore, we return a list with a single value. \n","\n","The next cell defines a `CyclicLR` class that expectes a single callback function. This function should accept the current training epoch and the base value of learning rate, and return a new learning rate value."]},{"cell_type":"code","metadata":{"id":"eYQh4ZCmmgW9"},"source":["class CyclicLR(_LRScheduler):\n"," \n"," def __init__(self, optimizer, schedule, last_epoch=-1):\n"," assert callable(schedule)\n"," self.schedule = schedule\n"," super().__init__(optimizer, last_epoch)\n","\n"," def get_lr(self):\n"," return [self.schedule(self.last_epoch, lr) for lr in self.base_lrs]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"1bpK5hOvw7Hg"},"source":["Our scheduler is very similar to [LambdaLR](https://pytorch.org/docs/stable/optim.html#torch.optim.lr_scheduler.LambdaLR) one but expects a bit different callback signature. \n","\n","So now we only need to define appropriate scheduling functions. We're createing a couple of functions that accept scheduling parameters and return a _new function_ with the appropriate signature:"]},{"cell_type":"code","metadata":{"id":"I6st2zPctj1T"},"source":["def triangular(step_size, max_lr, method='triangular', gamma=0.99):\n"," \n"," def scheduler(epoch, base_lr):\n"," period = 2 * step_size\n"," cycle = math.floor(1 + epoch/period)\n"," x = abs(epoch/step_size - 2*cycle + 1)\n"," delta = (max_lr - base_lr)*max(0, (1 - x))\n","\n"," if method == 'triangular':\n"," pass # we've already done\n"," elif method == 'triangular2':\n"," delta /= float(2 ** (cycle - 1))\n"," elif method == 'exp_range':\n"," delta *= (gamma**epoch)\n"," else:\n"," raise ValueError('unexpected method: %s' % method)\n"," \n"," return base_lr + delta\n"," \n"," return scheduler"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"k-CjYin0toWa"},"source":["def cosine(t_max, eta_min=0):\n"," \n"," def scheduler(epoch, base_lr):\n"," t = epoch % t_max\n"," return eta_min + (base_lr - eta_min)*(1 + math.cos(math.pi*t/t_max))/2\n"," \n"," return scheduler"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"oB_zTLW-wwdM"},"source":["To understand how the created functions work, and to check the correctness of our implementation, let's create a couple of plots visualizing learning rates changes depending on the number of epoch:"]},{"cell_type":"code","metadata":{"id":"Dl-TWx4OwwdN"},"source":["def plot_lr(schedule):\n"," ts = list(range(1000))\n"," y = [schedule(t, 0.001) for t in ts]\n"," plt.plot(ts, y)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"wCfhKAoMwwdN","executionInfo":{"status":"ok","timestamp":1633622731436,"user_tz":-330,"elapsed":1050,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"52bd67ed-e05a-4a07-9747-c8d68b1eae5a"},"source":["plot_lr(triangular(250, 0.005))"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzda2BU13no/f/aM+KiC4KRkGSwwEbC2AiMjAbQzUaXUZIGtyU9rtvYcRqgdfvGIUfmvDlxoW/cNuWUE2JwjEid06qkqXlTUjeQxrm4GoTA0iCQABkLjI0MGMsICzRCFyQkzex1PmwskCXQbUZ7Luv3xRaz9t7Pmj0zz8zaaz9LSCkliqIoinKTZnYAiqIoSmBRiUFRFEUZQCUGRVEUZQCVGBRFUZQBVGJQFEVRBlCJQVEURRnAanYAvnLp0qUxbRcfH8/Vq1d9HE1gU30OD6rP4WE8fZ41a9aQ/65+MSiKoigDqMSgKIqiDKASg6IoijKASgyKoijKACoxKIqiKAOMaFZSXV0du3btQtd1CgsLWb169YDH+/r6KCkp4dy5c8TExFBcXExCQgIAe/fupby8HE3TWLNmDenp6QA899xzTJkyBU3TsFgsbNmyBYDOzk62b9/OlStXmDlzJs8//zzR0dG+7LOiKIpyF8P+YtB1ndLSUjZu3Mj27dupqqqisbFxQJvy8nKioqLYsWMHq1atYvfu3QA0NjbicrnYtm0bmzZtorS0FF3X+7d78cUX2bp1a39SANi3bx+LFy/mlVdeYfHixezbt89XfVUURVFGYNjE0NDQQFJSEomJiVitVrKzs6mpqRnQpra2lry8PAAyMzOpr69HSklNTQ3Z2dlERESQkJBAUlISDQ0Ndz1eTU0NK1euBGDlypWDjqWYR3a0oVftR1VqV/xBSmm8vjrazA4l7A07lOR2u4mLi+v/Oy4ujrNnz96xjcViITIyko6ODtxuN/Pnz+9vZ7PZcLvd/X9v3rwZgKKiIhwOBwBtbW3MmDEDgOnTp9PWNvSLxOl04nQ6AdiyZQvx8fHD93YIVqt1zNsGq7H2uX3fT+j+1etMn7+ASQvT/RCZ/6jzHPh6T9fR+uMfMOWLTzDtzzaMaR/B1mdf8EefTbvz+bvf/S42m422tjb+7u/+jlmzZrFw4cIBbYQQCCGG3N7hcPQnE2DMd/6pOyVHRvb1oh/4LQDXfvUfaAn3+iM0v1HnOfDpv/oPALorfkvP43+MiJg06n0EW599wZQ7n202Gy0tLf1/t7S0YLPZ7tjG6/XS1dVFTEzMoG3dbnf/tp/+NzY2lmXLlvUPMcXGxtLa2gpAa2sr06ZNG3EnFf+RdUegqxNmzUHWViK7u8wOSQkhsrsLWVsJs+ZAVyfyRLXZIYW1YRNDSkoKTU1NNDc34/F4cLlc2O32AW0yMjKoqKgAoLq6mrS0NIQQ2O12XC4XfX19NDc309TURGpqKjdu3KC7uxuAGzducPLkSebMmQOA3W7n4MGDABw8eJBly5b5sr/KGMnKMrDNRPvqN6C3B1nzltkhKSFE1rwFvT3G68s203i9KaYZdijJYrGwdu1aNm/ejK7r5Ofnk5yczJ49e0hJScFut1NQUEBJSQnr168nOjqa4uJiAJKTk8nKymLDhg1omsa6devQNI22tja+//3vA8YvjNzc3P5prKtXr2b79u2Ul5f3T1dVzCVbmuHdtxGP/xHMWwD3JCOrnPDY580OTQkRssoJ9yTDvAWInELkG3uQVz9BxCeaHVpYEjJEppio6qojN9o+6//5U+Qb/4b29/+IiEtA/699yH//Z7S/KUHMmuPHSH1HnefAJS9dRH/xG4g/XIv2udXIlmb0v/wzxON/hPZ7T41qX8HSZ19S1VWVCSd1HenaDw8tQcQZNy2KrHywWNTPfcUnZGUZWCzG6wqM19lDS5BV+5G61+TowpNKDMrdnTkJLc2InFszwERMLCxZjqyuQHr6TAxOCXbS04esroAly43X1U0ixwHuK8brT5lwKjEodyUryyAyGvFI5oB/13KLoKMNTqobEJVxOFkDHW3G6+k24pFMiIxGVjpNCiy8qcSg3JG83oE8UY3IzBs8pzztEZgeh67euMo46JVOmB5nvJ5uIyImITLzkCcOIzvbTYoufKnEoNyRPHIQPH0DhpE+JTQLIrsA6o8jW1uG2FpR7k62tkD9cUR2AUKzDHpc5DjA40EeOWRCdOFNJQbljmRlGcxJQcyZN+TjIscB8ubFaUUZJenaD1If8osHYLzu5qQgK8tUfa4JphKDMiT54Qfw0XnEZ8Z+bycS7oEFi5FVTuRtVXMVZThSSuPehQWLjdfRHYjcImg8DxfPTWB0ikoMypBkVRlYIxDLH7trO5HjgCuX4ezpCYpMCQnvn4Irl+/4a+FTYvljYI1QU6MnmEoMyiCytwd55CBiaTYi6u6LJIml2TA1Ur1xlVGRlWUwNdJ4/dyFiIpGLM1GHjmI7O2ZoOgUlRiUQeSJaui6jsi9+7c5ADF5MmLZY8jjVciu6xMQnRLsZNd15PEqxLLHEJMnD9te5Dqg+7oqrDeBVGJQBpFVTohPhAWLR9Re5BZBb68qrKeMiFEwr/eu168GWLAY4hPVr9IJpBKDMoC8ctkomJdTiNBG+PK4LxVmz1VvXGVEZGUZzJ5rvG5GQGgaIqcQzpw0Xp+K36nEoAwgXeUgBCKrcMTbCCGMn/sXziIbL/gvOCXoycYLcOEsItdxx0W4hiKyCkEINTV6gqjEoPSTuhfpcsLCdETczFFtK1bkg8VqDEMpyh3IKidYrMbrZRRE3ExYmK4K600QlRiUW06/De6rg+rWjISImYZIX4GsPoDsU4X1lMFkXx+y+gAifQUiZvQrM2q5RdB6FU7X+SE65XYqMSj9ZJUTomNgyYoxbS9yHdDZASeP+jgyJSScPAqdHSOa7TakJSsgOkYV1psAw67gBlBXV8euXbvQdZ3CwkJWr1494PG+vj5KSko4d+4cMTExFBcXk5Bg1O7fu3cv5eXlaJrGmjVr+ldqA9B1nRdeeAGbzcYLL7wAwM6dOzl9+jSRkZEAPPfcc9x3332+6KtyF7KzHVlXjVj5O4iIiLHtZGE62OLRK8uwZOT4NkAl6OmVZWCLN14nYyAiIhAr8pAVv0F2tI/pV4cyMsP+YtB1ndLSUjZu3Mj27dupqqqisbFxQJvy8nKioqLYsWMHq1atYvfu3QA0NjbicrnYtm0bmzZtorS0FP220gm//vWvmT179qBjPvPMM2zdupWtW7eqpDBBZHUFeDwjn0I4BKOwXiGcOoF0X/FdcErQk+4rcOoEIrtwyIJ5IyVyi8DrQR6p8F1wyiDDJoaGhgaSkpJITEzEarWSnZ1NTc3AGvy1tbXk5eUBkJmZSX19PVJKampqyM7OJiIigoSEBJKSkmhoaACgpaWF48ePU1g48tkvin9IKY0phHNTEffeN659iexCkNKY3aQoN0lXOUhpvD7GQdx7H8xNVYX1/GzYoSS3201cXFz/33FxcZw9e/aObSwWC5GRkXR0dOB2u5k/f35/O5vNhtvtBuDHP/4xX/nKV+ju7h50zJ/+9Ke8/vrrLFq0iKeffpqIIYY2nE4nTqcx1rhlyxbi4+NH0t9BrFbrmLcNVp/tc1/Du7g//pCYP/8WkeN9LuLjaV2cgbf6AHFf/X9Gfi+En6nzbB6p67RUH8CyOIMZDy0a9/66vvAlOn60lenXrhAxf+GAxwKlzxPJH30e0TUGXzt27BixsbHMmzePU6dODXjsqaeeYvr06Xg8Hn70ox/xi1/8gieeeGLQPhwOBw7HrYtYY10MWy0eDvob/w4Rk7i+cCldPngu9OUrkaXbuFp1APHQknHvzxfUeTaPPHMS/ZNL6I//sU/ikQuXQsQkWn/1OtpXvj7gsUDp80QaT59nzZo15L8P+3XOZrPR0nJrIZaWlhZsNtsd23i9Xrq6uoiJiRm0rdvtxmaz8d5771FbW8tzzz3Hyy+/TH19Pa+88goAM2bMQAhBREQE+fn5/UNPin/Inh7k0UOIjGxEZJRP9imWZsHUKHVPgwJ8WjAvynhd+ICIjEJkZCOPHkL2qMJ6/jBsYkhJSaGpqYnm5mY8Hg8ulwu73T6gTUZGBhUVFQBUV1eTlpaGEAK73Y7L5aKvr4/m5maamppITU3lqaee4tVXX2Xnzp0UFxezaNEivvnNbwLQ2toK0H+NIjk52cddVm4nT7igu2tcF50/S0yajFixEnn8MLKr02f7VYKP7OpEHj+MWLESMWn4gnkjJXKLoLsLedzls30qtww7lGSxWFi7di2bN29G13Xy8/NJTk5mz549pKSkYLfbKSgooKSkhPXr1xMdHU1xcTEAycnJZGVlsWHDBjRNY926dWjDjDm/8sortLcba7zOnTuXZ5991gfdVO5EVjphZhLMT/PpfkWuA1nxa+PXSN4XfbpvJXjIo4egr3fs9y7cyQOLYGaS8as0a3R3USvDEzJELu1funRpTNuF85ikbG5C3/TniNVfQVv1pE+PIaVE/9tisFiw/NU2n+57LML5PJvJ+3cbwOtF+87Lo6qNNBL6r36G3Pca2uYf9a8CFwh9nmimXGNQQpes2g9CQ2QV+HzfRmG9IviwAfnReZ/vXwl88qPz8GEDIrfI50kBMF63QlPXsvxAJYYwZRTM2w9pjyBs/pneJ1Y8BlZVWC9cySonWK3G68APhC0e0h5BulRhPV9TiSFcnaqDay1jKpg3UiJ6GuKRLGR1hSqsF2aMgnkViEeyENH+K12h5RbBNTecOuG3Y4QjlRjClF5ZBtHTYMkyvx5H5DrgegeyTi3LGE5k3RG4Po6CeSO1ZBlETzNez4rPqMQQhvS2Vnj7KCIzH2EdY8G8kXpwCdhmqoqYYUZWloFtpnH+/UhYIxCZ+fD2UWRHm1+PFU5UYghD3QffBO/4CuaNVP+yjO/WIVua/X48xXyypRnerRvd8rDjYBTW8yIPH/D7scKFSgxhRkpJt/OXcP8DiNlzJuSYnxZOk1VqWcZw8GkBxfEWzBspMXsO3P+AKqznQyoxhJsLZ/F+dN7/Y7+3EfGJ8ODDN2eP6MNvoAQtqevGbKQHHzbO+wQRuQ5o+gjP2dMTdsxQphJDmJGVZTB5CmKZf6YQ3onILYKWZjhzckKPq0ywMyehpXlChilvJ5Y9BpMmG7+GlXFTiSGMfFowb0p2AWJq5IQeWzySCZHRRmJSQpasckJktHG+J5CYGonIyOFGpRPZc2NCjx2KVGIII/JYFdzoZmrhqgk/toiYZBTWO1GNvN4x4cdX/E9ev61gXsSkCT++yHUgu7uM17kyLioxhBFZVQYJ9xAxxjV3x0vkFoGnD3nkoCnHV/xLHqkAT9+EDyP1m5+G5Z5k9avUB1RiCBPyk0vw/ilEjsMvdWtGQsyZB3PmqRIZIUpWOWHOPOM8m0AIYfwaPnsaefljU2IIFSoxhAlZ5TQK5mX7vmDeaIjcIrh4DnnxA1PjUHxLXvwALp4z79fCTVPyf8corOdSXz7GQyWGMCC9XmNu+eIMxPS44TfwI7F8JVgj1M/9ECMry8AaYZxfE1lsM2FxBtJ1AOlVhfXGSiWGcHDqOLS50XIm7t6FOxFR0YilWcgjB5F9vWaHo/iA7OtFHjmIWJqFiIo2Oxzjdd7mhvrjZocStIZdwQ2grq6OXbt2oes6hYWFrF69esDjfX19lJSUcO7cOWJiYiguLiYhIQGAvXv3Ul5ejqZprFmzhvT0Wxc+dV3nhRdewGaz8cILLwDQ3NzMyy+/TEdHB/PmzWP9+vVYrSMKU7kDvbIMYmLhYf8WzBspkVtkrNd7cwaLEtzk8cPQdd30YaR+Dy+DmFj0yjIsfi4SGaqG/cWg6zqlpaVs3LiR7du3U1VVRWNj44A25eXlREVFsWPHDlatWsXu3bsBaGxsxOVysW3bNjZt2kRpaSn6bXe+/vrXv2b27NkD9vXaa6+xatUqduzYQVRUFOXl5b7oZ9iS7a1wsgaRVYAIlAS7YDHEJaiL0CFCVjkhLsE4rwFAWK3GIj7v1Bivf2XUhk0MDQ0NJCUlkZiYiNVqJTs7m5qamgFtamtrycvLAyAzM5P6+nqklNTU1JCdnU1ERAQJCQkkJSXR0NAAQEtLC8ePH6ew8FY9FSklp06dIjPTuDkmLy9v0LGU0ZHVFeD1TmgJjOEYhfUc8O7byKufmB2OMg7y6ifw7tvGbLcJKJg3UiLXcbOwXoXZoQSlYb9Cut1u4uJuXbCMi4vj7Nmzd2xjsViIjIyko6MDt9vN/Pnz+9vZbDbcbjcAP/7xj/nKV75Cd3d3/+MdHR1ERkZisVgGtf8sp9OJ02l849yyZQvx8WNbhcxqtY5520AnpaTl8AG0BYuwLX6k/98Doc/ex5/g6i9/ytQTh4n+8p/6/XiB0OeJNhF97izbx3UhiHv8CSwB8Pz29zk+HveCReiHy4l76k9Nm6I9Efxxnk0ZWzh27BixsbHMmzePU6dOjWkfDocDh+PWt+CxLoYdyouHyw/OoDdeQP/qNwb0MSD6LKzwUDrXnf9Jd+HvIjSLXw8XEH2eYP7us9S96M7/hIfSaRVWCIDn9/Y+6yvykD8p4erRKkTKgyZH5j/jOc+zZs0a8t+H/e1ns9loaWnp/7ulpQWbzXbHNl6vl66uLmJiYgZt63a7sdlsvPfee9TW1vLcc8/x8ssvU19fzyuvvEJMTAxdXV14b04z+7S9MjayynmzYF6u2aEMSeQWgfsqvKsK6wWld0+C+2rgXHT+DLEsFyZPUVOjx2DYxJCSkkJTUxPNzc14PB5cLhd2u31Am4yMDCoqKgCorq4mLS0NIQR2ux2Xy0VfXx/Nzc00NTWRmprKU089xauvvsrOnTspLi5m0aJFfPOb30QIQVpaGtXVxjKQFRUVg46ljIy80Y08+hbCnoOYMrEF80ZKpK+AqBj1xg1SsrIMomKM8xiAxJRIhD0HWVOJvNE9/AZKv2ETg8ViYe3atWzevJnnn3+erKwskpOT2bNnD7W1tQAUFBTQ2dnJ+vXreeONN3j66acBSE5OJisriw0bNrB582bWrVuHNswFqqeffpo33niD9evX09nZSUGBuXfqBit5zAU93QH7bQ5AREQgMvOQddXIznazw1FGQXa2I+uqEZl5iAg/Lw87DiK3CHq6VWG9URIyRJY8unTp0pi2C9WxZ+//fgE629D+9oeDLrwFUp9l43n0v/nviD/+M7TC3/XbcQKpzxPFn33W9/8S+W//iPbiDxD33u+XY4zFZ/sspUT/ztchehqWb/9vEyPzH1OuMSjBR15uhIbTphbMGylx7/0wN1UtyxhEpJTGMNLc1IBKCkMRQhhToxveRTY1Dr+BAqjEEJJk1X7QNOMmnyAgch3QeAFUYb3gcPEDaLwQUPfG3I3IKgBNUzdUjoJKDCFGer3Iw+Ww2I6InWF2OCMilj8GEZPUReggISvLIGKScd6CgIidAYvtyMPlSI/H7HCCgkoMoeadWmhrRQvgi86fJSI/Lax3CNnbY3Y4yl3I3h7kkUNGwbxI8wvmjZSWWwTt16D+mNmhBAWVGEKMXuWEadNhUYbZoYyKyC2C7utGQTYlYMnjh6E7gArmjdSiDJg23SgoqQxLJYYQItsCsGDeSD2wCGYmqeGkACcry2BmknG+gsitwnq1yGtDl9lRblGJIYTIw+Wg60FzUfB2QtMQ2YXw3jvI5iazw1GGIK9chvfeQWQXBlTBvJESuQ7QdWT1AbNDCXjBd3aVIUkpjVkXqQ8hku41O5wxEdmFN5dl3G92KMoQbi0PWzh84wAkku6F1IeQlU41NXoYKjGEig/ehcsfB9/Y722ELR7SHkG6ypG6WpYxkEj95vKwaY8Y5ylIidwi+ORjaHjX7FACmkoMIUJWlsHkqYiMHLNDGRct1wGtV+FUndmhKLc7XQetV43zE8RERg5MnoqsUtey7kYlhhAgb3Qha6sQy3IRU6aaHc74LFkO0dPQ1Rs3oOiVZRA9zTg/QUxMmYpYlousrULe6DI7nIClEkMIkDWV0HMjqIeRPiWsEYjMfKg7iuxoMzscBYzzUHcUkZmPsAZuwbyRMgrr3TDeN8qQVGIIAbLKCfckw7wFZofiE8ayjB5jWVLFdPJIBXg9QTnbbUjzFsA9yapExl2oxBDkZNNH8MGZoCiYN1Ji9ly4/wFVWC8AGAXznHD/A8Z5CQH9hfU+OGO8f5RBVGIIcrLSCRYLIivP7FB8SuQ44NJFuHB2+MaK/1xogI8/NM5HCBFZ+WCxGO8fZZAR3R5bV1fHrl270HWdwsJCVq9ePeDxvr4+SkpKOHfuHDExMRQXF5OQkADA3r17KS8vR9M01qxZQ3p6Or29vbz44ot4PB68Xi+ZmZk8+eSTAOzcuZPTp08TGWmsOvbcc89x3333+bDLoUN6PDcL5i1DTAuOgnkjJZY9ivzZPyErnYj7HzA7nLAlK8tg0iTEskfNDsWnxLTp8PAyo7Del54JvkoBfjbss6HrOqWlpfzVX/0VcXFx/OVf/iV2u5177711E1V5eTlRUVHs2LGDqqoqdu/ezfPPP09jYyMul4tt27bR2trKd7/7XX7wgx8QERHBiy++yJQpU/B4PHznO98hPT2dBx4wPgCeeeYZMjMz/dfrUPFOLXS0BVXBvJESkVGIjBxkzSHkk+sQkyebHVLYkT09yJpDiIwcRGSU2eH4nJZThH6iGk7WwNIss8MJKMMOJTU0NJCUlERiYiJWq5Xs7GxqamoGtKmtrSUvLw+AzMxM6uvrkVJSU1NDdnY2ERERJCQkkJSURENDA0IIpkyZAoDX68Xr9YbM+PhE0ivLINYGi5aaHYpfiJwi6O5SyzKaRB53QXeXcR5C0aKlEGszCk8qAwz7i8HtdhMXF9f/d1xcHGfPnr1jG4vFQmRkJB0dHbjdbubPn9/fzmaz4XYbBax0Xefb3/42ly9f5vOf//yAdj/96U95/fXXWbRoEU8//TQRQ6wp63Q6cTqNE7plyxbi48d2N6bVah3ztmbyuq9wtf4YkaufJiYxcVTbBkufZdxKWu65F+3oQWy/9+S49hUsffal8fbZfaQC/Z57icteGTRf3Ebb547CVXTt280MTWKxzfRjZP7jj9e2aQNrmqaxdetWrl+/zve//30uXrzInDlzeOqpp5g+fToej4cf/ehH/OIXv+CJJ54YtL3D4cDhuHVBbKxrngbrWsD6b/4DdJ0bS3PoGWX8wdRnPTMf795/5crpk4iEodenHYlg6rOvjKfPsvkS+qkTiC89Q0tLi48j85/R9lkuzYGf/ystb7yO9sU/9GNk/mPKms82m23AC6OlpQWbzXbHNl6vl66uLmJiYgZt63a7B20bFRVFWloadXVGCYQZM2YghCAiIoL8/HwaGhpG2MXw0T+F8IE0ROLYPyyDgcgqMArrqdkjE0pW7TcK5gXJ8rBjJRJnwQNpyCpVWO92wyaGlJQUmpqaaG5uxuPx4HK5sNvtA9pkZGRQUVEBQHV1NWlpaQghsNvtuFwu+vr6aG5upqmpidTUVNrb27l+/ToAvb29nDx5ktmzZwPQ2toK0H+NIjk52Zf9DQ1nT0PzpZCbQjgUMSMOFi01Zo94VWG9iSC9XqPC7aKlxvMf4kSOA5qb4Owps0MJGMMOJVksFtauXcvmzZvRdZ38/HySk5PZs2cPKSkp2O12CgoKKCkpYf369URHR1NcXAxAcnIyWVlZbNiwAU3TWLduHZqm0drays6dO9F1HSklWVlZZGQYK4698sortLe3AzB37lyeffZZP3Y/OMnKMpgS/AXzRkrLLUL/h7+HU8fh4WVmhxP6Th2Ha260L/+52ZFMCJGRg/zp/0FWliGCbAEifxEyRH4/Xbp0aUzbBdvYs+zuQv9//wSRmYf2zHNj2kfQ9dnTh/4/10LqQ1i+vnFM+wi2PvvCWPvs/Ye/h7On0b73z0FXG2msfdb/dSey+gDa93+CmBrph8j8x5RrDEpgkTVvQW9PWAwjfUpYI4w7VU/WINuvmR1OSJPt1+Dto4is0CiYN1IixwG9vciaQ2aHEhBUYggysrIMZs2BMLsbWOQ4wOtVyzL6mayuAK83rL54AMb7adYcNcnhJpUYgoj8+CKcfx+RWxQ088p9RcyaA/MWqGUZ/ciY7VYG8xYYz3cYEUIY5bjPv2+8z8KcSgxBRFaVgcWKyMwzOxRTiNwiaPoIzr1ndiih6dx70PRRSKzrMRYiMw8sViM5hjmVGIKE9PQhDx+AJcsRMbFmh2MKYc+FSZNVHX0/kVVOmDTZeJ7DkIiJhSXLkdUHkJ4+s8MxlUoMweJkDXS2B/2au+MhpkYi7LnIo28he26YHU5IkT03kDVvIey5QTcrx5e03CLobIe3a4ZvHMJUYggSeqUTpsdB2iNmh2IqY1nGbmStKqznS7K2Cm50h+0wUr+0dJgeZxSoDGMqMQQB2doC9ccR2YUIzWJ2OOZKfQgSZ6txYB+TVWWQONt4fsOY0CyI7EI4dQLpDq/7Xm6nEkMQkK79IHVETqHZoZiuf1nGhtPIyx+bHU5IkJc/hrOnQ2p52PEQOYUgdWMRrDClEkOAk7puXBRcsBiRcI/Z4QQEkZUPmqYuQvuIrHKCphnPq2K8zxYsNgrr6brZ4ZhCJYZAd/YUXLmMCOOLzp8lpttgsV0V1vMB6fXeXB7WbjyvCoDxfrtyOWwL66nEEOBkpROmRiIeyTY7lICi5TqgrRXqj5kdSnCrPwZtrWE9220o4pFsmBoZtteyVGIIYLLrOvJ4FWL5Y2rN489aZIdp08N+9sh46ZVOmDbdeD6VfmLyZMTyx5DHXMiu62aHM+FUYghgRsG8XjWFcAjCar1VWK+t1exwgpJsb4V3am4WzDNtMceAJXKLoK8XeTT8CuupxBDAZGUZzJ4Lc1PNDiUgiZwi0HVVWG+M5OEDNwvmqS8eQ5qbCrPnhuVwkkoMAUo2XoALZ8OyYN5IiXvuhZQHkZVlqrDeKPUvD5vyoPE8KoP0F9b7sAHZeN7scCbUiH4/1tXVsWvXLnRdp7CwkNWrVw94vK+vj5KSEs6dO0dMTAzFxcUkJNweGwgAACAASURBVCQAsHfvXsrLy9E0jTVr1pCenk5vby8vvvgiHo8Hr9dLZmYmTz75JADNzc28/PLLdHR0MG/ePNavX481DH/myionWMO3YN5Iidwi5L/sgA/OhP3NWaPywRm43Ij4k/VmRxLQRGYe8j9+jKx0Iv74z8wOZ8IM+4tB13VKS0vZuHEj27dvp6qqisbGxgFtysvLiYqKYseOHaxatYrdu3cD0NjYiMvlYtu2bWzatInS0lJ0XSciIoIXX3yRrVu38r3vfY+6ujref/99AF577TVWrVrFjh07iIqKorw8/G4ykX19yOoDiCUrENHTzA4noAl7LkyeEpY/98dDVpbB5ClhWzBvpET0NMSSFcjqCmRf+BTWGzYxNDQ0kJSURGJiIlarlezsbGpqBhaYqq2tJS8vD4DMzEzq6+uRUlJTU0N2djYREREkJCSQlJREQ0MDQgimTJkCgNfrxev1IoRASsmpU6fIzMwEIC8vb9CxwsLbR6CzQ110HgExZapRWK+2Enmjy+xwgoK80Y2srTQK5k2ZanY4AU/kFsH1DuN9GSaGHaNxu93ExcX1/x0XF8fZs2fv2MZisRAZGUlHRwdut5v58+f3t7PZbLjdbsD4JfLtb3+by5cv8/nPf5758+fT3t5OZGQkFotlUPvPcjqdOJ3Gna9btmwhPj5+NP3uZ7Vax7ytv7QePYQnPpH4RwsRFt/XRgrEPo9H7+N/SGuVk+gzJ5nqeHzINqHW55G4U5+7nW/Q3nOD6Y//IZNC7Dnxx3mWjxZydfc/YD16kBlfWD38BhPMH302bfBe0zS2bt3K9evX+f73v8/FixeZPn36iLd3OBw4HLduyhnrYtiBtki8dF9BrzuCWPUkLa3+mYYZaH0eLxmXBEn30v7bn3M9PXPINqHW55G4U5+9v/05JN1LW1wSIsSeE3+dZ5mZR++vfsaV995FxM30+f7HYzx9njVr1pD/PuxQks1mo6Wlpf/vlpYWbDbbHdt4vV66urqIiYkZtK3b7R60bVRUFGlpadTV1RETE0NXVxfem2UOhmof6oyCedKo8KiMiDF7xAEfnEE2fWR2OAFNNjXCB2cQuapg3miI7EKQEnl4v9mhTIhhE0NKSgpNTU00Nzfj8XhwuVzY7QPvkszIyKCiogKA6upq0tLSEEJgt9txuVz09fXR3NxMU1MTqamptLe3c/26cTdhb28vJ0+eZPbs2QghSEtLo7q6GoCKiopBxwplRsG8/fDgw4iZSWaHE1RUYb2RkVVlqmDeGIiZSfDgw8aa42FQWG/YoSSLxcLatWvZvHkzuq6Tn59PcnIye/bsISUlBbvdTkFBASUlJaxfv57o6GiKi4sBSE5OJisriw0bNqBpGuvWrUPTNFpbW9m5cye6riOlJCsri4yMDACefvppXn75Zf7t3/6N+++/n4KCAv8+A4HkvXfg6ieI1V8xO5KgI6bNgIeXIV3lyNXPqDt5hyA9HuOmtoeXGc+XMioitwj5Ty8Z79OHlpgdjl8JGSJ3Bl26dGlM2wXS2LP+jy8h62vRtv4YMcl/tZECqc++JN8+il7yd2hf34h4ZOC1hlDt8918ts+yrhp95/9C+8ZfIZYsNzEy//HneZa9Pejf+hpikR3tz/6HX44xFqZcY1AmhrzeiTzuQixf6dekENIWZUDsDHQ1nDQkvdIJsTOM50kZNTFpMmL5SuRxF/J6p9nh+JVKDAFCHj0Enj5178I4CIsFkVUA79Qirw09zTlcyWtueKcWkVXglynQ4ULkFoGnL+QL66nEECBklROS70fMTTE7lKAmchxGYb0wXpZxKPLwAdB14/lRxkzMTYHk+0P+TnuVGAKA/Og8fNigqlz6gEiaDfMXGrNHQuPy2bhJKY0vHvMXGs+PMi4ipwgufoC8eM7sUPxGJYYAICvLwBqByFxpdighQeQUQfMlOHva7FACQ8O78MnH6ouHj4jMlWCNCOmp0SoxmEz29SKrKxCPZCKiYswOJyQIew5MmRrSb9zRkJVlMGWq8bwo4yaiYhCPZN4srNdrdjh+oRKDyWTdEejqNO7cVXxCTJ6CWPaoUVivO7wL68nuLqNg3rJHEZOnmB1OyBC5DujqRJ6oNjsUv1CJwWSysgxsM+HB0L5hZqKJHAf09hjLo4YxWVsJvT3qorOvPbgE4hJC9lepSgwmki3N8O7biJxChKZOhU/NWwD3JIfsG3ekZGUZ3JNsPB+KzwhNM+onvfu28T4OMerTyESyyijIpb7N+V5/Yb1z7yEvXTQ7HFN4PjoP595TBfP8ROQYhS5D8cuHSgwmkbpuVFJ9aAkiLsHscEKSyMwHiyXk55zfSff+N8BiMZ4HxedEXAI8tARZtT/kCuupxGCWMyehpVn9WvAjMW06LFkedssyAkhPH90HfgNLlhvPg+IXIscB7itw5m2zQ/EplRhMIivLIDJ6ULE3xbe0HAd0tNFTW2V2KBPrZC2y/ZrRf8VvxCOZEBmNrAyt4SSVGEwgr3cgT1QjMvMQEZPMDie0pS2F6Ta69//S7EgmlF5ZhmaLN/qv+I2ImITIzEOeqEZe7zA7HJ9RicEE8shBo2Ce+jbnd8JiQWQX0nviCLK1ZfgNQoBsbYH640zN/6IqmDcBRI7DKKxXfdDsUHxGJQYTyMoymJOCmDPP7FDCgsgpNArrucJjWUZ5uBykzpTCx80OJSyIOfNgToqxOl6IGNEyV3V1dezatQtd1yksLGT16tUDHu/r66OkpIRz584RExNDcXExCQnGTJu9e/dSXl6OpmmsWbOG9PR0rl69ys6dO7l27RpCCBwOB1/84hcB+NnPfsb+/fuZNm0aAF/+8pdZujR0fg7Lix/AR+cRT/2F2aGEDZEwi4i0R+irciJ/54mQvmekv2DeA4uw3nMvhNniRGYRuUXI//9V5IcfhESF5GHfIbquU1paysaNG9m+fTtVVVU0NjYOaFNeXk5UVBQ7duxg1apV7N69G4DGxkZcLhfbtm1j06ZNlJaWous6FouFZ555hu3bt7N582befPPNAftctWoVW7duZevWrSGVFOC2gnnLHzM7lLAy1fE4XLkc+oX13j8FzU1qXY8JJpY/ZhTWC5Gp0cMmhoaGBpKSkkhMTMRqtZKdnU1NTc2ANrW1teTl5QGQmZlJfX09UkpqamrIzs4mIiKChIQEkpKSaGhoYMaMGcybZwyjTJ06ldmzZ+N2h/7CKrK3B3nkIGJpNiIq2uxwwsqUrHyYGhkyb9w7kVVlMDUSsTTb7FDCioiKRizNRh49iOztMTuccRt2KMntdhMXF9f/d1xcHGfPnr1jG4vFQmRkJB0dHbjdbubPn9/fzmazDUoAzc3NnD9/ntTU1P5/e/PNNzl06BDz5s3jq1/9KtHRgz9EnU4nTqcxRWzLli3Ex8ePpL+DWK3WMW87Wt1v/RftXdeJXfXfmDxBxxzKRPY5UFitVqY++jm6K36D7Rt/iRaCiVm/3smVYy6m5v0O02bPDtvzbFafe1f9N1qPHiS64RRTH/vchB3XH30e0TUGf7lx4wYvvfQSX/va14iMjATgc5/7HE888QQAe/bs4Sc/+Qlf//rXB23rcDhwOG7N6hnrYtgTuUi89zd7IT6R9qQ5CBPHfieyz4EiPj6eHnsu/Nc+rv52H9rKL5gdks/ph34LvT302HO5evVq2J5ns/osk+YY7+/f/JzrCyduCHw8fZ41a9aQ/z7sUJLNZqOl5dY0v5aWFmw22x3beL1eurq6iImJGbSt2+3u39bj8fDSSy/x6KOPsmLFiv4206dPR9M0NE2jsLCQDz74YBTdDFzyymVVMM9s982H2XNDdjhJVjph9lyjn8qEE5pmzIA7c9J4vwexYT+hUlJSaGpqorm5GY/Hg8vlwm63D2iTkZFBRUUFANXV1aSlpSGEwG6343K56Ovro7m5maamJlJTU5FS8uqrrzJ79mwef3zglLrW1tb+/z969CjJyck+6Kb5pKschEBkFZodStjqL6x34Syy8YLZ4fiU/PhDOP++KphnMpFdCEIE/dToYYeSLBYLa9euZfPmzei6Tn5+PsnJyezZs4eUlBTsdjsFBQWUlJSwfv16oqOjKS4uBiA5OZmsrCw2bNiApmmsW7cOTdM4c+YMhw4dYs6cOXzrW98Cbk1Lfe2117hw4QJCCGbOnMmzzz7r32dgAkjdi3Q5YWE6Im6m2eGENbEiH/n6vyCrnIg/+lOzw/EZWekEixWxQhXMM5OwzYSF6UjXfuTv/jFCC84bDIUMkRXTL126NKbtJmJMUtYfR//BX6P9+f9E2HP9eqyRCPexZ++rW+C9d9C+92NERITJkY2f9PShf2sNLFiE5S9e6P/3cD/PZpG1leg/+h7af/9rxCL/X2sw5RqDMn6yygnRMbBkxfCNFb/TcougswNOHjU7FN94+yh0thv9Usy3ZAVExwT1tSyVGPxMdrYj66oRK/JC4ttpSFiYDjPi0YP4jXs7vdIJM+KNfimmExERiBV5yLojyI52s8MZE5UY/ExWV4DHo+5EDSBCsyCyC+DUCaT7itnhjIt0X4VTJxDZBUE7nh2KRG4ReD3IIxVmhzImKjH4kZTS+Dk5NxVx731mh6PcRuQ4QEpjtlgQk679IHVVqTfAiHvvg7mpyMoygvEyrkoM/vRhA3z8ofq1EIDEzCRYsBhZ5QzaZRn7l4ddsNjojxJQRG4RfPyh8TkQZFRi8CNZ5YSISapgXoASuUVw9RN47x2zQxmb9+vhymX1xSNAieWPQcSkoLwIrRKDnxgF8w4hMrIRkVFmh6MMQSzNgqlRRgIPQrLKCVOjjH4oAUdERiEyspFHDyF7gquwnkoMfiKPu6D7uvo2F8DEpMmIFY8hjx9GdnWaHc6oyK5O5DEXYsVjiEmTzQ5HuQORWwTdXcgTLrNDGRWVGPxEVjphZhLMTzM7FOUuRG4R9PUijx4yO5RRkUcPQV+v+uIR6B5YBDOTjM+DIKISgx/I5iZ47x1EjkMVzAt0c1Lg3vuC7o0rK51w731G/ErAEkIYM8bee8f4XAgS6lPLD2TVfhAaIqvA7FCUYRiF9YrgwwbkR+fNDmdEZON5+LABkVukCuYFAaOwnmZ8LgQJlRh8zCiYtx/SHkHYwmuRlGAlVqwEqzVoLkLLSidYrUbcSsATM+Jg0VKjsJ7uNTucEVGJwddO1cG1FlW3JoiI6GmI9ExkdQWyr8/scO5K9vUhqysQ6ZmI6Glmh6OMkJbjgGstxudDEFCJwcf0yjKIngZLlpkdijIKIrcIrncg66rNDuWuZN0RuN6hLjoHmyXLICY2aOpzqcTgQ7KjDd4+isjMR1hVwbyg8tDDYJsZ8BehZWUZ2GYa8SpBQ1gjEJl58PZR43MiwI1ozee6ujp27dqFrusUFhayevXqAY/39fVRUlLCuXPniImJobi4mISEBAD27t1LeXk5mqaxZs0a0tPTuXr1Kjt37uTatWsIIXA4HHzxi18EoLOzk+3bt3PlyhVmzpzJ888/T3R0cCzcLqsrwKsK5gUjo7BeIfJXe5AtzYi4BLNDGkS2XIF36xCr/kgVzAtCIqcIWfYLYyiw6PfNDueuhv3FoOs6paWlbNy4ke3bt1NVVUVjY+OANuXl5URFRbFjxw5WrVrF7t27AWhsbMTlcrFt2zY2bdpEaWkpuq5jsVh45pln2L59O5s3b+bNN9/s3+e+fftYvHgxr7zyCosXL2bfvn1+6Lbv9RfMu/8BxOw5ZoejjIHIMZZdDdTZI58uF/lpnEpwEbPnwP0PBEVhvWETQ0NDA0lJSSQmJmK1WsnOzqampmZAm9raWvLy8gDIzMykvr4eKSU1NTVkZ2cTERFBQkICSUlJNDQ0MGPGDObNmwfA1KlTmT17Nm63G4CamhpWrjRmW6xcuXLQsQLWhbNw6aKxprASlER8Ijz48M3ZI4FVWE/qujFr6sGHjTiVoCRyHXDpIpx/3+xQ7mrYoSS3201cXFz/33FxcZw9e/aObSwWC5GRkXR0dOB2u5k/f35/O5vN1p8APtXc3Mz58+dJTU0FoK2tjRkzZgAwffp02tqGHo9zOp04ncZ48JYtW4iPH9vUUKvVOuZtb9f+76V0T55C/Be+hBbgtZF81edgMtI+d//Ol2jf9tdMa/qQyQE0gaDnZC3XWpqZ9idfZ+oIz506z4FH/8KXuPKzf2bysUqmLc/xyT790ecRXWPwlxs3bvDSSy/xta99jcjIyEGPCyHueAOPw+HA4bj17Xysa576Yo1Y2dODfui/EEuzcXd1Q1f3uPbnb4GwLu5EG2mfZeoiiIyi7Vevo82+fwIiGxn9V69DZBSdqYu4PsJzp85zYBJLs+k+9F/0/N7TiMlTxr0/U9Z8ttlstLS09P/d0tKCzWa7Yxuv10tXVxcxMTGDtnW73f3bejweXnrpJR599FFWrLi1FnJsbCytra0AtLa2Mm1a4M/Vlseq4Ea3GkYKASJiEmLFSuSJauT1DrPDAUBe70QeP4xYsRIRMcnscJRxErkOuNGNPBa4hfWGTQwpKSk0NTXR3NyMx+PB5XJht9sHtMnIyKCiogKA6upq0tLSEEJgt9txuVz09fXR3NxMU1MTqampSCl59dVXmT17No8//viAfdntdg4ePAjAwYMHWbYscH7O34msKoOEe1TBvBAhcovA04c8ctDsUACQRw+Cp0/NdgsV89MgYZbxuRGghh1KslgsrF27ls2bN6PrOvn5+SQnJ7Nnzx5SUlKw2+0UFBRQUlLC+vXriY6Opri4GIDk5GSysrLYsGEDmqaxbt06NE3jzJkzHDp0iDlz5vCtb30LgC9/+cssXbqU1atXs337dsrLy/unqwYy+ckleP8U4kvPqLo1IULMSYE584yLvQWPD7+Bn8nKMpgzz4hLCXpGfS4H8uc/QX5yCZE49HCOmYQM9HlTI3Tp0qUxbTfeMUn95z9B/vbnaN8rRUyPG36DABAM47C+Nto+6+VvIH/6f9D+v+2mfiDLix+gf/d5xJefRRtlklLnOXDJay3o/3Md4gt/gPYHXx3Xvky5xqDcmfR6jcXkF2cETVJQRkasyANrhOnLMhoF8yKMeJSQIabHweIMpKsc6Q28wnoqMYzHqePQ5jYKZCkhRURFI5ZmIY8cRPb1mhKD7OtFHjmIWJqFiAqOu/+VkdNyi6DNDfXHzQ5lEJUYxkGvLIOYWHg48C+QK6MnchzQdR15/LApx5cnqqGr04hDCT2L7UZhvQC8CK0SwxjJ9lY4WYPIKkBYTb0dRPGXBx+GuATT1mmQlWUQl2DEoYQcYbUai3mdrDE+TwKISgxjZBTM86p7F0KY0DTj2/q7byOvfjKhx5ZXP4EzJ9XysCFO5DrA60UerjA7lAHUK24MjIJ5Tkh5EHFPstnhKH5kLMsoJrywXn/BvGxVMC+UiXuSIeVBZJUzoArrqcQwFufeg6aP1NhvGBBxM+GhdKTLOWHLMkrdaySih9KN4yshTeQ4oOkj43MlQKjEMAayygmTpyCW5ZodijIBRK4D3Ffh3ZMTc8AzJ8F9RQ1ThgmxLBcmTwmoNcdVYhgleaMbefQthD0HMWVw4T8l9Ij0TIiKmbB7GmSlE6JijOMqIU9MiUTYc5BH30LeCIwCnCoxjJI85oKeblW3JoyICGNZRllXjexs9+uxZGc78sRhRGYeIkItDxsuRG4R9HQbBTkDgEoMoyQryyBpNqQ8ZHYoygQSOQ7wePxeWE8eOQQej7p+FW5SHoKk2QGz5rhKDKMgLzdCw2ljCqEqmBdWRPL9MDfVr8sy9i8POzfVOJ4SNoQQxpeBhtPG54zJVGIYBVm1HzTNuClFCTsi1wGNF+DiB/45wMUPoPG8uugcpkRWAWhaQPxqUIlhhKTXizxcDovtiNgZZoejmEAsfwwiJvntIrSsdELEJOM4StgRsTNgsR15uBzp8Zgai0oMI/VOLbS1GoWvlLAkIj8trHcI2dvj033L3p5bBfMiVcG8cKXlFkH7Nag/Zm4cph49iOhVTpg2HRZlmB2KYiKRWwTdvi+sJ48fhu7rarZbuFtsh9gZRoFOE42o+ltdXR27du1C13UKCwtZvXr1gMf7+vooKSnh3LlzxMTEUFxcTEJCAgB79+6lvLwcTdNYs2YN6enpAPzwhz/k+PHjxMbG8tJLL/Xv62c/+xn79+/vX+v505XdzCTbbhbMK1qtCuaFuwcWQXyiMZyUmeez3coqJ8QnGvtXwpawWBCZ+ciyfci2VtOGrYf9xaDrOqWlpWzcuJHt27dTVVVFY+PAq+bl5eVERUWxY8cOVq1axe7duwFobGzE5XKxbds2Nm3aRGlpKbquA5CXl8fGjRuHPOaqVavYunUrW7duNT0pAMa1BV1XFwWVW4X13nsH2dzkk33KK5dVwTyln8h1gK4bnzsmGfZV2NDQQFJSEomJiVitVrKzs6mpqRnQpra2lry8PAAyMzOpr69HSklNTQ3Z2dlERESQkJBAUlISDQ0NACxcuJDo6MAfS5VSGt/mUh9CJN1rdjhKABDZBUZhPZdvCutJ134QwtivEvZE0r2QutDUwnrDjou43W7i4m4tWxkXF8fZs2fv2MZisRAZGUlHRwdut5v58+f3t7PZbLjd7mGDevPNNzl06BDz5s3jq1/96pAJxOl04nQa07q2bNlCfHz8sPsditVqveu2ve+epPXyx0z7xp8wdYzHCDTD9TkU+bTP8fG0pq/AU11B3Jr1CItlzLuSXi9XDx8gIn0FMx7w7U2T6jwHr+4vrKa95H8Re7WJSQ/dfT0Of/Q54AbMP/e5z/HEE08AsGfPHn7yk5/w9a9/fVA7h8OBw3FraGesi2EPt5C2/qvXYfJUOhcs4XoQLDI+EsGyYLov+brPcsVK9BPVXD3kRCwe+4QEWX8MvaUZzx+u8fk5Uec5eMkFS2DyVK796t/RZs66a9vx9HnWrKH3PexQks1mo6Wlpf/vlpYWbDbbHdt4vV66urqIiYkZtK3b7R607WdNnz4dTdPQNI3CwkI++MBPNxONgLzRhaytRCzLRUyZalocSgB6eDlEx4x7WUZZ6YToGGN/inKTmDIVsSwXWVuFvNE14ccfNjGkpKTQ1NREc3MzHo8Hl8uF3W4f0CYjI4OKigoAqqurSUtLQwiB3W7H5XLR19dHc3MzTU1NpKam3vV4ra23lrg7evQoycnmLYQjayqh54aaQqgMYhTWy4e6o8iOtjHtQ3a0I+uOIDLzVcE8ZRCjsN4N43Nogg07lGSxWFi7di2bN29G13Xy8/NJTk5mz549pKSkYLfbKSgooKSkhPXr1xMdHU1xcTEAycnJZGVlsWHDBjRNY926dWg3Z128/PLLnD59mo6ODv7iL/6CJ598koKCAl577TUuXLiAEIKZM2fy7LPP+vcZuAtZ5YR7kmHeAtNiUAKXyC1COv8TWV2BKPr9UW8vjxwAr0d98VCGNm8B3JNsfA49+rkJPbSQgbSe3DhcunRpTNvdaXxONn2E/p3nEE+sQfv8l8YbXkAJlXHY0fBXn72b/wf09qD99Y5RFVaUUqL/zTchYhKWTS8Nv8EYqPMc/PQ39yJf34X2tzvvuIywKdcYwpWsdILFgsjKMzsUJYCJ3CK4dBEunB2+8e0uNMDHH6pfC8pdiax8sFgmvLCeSgxDkB7PzYJ5yxDTVME85c7Eskdh0qRRv3FlZRlMmmRsryh3IKZNh4eXTXhhPZUYhvJOLXS0qYJ5yrBEZBRiaQ6y5hCyZ2SF9WRPD7LmEGJpDiIyys8RKsFOyymCjjbjc2mijjlhRwoiemUZxNpgkfnlOJTAZxTW6xrxsozyuAu6u9QwkjIyi5ZCrG1CC+upxPAZ8loLvHMMkZ0/rjtalTDyQBrMTDJmj4yArHLCzCRjO0UZhrBYjHIp7xwzPp8mgEoMnyEPHwCpI3LUtzllZPqXZXy/Htl899lxsrkJ3ntHLQ+rjIrIcYDUjc+nCaASw22MNXed8EAaIvHut6Eryu1EdiGI4ZdllFVOEJrRXlFGSCTOggfSkJUTU1hPJYbbnT0NzZeM7KwooyBmxMGipcbsEa93yDZS9xqVVBctNdoryiiInCJovmR8TvmZSgy3kZVlMGUqIiPH7FCUIKTlOuCaG04dH7rBqRNwzW20U5RREhnZMGWq39Ycv51KDDfJm7NKxPLHEJOnmB2OEoweXgYxsXecPaJXlkFMrNFOUUZJTJ6CWP4Y8lgVstu/hfVUYrhJ1rwFvT1qGEkZM2GNQGTmwckaZPu1AY/JjjZ4+ygiMw9hVQXzlLEROQ7o7TE+r/xIJYabZGUZzJoD9z9gdihKEBO5ReD1IqsHzh6Rhw+A16vuXVDG5/4HYNYcvw8nqcQAyI8vwvn3EblFagqhMi5i1hyYt2DA7BFjtlsZzFtgPK4oYySEML5cnH/f+NzyE5UYAFlVBharMQygKOMkchzQ9BGce8/4h/PvQ9NHaphS8QmRmQcWq/G55SdhnxhkX5/xM3/JckRMrNnhKCHAKKw3uf9OaKNg3mRVME/xCRETC0uWIw8fQHr6/HKMsE8MPbVV0NmuphAqPiOmRiIycpBH30K2X0PWvIXIyEFMjTQ7NCVEaLlF0NkOb9f4Zf/DruAGUFdXx65du9B1ncLCQlavXj3g8b6+PkpKSjh37hwxMTEUFxeTkJAAwN69eykvL0fTNNasWUN6ejoAP/zhDzl+/DixsbG89NKthUo6OzvZvn07V65cYebMmTz//PNER0f7qr+DdO//JUyPg7RH/HYMJfyI3CLk4XL0f3oJbnSri86Kb6Wlw/Q49ConfP73fL77YX8x6LpOaWkpGzduZPv27VRVVdHY2DigTXl5OVFRUezYsYNVq1axe/duABobG3G5XGzbto1NmzZRWlqKrusA5OXlsXHjxkHH27dvH4sXL+aVV15h8eLF7Nu3zxf9HJJsbaH3Xe3qiwAACSBJREFUxBFEdiFCUwXzFB+avxASZsG7bxv/nb/Q7IiUECI0i1FWpf443pYrPt//sImhoaGBpKQkEhMTsVqtZGdnU1Mz8OdLbW0teXl5AGRmZlJfX4+UkpqaGrKzs4mIiCAhIYGkpCQaGhoAWLhw4ZC/BGpqali5ciUAK1euHHQsX5Ku/aDriBxVt0bxLWP2iDE8KXJVwTzF90ROIUTH4Gm84PN9DzuU5Ha7iYu7VdclLi6Os2fP3rGNxWIhMjKSjo4O3G438+fP729ns9lwu913PV5bWxszZhirpk2fPp22trYh2zmdTpxO4+Leli1biI+PH64rg3TfOwdP0e8Ss3DxqLcNZlardUzPVzAzo8/6l57ieu8Nor70FFr0tAk9NqjzHPLi45G7fknE5ClM9vHqbiO6xmAWIcQdv2k5HA4cjlsXjMe0GPaSTOILHw+pxcNHItQWTB8J0/r8+1+h50Yv3Jj4Y6vzHB7G0+dZs4auIj3sUJLNZqOl5dbiEC0tLdhstju28Xq9dHV1ERMTM2hbt9s9aNvPio2NpbW1FYDW1lamTZv4b1qKoijhbNjEkJKSQlNTE83NzXg8HlwuF3a7fUCbjIwMKioqAKiuriYtLQ0hBHa7HZfLRV9fH83NzTQ1NZGamnrX49ntdg4ePAjAwYMHWbZMFRxTFEWZSEKOYNWH48eP8y//8i/ouk5+fj5/8Ad/wJ49e0hJScFut9Pb20tJSQnnz58nOjqa4uJiEhMTAfj5z3/OgQMH0DSNr33tazzyiDEt9OWX/2979xfSVP/HAfx9mCktbfOc0MiKmtmFhgkpSlCmRhdREF4IRRde5kqR6GLdRDcRBMtBTraL0PCuixTsoiBMQ0SYfzFXy8xCeKrljsqZU6fb57kQx3OenueHS5/Obzuf15U7Dvx89mZ+zvnu6NcBr9cLRVFgMplQU1ODyspKKIqCpqYmzM7OxnW76h9//O+ds/4NX3rqA/esD9xzfP5tKWlTgyER8GDYPO5ZH7hnfdDkMwbGGGP6woOBMcaYCg8GxhhjKjwYGGOMqSTNh8+MMca2h+6vGGw2m9Yl/Hbcsz5wz/rwX/Ss+8HAGGNMjQcDY4wxFcPdu3fval2E1iwWi9Yl/Hbcsz5wz/qw3T3zh8+MMcZUeCmJMcaYCg8GxhhjKv/XG/X810ZHR9Ha2opoNIqqqipcunRJ65K2bHZ2Fk6nE/Pz8xAEAWfPnsX58+cRDAbR1NSEHz9+qP5rLRGhtbUVIyMjSEtLg9VqTdg12mg0CpvNBlEUYbPZ4Pf74XA4oCgKLBYL6uvrkZKSgtXVVTQ3N+PTp0/IyMhAY2MjsrKytC4/bouLi3C5XJiZmYEgCKirq8O+ffuSOufnz5+ju7sbgiDgwIEDsFqtmJ+fT6qcW1paMDw8DJPJBLvdDgC/9P7t6enBs2fPAADV1dWx7Zc3hXQqEonQjRs36Nu3b7S6ukq3bt2imZkZrcvaMlmWaWpqioiIQqEQNTQ00MzMDLW3t1NHRwcREXV0dFB7ezsREQ0NDdG9e/coGo2Sz+ej27dva1b7VnV1dZHD4aD79+8TEZHdbqe+vj4iInK73fTy5UsiInrx4gW53W4iIurr66OHDx9qU/AWPXr0iF69ekVERKurqxQMBpM650AgQFarlVZWVohoPd/Xr18nXc4TExM0NTVFN2/ejB2LN1dFUej69eukKIrq683S7VLSx48fsXfvXmRnZyMlJQUnT56Ex+PRuqwty8zMjJ0x7Ny5Ezk5OZBlGR6PB+Xl5QCA8vLyWK+Dg4M4ffo0BEHA0aNHsbi4GNtBL5EEAgEMDw+jqqoKAEBEmJiYQFlZGQDgzJkzqp43zp7Kysrw9u1bUILdgxEKhfDu3TtUVlYCWN/reNeuXUmfczQaRTgcRiQSQTgchtlsTrqc8/Pzf9qDJt5cR0dHUVhYiPT0dKSnp6OwsBCjo6ObrkG3S0myLEOSpNhjSZIwOTmpYUXbz+/3Y3p6GkeOHMHCwgIyMzMBAGazGQsLCwDWX4e/bp4uSRJkWY49N1G0tbXh6tWrWFpaAgAoigKj0QiDwQBgfftZWZYBqLM3GAwwGo1QFCWhtpH1+/3YvXs3Wlpa8OXLF1gsFtTW1iZ1zqIo4uLFi6irq0NqaiqOHz8Oi8WS1DlviDfXv/9+++vrshm6vWJIdsvLy7Db7aitrYXRaFR9TxAECIKgUWXbb2hoCCaTKSHXzH9VJBLB9PQ0zp07hwcPHiAtLQ2dnZ2q5yRbzsFgEB6PB06nE263G8vLy3GdBSeL35Grbq8YRFFEIBCIPQ4EAhBFUcOKts/a2hrsdjtOnTqF0tJSAIDJZMLc3BwyMzMxNzcXO2sSRVG1+1Mivg4+nw+Dg4MYGRlBOBzG0tIS2traEAqFEIlEYDAYIMtyrK+N7CVJQiQSQSgUQkZGhsZdxEeSJEiShLy8PADrSyWdnZ1JnfP4+DiysrJiPZWWlsLn8yV1zhvizVUURXi93thxWZaRn5+/6Z+n2yuG3NxcfP36FX6/H2tra+jv70dxcbHWZW0ZEcHlciEnJwcXLlyIHS8uLkZvby8AoLe3FyUlJbHjb968ARHhw4cPMBqNCbW8AABXrlyBy+WC0+lEY2Mjjh07hoaGBhQUFGBgYADA+h0aG/meOHECPT09AICBgQEUFBQk3Jm12WyGJEmxLW3Hx8exf//+pM55z549mJycxMrKCogo1nMy57wh3lyLioowNjaGYDCIYDCIsbExFBUVbfrn6fovn4eHh/HkyRNEo1FUVFSgurpa65K27P3797hz5w4OHjwYexNcvnwZeXl5aGpqwuzs7E+3uz1+/BhjY2NITU2F1WpFbm6uxl38uomJCXR1dcFms+H79+9wOBwIBoM4fPgw6uvrsWPHDoTDYTQ3N2N6ehrp6elobGxEdna21qXH7fPnz3C5XFhbW0NWVhasViuIKKlzfvr0Kfr7+2EwGHDo0CFcu3YNsiwnVc4OhwNerxeKosBkMqGmpgYlJSVx59rd3Y2Ojg4A67erVlRUbLoGXQ8GxhhjP9PtUhJjjLF/xoOBMcaYCg8GxhhjKjwYGGOMqfBgYIwxpsKDgTHGmAoPBsYYYyp/AkyDqgXSPnXJAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"UPXizw35wwdO","executionInfo":{"status":"ok","timestamp":1633622732435,"user_tz":-330,"elapsed":29,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"674c0c79-c01c-41d7-a10d-466fb3dbefd1"},"source":["plot_lr(triangular(250, 0.005, 'triangular2'))"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3xU1bnw8d/aM+GSC4FJSCIYVBK8EFAkgyQhSm5oK/aUtlaPolagpSrFE6hWBN9jz/HwaVqEIASL2hRvVLFWsNpWyxADJmMkASMGFAkXIRIMZEIIBMhk9nr/GA1GLrnNZM9lff/RZNbe+1mZkGf22ms9S0gpJYqiKIryNc3oABRFURTfohKDoiiK0o5KDIqiKEo7KjEoiqIo7ajEoCiKorSjEoOiKIrSjtnoADzl4MGD3TouOjqaI0eOeDga36b6HBxUn4NDT/o8ZMiQc35f3TEoiqIo7ajEoCiKorSjEoOiKIrSjkoMiqIoSjsqMSiKoijtdGpWUmVlJatWrULXdbKzs5kyZUq7151OJwUFBezZs4eIiAhyc3OJiYkBYO3atRQVFaFpGtOmTWPMmDEAzJo1i379+qFpGiaTiby8PACOHz9Ofn4+hw8fZvDgwcyZM4fw8HBP9llRFEW5gA7vGHRdp7CwkPnz55Ofn09paSk1NTXt2hQVFREWFsby5cuZPHkyq1evBqCmpga73c6SJUtYsGABhYWF6Lredtzjjz/OokWL2pICwLp16xg9ejTLli1j9OjRrFu3zlN9VRRFUTqhw8RQXV1NXFwcsbGxmM1m0tLSKC8vb9emoqKCjIwMAFJSUqiqqkJKSXl5OWlpaYSEhBATE0NcXBzV1dUXvF55eTkTJ04EYOLEiWddSzGObGpEL92AqtSuKIGtw6Ekh8NBVFRU29dRUVHs2rXrvG1MJhOhoaE0NTXhcDgYMWJEWzuLxYLD4Wj7euHChQBMmjSJnJwcABobGxk0aBAAAwcOpLGx8Zxx2Ww2bDYbAHl5eURHR3fc23Mwm83dPtZfdbfPx9a9yMl/vM7AEVfQZ+QYL0TmPep9Dg6qzx46p0fP1gVPPPEEFouFxsZG/u///o8hQ4YwcuTIdm2EEAghznl8Tk5OWzIBur3yT62U7BzpbEF/7x0Ajv7jb2gxF3sjNK9R73NwUH3umm6vfLZYLNTX17d9XV9fj8ViOW8bl8tFc3MzERERZx3rcDjajv3mv5GRkYwbN65tiCkyMpKGhgYAGhoaGDBgQKc7qXiPrPwQmo/DkGHIihLkyWajQ1IUxUs6TAwJCQnU1tZSV1dHa2srdrsdq9Xark1ycjLFxcUAlJWVkZSUhBACq9WK3W7H6XRSV1dHbW0tiYmJnDp1ipMnTwJw6tQptm3bxrBhwwCwWq1s3LgRgI0bNzJu3DhP9lfpJlmyHiyD0e75FbScRpa/b3RIiqJ4SYdDSSaTienTp7Nw4UJ0XSczM5P4+HjWrFlDQkICVquVrKwsCgoKmD17NuHh4eTm5gIQHx9Pamoqc+fORdM0ZsyYgaZpNDY28uSTTwLuO4z09PS2aaxTpkwhPz+foqKitumqirFkfR18+jHiltth+BVwUTyy1AY33GR0aIqieIGQATLFRFVX7byu9ln/+yvIt19F+91ziKgY9H+vQ/71z2j/U4AYMsyLkXqOep+Dg+pz16jqqkq3SF1H2jfAVdcgotyLFkVqJphM7uElRVECjkoMyoV9tg3q6xATzswAExGRcM11yLJiZKvTwOAURfEGlRiUC5Il6yE0HHFtSrvva+mToKkRtqkFiIoSaFRiUM5LnmhCflSGSMlAhPRp/2LStTAwCr3EZkxwiqJ4jUoMynnJDzdCq7PdMNI3hGZCpGVB1VZkQ/05jlYUxV+pxKCclyxZD8MSEMOGn/N1MSEH5NcPpxVFCRgqMSjnJL/YDQf2ItInnbeNiLkIrhiNLLUhv1U1V1EU/6YSg3JOsnQ9mEMQ191wwXZiQg4cPgS7dvRSZIqieJtKDMpZZMtp5IcbEWPTEGEX3iRJjE2D/qFqTYOiBBCVGJSzyI/KoPkEIv3sh87fJfr2RYy7Abm1FNl8oheiUxTF21RiUM4iS20QHQtXjO5Ue5E+CVpaVGE9RQkQKjEo7cjDh9wF8yZkI7RO/npcmghDL1HDSYoSIFRiUNqR9iIQApGa3eljhBDuYad9u5A1+7wXnKIovUIlBqWN1F1Iuw1GjkFEDe7SsWJ8JpjM7mEoRVH8mkoMyhk7PgbHEXcdpC4SEQMQY8Yjy95DOlVhPUXxZyoxKG1kqQ3CI+Ca8d06XqTnwPEm2LbZw5EpitKbOtzBDaCyspJVq1ah6zrZ2dlMmTKl3etOp5OCggL27NlDREQEubm5xMS4a/evXbuWoqIiNE1j2rRpbTu1Aei6zrx587BYLMybNw+AFStWsGPHDkJDQwGYNWsWl156qSf6qlyAPH4MWVmGmPh9REhI904ycgxYotFL1mNKnuDZABVF6TUdJgZd1yksLOSxxx4jKiqKRx99FKvVysUXX9zWpqioiLCwMJYvX05paSmrV69mzpw51NTUYLfbWbJkCQ0NDTzxxBM89dRTaF/PdvnnP//J0KFD2/Z//sbdd99NSkr7Ms+Kd8myYmhtvWAJjI64C+tlI//xGtJxGGHp2nMKRVF8Q4dDSdXV1cTFxREbG4vZbCYtLY3y8vY1+CsqKsjIyAAgJSWFqqoqpJSUl5eTlpZGSEgIMTExxMXFUV1dDUB9fT1bt24lO7vzs18U75BSuqeaXpKIuPjSHp1LpGWDlO7ZTYqi+KUO7xgcDgdRUVFtX0dFRbFr167ztjGZTISGhtLU1ITD4WDEiBFt7SwWCw6HA4Dnn3+eu+6666y7BYBXXnmF119/nVGjRjF16lRCzjG0YbPZsNncM2Dy8vKIjo7uTH/PYjabu32sv/pun53Vn+L48gsifvkwoT39WURH0zA6GVfZe0Tdc3/n10J4mXqfg4Pqs4fO6dGzddKWLVuIjIxk+PDhbN++vd1rd955JwMHDqS1tZVnnnmGN998k1tvvfWsc+Tk5JCTc6ZkQ3c3w1abh4P+9l8hpA8nRo6l2QM/C/26icjCJRwpfQ9x1TU9Pp8nqPc5OKg+d82QIUPO+f0OP85ZLBbq689sxFJfX4/FYjlvG5fLRXNzMxEREWcd63A4sFgs7Ny5k4qKCmbNmsXSpUupqqpi2bJlAAwaNAghBCEhIWRmZrYNPSneIU+fRm7ehEhOQ4SGeeScYmwq9A9TaxoUxU91mBgSEhKora2lrq6O1tZW7HY7Vqu1XZvk5GSKi4sBKCsrIykpCSEEVqsVu92O0+mkrq6O2tpaEhMTufPOO1m5ciUrVqwgNzeXUaNG8eCDDwLQ0NAA0PaMIj4+3sNdVr5NfmSHk809euj8XaJPX8T4icitHyCbj3vsvIqi9I4Oh5JMJhPTp09n4cKF6LpOZmYm8fHxrFmzhoSEBKxWK1lZWRQUFDB79mzCw8PJzc0FID4+ntTUVObOnYumacyYMaNtRtL5LFu2jGPHjgFwySWXMHPmTA90UzkfWWKDwXEwIsmj5xXpOcjif7rvRjJu9ui5FUXxLiGllEYH4QkHDx7s1nHBPCYp62rRF/wSMeUutMm3efQaUkr0/80FkwnTY0s8eu7uCOb3OZioPndNt58xKIFLlm4AoSFSszx+bndhvUnwRTXywF6Pn19RFO9RiSFIuQvmbYCkaxEW70zvE+NvALMqrKco/kYlhmC1vRKO1nerYF5nifABiGtTkWXFqrCeovgRlRiClF6yHsIHwDXjvHodkZ4DJ5qQlWVevY6iKJ6jEkMQ0hsb4OPNiJRMhLmbBfM668prwDLYPftJURS/oBJDEDq58V1w9axgXmcJTUNMyIZPK5H1dV6/nqIoPacSQ5CRUnLS9hZcdjli6LBeuaZIcxdKlKUbeuV6iqL0jEoMwWbfLlwH9rrH/nuJiI6FK69G2jcgdb3XrqsoSveoxBBkZMl66NsPMe6GXr2uSJ8E9XXw2bZeva6iKF2nEkMQ+aZgXr+0LET/0F69trg2BULD1ZoGRfEDKjEEEbmlFE6dpH/25F6/tgjpc6aw3glVWE9RfJlKDEFElq6HmIsIGTmm48ZeINInQasT+WGxIddXFKVzVGIIEvKrg/D5dsSEHIQQhsQghg2HYcPVcJKi+DiVGIKELLW5C+aleb5gXleI9Emwfw9y/25D41AU5fxUYggC0uVC2otgdDJiYFTHB3iRuG4imEPcs6MURfFJKjEEg+1bodGBNqH31i6cjwgLR4xNRX64EelsMTocRVHOocMd3AAqKytZtWoVuq6TnZ3NlClT2r3udDopKChgz549REREkJubS0xMDABr166lqKgITdOYNm0aY8acefCp6zrz5s3DYrEwb948AOrq6li6dClNTU0MHz6c2bNnYzZ3KkzlPPSS9RARCVd7t2BeZ4n0ScjNm5BbP0CMn2h0OIqifEeHdwy6rlNYWMj8+fPJz8+ntLSUmpqadm2KiooICwtj+fLlTJ48mdWrVwNQU1OD3W5nyZIlLFiwgMLCQvRvrXz95z//ydChQ9ud6+WXX2by5MksX76csLAwioqKPNHPoCWPNcC2ckRqFsJXEuwVoyEqRj2EVhQf1WFiqK6uJi4ujtjYWMxmM2lpaZSXl7drU1FRQUZGBgApKSlUVVUhpaS8vJy0tDRCQkKIiYkhLi6O6upqAOrr69m6dSvZ2dlt55FSsn37dlJSUgDIyMg461pK18iyYnC5erUERkfchfVy4NOPkUe+MjocRVG+o8OPkA6Hg6ioMw8so6Ki2LVr13nbmEwmQkNDaWpqwuFwMGLEiLZ2FosFh8MBwPPPP89dd93FyZMn215vamoiNDQUk8l0Vvvvstls2GzuT5x5eXlER3dvFzKz2dztY32dlJL6D95Du2IUltHXtn3fF/rsuuVWjrz1Cv0/+oDwO37u9ev5Qp97m+pzcPBGnw0ZW9iyZQuRkZEMHz6c7du3d+scOTk55OSc+RTc3c2wA3nzcLn7M/Safej3/KpdH32iz8IMV43hhO3vnMz+AUIzefVyPtHnXqb6HBx60uchQ4ac8/sdDiVZLBbq6+vbvq6vr8disZy3jcvlorm5mYiIiLOOdTgcWCwWdu7cSUVFBbNmzWLp0qVUVVWxbNkyIiIiaG5uxuVytWuvdI8stX1dMC/d6FDOSaRPAscR+FQV1lMUX9JhYkhISKC2tpa6ujpaW1ux2+1YrdZ2bZKTkykuLgagrKyMpKQkhBBYrVbsdjtOp5O6ujpqa2tJTEzkzjvvZOXKlaxYsYLc3FxGjRrFgw8+iBCCpKQkysrc20AWFxefdS2lc+Spk8jN7yOsExD9erdgXmeJMeMhLEKtaVAUH9PhUJLJZGL69OksXLgQXdfJzMwkPj6eNWvWkJCQgNVqJSsri4KCAmbPnk14eDi5ubkAxMfHk5qayty5c9E0jRkzZqBpF85FU6dOZenSpbz66qtcdtllZGUZu1LXX8ktdjh9sld2aesuERKCSMlAbvwX8vgxRPgAo0NSFAUQUkppdBCecPDgwW4dF6hjkq7fz4PjjWj/+/RZtZF8qc+yZi/6//wX4j9/gZb9A69dx5f63FtUn4ODIc8YFP8jD9VA9Q5DC+Z1lrj4MrgkEVmyngD5jKIofk8lhgAkSzeApiFS/WMYTqTnQM0+UIX1FMUnqMQQYKTLhfygCEZbEZGDjA6nU8R1N0BIH/UQWlF8hEoMgeaTCmhsQPPhh87fJUK/Kay3Cdly2uhwFCXoqcQQYPRSGwwYCKOSjQ6lS0T6JDh5Arn1A6NDUZSgpxJDAJGNPlgwr7MuHwWD49RwkqL4AJUYAoj8oAh03acK5nWW0DREWjbs/ARZV2t0OIoS1FRiCBBSSncJjMSrEHEXGx1Ot4i0bBAa0r7B6FAUJaipxBAodn8Kh7706ZXOHRGWaEi6FmkvQuouo8NRlKClEkOAkCXroW9/RPIEo0PpES09BxqOwI5Ko0NRlKClEkMAkKeakRWliHHpiH79jQ6nZ665DsIHuLcjVRTFECoxBABZXgKnT/n1MNI3hDkEkZIJlZuRTY1Gh6MoQUklhgAgS21wUTwMv8LoUDxCpOeAqxX5YbHRoShKUFKJwc/J2gOw+zO/KJjXWWLoJXDZ5cgSmyqspygGUInBz8kSG5hMiNQMo0PxKDEhB778Avbt6rixoige1anlsZWVlaxatQpd18nOzmbKlCntXnc6nRQUFLBnzx4iIiLIzc0lJiYGgLVr11JUVISmaUybNo0xY8bQ0tLC448/TmtrKy6Xi5SUFG677TYAVqxYwY4dOwgNde86NmvWLC699FIPdjlwyNbWrwvmjUMM8I+CeZ0lxl2PfO1PyBIb4rLLjQ5HUYJKh4lB13UKCwt57LHHiIqK4tFHH8VqtXLxxWcWURUVFREWFsby5cspLS1l9erVzJkzh5qaGux2O0uWLKGhoYEnnniCp556ipCQEB5//HH69etHa2sr//3f/82YMWO4/HL3H4C7776blJQU7/U6UHxSAU2NflUwr7NEaBgieQKyfBPythmIvn2NDklRgkaHQ0nV1dXExcURGxuL2WwmLS2N8vLydm0qKirIyMgAICUlhaqqKqSUlJeXk5aWRkhICDExMcTFxVFdXY0Qgn79+gHgcrlwuVwBMz7em/SS9RBpgVFjjQ7FK8SESXCyGbml1OhQFCWodHjH4HA4iIqKavs6KiqKXbt2nbeNyWQiNDSUpqYmHA4HI0aMaGtnsVhwOByA+07kkUce4dChQ9x0003t2r3yyiu8/vrrjBo1iqlTpxISEnJWXDabDZvNBkBeXh7R0dFd6Xcbs9nc7WON5HIc5kjVFkKnTCUiNrZLx/pLn2XUROovuhht80Ys/3Fbj87lL332JNXn4OCNPhtWglPTNBYtWsSJEyd48skn2b9/P8OGDePOO+9k4MCBtLa28swzz/Dmm29y6623nnV8Tk4OOTlnisV1d89Tf90jVv/X30DXOTV2Aqe7GL8/9VlPycS19iUO79iGiDn3/rSd4U999hTV5+BgyJ7PFouF+vr6tq/r6+uxWCznbeNyuWhubiYiIuKsYx0Ox1nHhoWFkZSURGWluwTCoEGDEEIQEhJCZmYm1dXVnexi8JBSumcjXZ6EiO3+H0t/IFKz3IX1SmxGh6IoQaPDxJCQkEBtbS11dXW0trZit9uxWq3t2iQnJ1NcXAxAWVkZSUlJCCGwWq3Y7XacTid1dXXU1taSmJjIsWPHOHHiBAAtLS1s27aNoUOHAtDQ0ADQ9owiPj7ek/0NDLt2QN1B95TOACcGRcGoscgPipAuVVhPUXpDh0NJJpOJ6dOns3DhQnRdJzMzk/j4eNasWUNCQgJWq5WsrCwKCgqYPXs24eHh5ObmAhAfH09qaipz585F0zRmzJiBpmk0NDSwYsUKdF1HSklqairJye4dx5YtW8axY8cAuOSSS5g5c6YXu++fZMl66Of/BfM6S0ufhP7H38H2rXD1OKPDUZSAJ2SALC09ePBgt47ztzFJebIZ/aGfIVIy0O6e1a1z+F2fW53ov5kOI0Ziuv/Rbp3D3/rsCarPwcGQZwyKb5Hl70PL6aAYRvqGMIcgUjPh483IY0eNDkdRAp5KDH5GlqyHIcMgyFYDiwk54HIhy94zOhRFCXgqMfgR+eV+2Ps5In1S0C0IFEOGwfArVGE9RekFKjH4EVm6HkxmREqG0aEYQqRPgtoDsGen0aEoSkBTicFPyFYn8oP34JrrEBGRRodjCGFNhz593ftPKIriNSox+Itt5XD8mHtP5CAl+ocirOnIze8jT58yOhxFCVgqMfgJvcQGA6Mg6VqjQzGUSJ8Ep08iK1RhPUXxFpUY/IBsqIeqrYi0bIRmMjocYyVeBbFD3bOzFEXxCpUY/IC0bwCpIyZkGx2K4YQQ7qmr1TuQh740OhxFCUgqMfg4qevuh61XjEbEXGR0OD5BpGaCpqmH0IriJSox+Lpd2+HwIUQQP3T+LjHQAqOtqrCeoniJSgw+TpbYoH8o4to0o0PxKVp6DjQ2QNUWo0NRlICjEoMPk80nkFtLEdfdoPY8/q5RVhgw0L29qaIoHqUSgw9zF8xrcU/RVNoRZrP7WcO2cmRjg9HhKEpAUYnBh8mS9TD0Ergk0ehQfJKYMAl0XRXWUxQPU4nBR8mafbBvV1AWzOsscdHFkHAlsmS9KqynKB7U4Q5uAJWVlaxatQpd18nOzmbKlCntXnc6nRQUFLBnzx4iIiLIzc0lJiYGgLVr11JUVISmaUybNo0xY8bQ0tLC448/TmtrKy6Xi5SUFG677TYA6urqWLp0KU1NTQwfPpzZs2djNncqzIAiS21gDt6CeZ0l0ichX1gOuz9zL35TFKXHOrxj0HWdwsJC5s+fT35+PqWlpdTU1LRrU1RURFhYGMuXL2fy5MmsXr0agJqaGux2O0uWLGHBggUUFhai6zohISE8/vjjLFq0iD/84Q9UVlby+eefA/Dyyy8zefJkli9fTlhYGEVFRV7otm+TTiey7D3ENeMR4QOMDsenCWs69O2nVkIrigd1mBiqq6uJi4sjNjYWs9lMWloa5eXl7dpUVFSQkZEBQEpKClVVVUgpKS8vJy0tjZCQEGJiYoiLi6O6uhohBP369QPA5XLhcrkQQiClZPv27aSkpACQkZFx1rWCwscfwvEm9dC5E0S//u7CehUlyFPNRoej9IB0tqC//jzyyy+MDiXodThG43A4iIqKavs6KiqKXbt2nbeNyWQiNDSUpqYmHA4HI0aMaGtnsVhwOByA+07kkUce4dChQ9x0002MGDGCY8eOERoaislkOqv9d9lsNmw298rXvLw8oqOju9LvNmazudvHekvD5k20RscSfX02wuT52ki+2OeeaLnlpzSU2gj/bBv9c245Z5tA63Nn+Fufj634HSdtbyE++gDLk6vQIrp+t+xvffYEb/TZsMF7TdNYtGgRJ06c4Mknn2T//v0MHDiw08fn5OSQk3NmNXB3N8P2tc3DpeMweuWHiMm3Ud/gnWmYvtbnnpJRcRB3McfeeYMTY1LO2SbQ+twZ/tRnvdSGtL2FsKajf1TG4UWPof3qMYTWtfkx/tRnT+lJn4cMGXLO73f4U7dYLNTX17d9XV9fj8ViOW8bl8tFc3MzERERZx3rcDjOOjYsLIykpCQqKyuJiIigubkZ19dlDs7VPtC5C+ZJRJoqmNdZQgh3yZDdnyFrDxgdjtJFcv8e5OqVcOXViF/8GnH7DPikAvmv140OLWh1mBgSEhKora2lrq6O1tZW7HY7Vqu1XZvk5GSKi4sBKCsrIykpCSEEVqsVu92O0+mkrq6O2tpaEhMTOXbsGCdOnACgpaWFbdu2MXToUIQQJCUlUVZWBkBxcfFZ1wpk7oJ5G9z/QAbHGR2OX1GF9fyTbD6OvjIPwsLRfvEQQjMhMm5GXHcD8s2/ID/92OgQg1KHQ0kmk4np06ezcOFCdF0nMzOT+Ph41qxZQ0JCAlarlaysLAoKCpg9ezbh4eHk5uYCEB8fT2pqKnPnzkXTNGbMmIGmaTQ0NLBixQp0XUdKSWpqKsnJyQBMnTqVpUuX8uqrr3LZZZeRlZXl3Z+AL9n5CRz5CjHlLqMj8TtiwCC4ehzSXoSccjciCKc4+xspJfqqp8BxGO2hhYgB7qFkIQTcPQt5YC/6c0+iPZaPsATXcwOjCRkgK4MOHjzYreN8aUxSf24xsqoCbdHziD7eq43kS332JPnxZvSC/0ObNR/xnWcNgdrnC/H1Puvv/A35txcQt89Ay/nhWa/L2hr0hb+Giy9xJw5zSIfn9PU+e4MhzxiU3iFPHEdutSOum+jVpBDQRiVD5CD3NqiKT5M7q5BvvIRInoDI/o9zthEXXYz42Wz3s6PXn+/dAIOcSgw+Qm7eBK1OtXahB4TJhEjNcj+4PHruac6K8eRRB/qzf4DYixA/m33Bki/auHRE9g+QG95CLy/pxSiDm0oMPkKW2iD+MsQlCUaH4tfEhBx3Yb0PVGE9XyRbW91J4dRJtPseRfQP7fAYceu97ppYLyxH1tZ02F7pOZUYfIA8sBe+qHZXC1V6RMQNhREjkaU2VVjPB8m1L8GuHYi7ZyGGDuvUMcIcgjbzNxASgv7H3yFPnfRylIpKDD5AlqwHcwgiZaLRoQQEMWESfPUl7NphdCjKt8itHyD/vRaR8X20LhaHFJZotF88BIdqkC89rZK+l6nEYDDpbEGWFSOuTUGERRgdTkAQ1gnQr79a0+BD5FcH0Z9/Ci4dgbjt5906hxg5BvHDqcjNG5HF//JwhMq3qcRgMFn5ITQfd6/cVTxC9O2HGHe9u7DeSVVYz2jy9Gn0P/4ONBPafY8gQjqedno+4vu3wmgrcs2fkHt2ejBK5dtUYjCYLFkPlsFw5TVGhxJQxIQcaDnt3h5VMYyUErn6j3BwP9rP5yKiYnp0PqFpaDPmwEAL+jO/RzYd81CkyrepxGAgWV8Hn36MmJDd5WJhSgeGXwEXxavhJIPJ9/+N/KAIMfl2xKhkj5xThEWg3T8Pjh1F/9NipO7yyHmVM9RfIwPJ0g3A159uFY9qK6y3Zyfy4H6jwwlK8otq5CvPwshrET+43aPnFpckIu6YCTs+Qr69xqPnVlRiMIzUdXcl1auu6fHttXJuIiUTTCa1u5sB5Ikm9D/mwYBItJ//GqF5fl8Rcf1NiNRM5NtrkFVbPH7+YKYSg1E+2wb1depuwYvEgIFwzXXIsmKk02l0OEFD6jp6YT4cdaD98hFENzbc6QwhBGLqAzBkGPqfliDrD3vlOsFIJQaDyJL1EBqOuPbcG8sonqFNyIGmRk5XlBodStCQ/3odPqlA3D4DMfwKr15L9O2Ldv+joLvcD6OdLV69XrBQicEA8kQT8qMyREoGIqSP0eEEtqSxMNDCyQ1vGx1JUJCffox88y+I625AZNzcK9cUsUPQ7n0Q9n5O05+X9co1A51KDAaQH250F8xTw0heJ0wmRFo2LR+VIRvqOz5A6TbZUI/+3JMQN9Rd8uICxfE8TYxNQ9w4hZPvvJ9NuAoAACAASURBVIH+4cZeu26gUonBALJkPQxLQAwbbnQoQUFMyHYX1rNvMDqUgCVbW9Gf+T20nEa7fx6iX/9ej0H86B5CRl6DfLEA+aWaidYTndrmqrKyklWrVqHrOtnZ2UyZMqXd606nk4KCAvbs2UNERAS5ubnExLhn2qxdu5aioiI0TWPatGmMGTOGI0eOsGLFCo4ePYoQgpycHG6+2X3b+dprr7FhwwYGDHA/sLrjjjsYO3asJ/tsKLl/NxzYi7jzPqNDCRoiZgghSdfiLLUhb/5pr36SDRbyb8/D7s8QMx9GXBRvSAzCbCby109wZM496Ct/h7ZgMaJfx9VblbN1eMeg6zqFhYXMnz+f/Px8SktLqalpX/q2qKiIsLAwli9fzuTJk1m9ejUANTU12O12lixZwoIFCygsLETXdUwmE3fffTf5+fksXLiQd999t905J0+ezKJFi1i0aFFAJQX4VsG8624wOpSg0j/nFjh8CD7fbnQoAUcvL0Ha/o7I/gHauOsNjcVkiUab+TB8VYt8frkqttdNHSaG6upq4uLiiI2NxWw2k5aWRnl5ebs2FRUVZGRkAJCSkkJVVRVSSsrLy0lLSyMkJISYmBji4uKorq5m0KBBDB/uHkbp378/Q4cOxeEI/I1VZMtp5Icb3eOhYeFGhxNU+qVmQv9QtabBw2RtDfKF5ZBwpXvfBB8grhiN+NHdyC2lyA1vGR2OX+pwKMnhcBAVFdX2dVRUFLt27TpvG5PJRGhoKE1NTTgcDkaMGNHWzmKxnJUA6urq2Lt3L4mJiW3fe/fdd9m0aRPDhw/nnnvuITz87D+iNpsNm81d7iAvL4/o6O5tFm42m7t9bFedfP/fHGs+QeTkn9C3l655Lr3ZZ19hNpvpf/2NnCz+F5ZfPYoWBInZ2++zPHWS+ucWIfr2JWpeHqZo4xdqftNneddMGmv2cPr1VQwYM44+V442OjSv8cb73KlnDN5y6tQpFi9ezL333ktoqHss8MYbb+TWW28FYM2aNbz44os88MADZx2bk5NDTs6ZWT3d3Qy7NzcPd/1rLUTHcixuGMLADcuDdcP009Z0+Pc6jryzDm3i94wOyeu8+T5LKZF/WoKs2YeW+z80oIEP/E59u89y6v2wdxcNv38U7f8tdS94DEA9eZ+HDBlyzu93OJRksViorz8zza++vh6LxXLeNi6Xi+bmZiIiIs461uFwtB3b2trK4sWLuf766xk/fnxbm4EDB6JpGpqmkZ2dze7du7vQTd8lDx9SBfOMdukIGHqJGk7yAFn8L+TmjYj/uBMxcozR4ZyTCA1Hu28enDiuiu11UYd/oRISEqitraWuro7W1lbsdjtWq7Vdm+TkZIqLiwEoKysjKSkJIQRWqxW73Y7T6aSuro7a2loSExORUrJy5UqGDh3KLbfc0u5cDQ0Nbf+/efNm4uONmeHgadJeBEIgUrONDiVotRXW27cLWbPP6HD8ltz7OXLNn2C0FXHzT40O54LEsOGIqffBpx8j33zF6HD8RodDSSaTienTp7Nw4UJ0XSczM5P4+HjWrFlDQkICVquVrKwsCgoKmD17NuHh4eTm5gIQHx9Pamoqc+fORdM0ZsyYgaZpfPbZZ2zatIlhw4bx8MMPA2empb788svs27cPIQSDBw9m5syZ3v0J9AKpu5B2G4wcg4gabHQ4QU2Mz0S+/gKy1Ia4vXs7iQUz2XQMfeXvYaAFbcYcv7j71SbkoFd/ivzna8iEKxBXjzM6JJ8nZIDM5zp48GC3juuN8XZZtRX9qd+i/fI3CGu6V6/VGcH6jOGbPrtW5sHOT9D+8HyPdhPzdZ5+n6XuQl/2v+6f3SO/R1w6osNjetv5+ixbTqPn/Qbq69Aey0cMjjMgOu8w5BmD0nOy1AbhEXDN+I4bK16npU+C402wbbPRofgV+fZrsP0jxH/OxBeTwoWIPl8X25Ogr1TF9jqiEoOXyePHkJVliPEZAf3p1K+MHAODotHVQ+hOk1VbkW+/ikjJRNxwk9HhdIsYHIc2PRf270a++pzR4fg0lRi8TJYVQ2srIn2S0aEoXxOaCZGWBds/QjpUDf+OyPrD6IWLYcgwxF0P+HVJETFmPOL7P0Fuehdd1c46L5UYvEhK6Z4aeUki4uJLjQ5H+RYxIQekdM8WU85LOp3u4nitrWj3zUP07Wt0SD0mfngXXDEa+fIfkTV7jQ7HJ6nE4E1fVMOXX6i7BR8kBse5/ziU2pC6bnQ4Pkv+tRD2fo427b8QcUONDscjhMmENvMhCA1H/2MesvmE0SH5HJUYvEiW2iCkjyqY56NE+iQ48hV8XmV0KD5J/3Aj8r1/Im6cghibZnQ4HiUGDEL75W/gyFfozz+liu19h0oMXuIumLcJkZyGCA0zOhzlHMTYVOgfplZCn4P8cj/yxQJIHIn40T1Gh+MVYsRIxE/uhY/KkP9eZ3Q4PkUlBi+RW+1w8oQaRvJhok9fxPgbkFs/QDYfNzocnyFPNaOv/B3064/2y4cRZkNLqnmVmPRDGJuGfOMFpLpzbKMSg5fIEhsMjoMRSUaHolyASJ8Ezhbk5k1Gh+ITpJTIFwrgq1q0mQ8jBkZ1fJAfE0K494uOjkN/dhGysaHjg4KASgxeIOtqYecniAk5flEyIKgNS4CLL3UncgVZ9DayogTxo7sRVwRuqepvE/1D0e6fBydPoD/7B6RLFdtTf7W8QJZuAKEhUrOMDkXpgLuw3iT4ohp5ILinLsrqT5F//TNccx3iez82OpxeJS6+FHHXLPh8O3LtS0aHYziVGDzMXTBvAyRdi7AE12Y4/kqMnwhms3sWWZCSx46iP/MHsAxGm57r14vYuktLzUTc8D3ku28gK8uMDsdQKjF42vZKOFrvrsej+AURPgAxJgVZVox0Oo0Op9dJ3YX+p8Vw/BjafY8gQgN/d7vzEf/5c7gkEf3PTyHruleYMxCoxOBhesl6CB8A16jSvv5EpE+CE03Iyg+NDqXXyb+/4t5Eaup9iGEJRodjKBHSB+2+R0AI9D/+Htly2uiQDKESgwfJpkb4eLO70JhZFczzK1ddDZbBQbemQW4rR/7jNcSEHHWX+zURHYv287lQsxf5l5VGh2OITk1QrqysZNWqVei6TnZ2NlOmTGn3utPppKCggD179hAREUFubi4xMe6NwdeuXUtRURGapjFt2jTGjBnDkSNHWLFiBUePHkUIQU5ODjfffDMAx48fJz8/n8OHDzN48GDmzJlDeLh/3NrKsmJwqYJ5/shdWC8b+Y81yPo6RJTxG9t7mzzyFXphPsRfhrjzl0aH41PEaCviltuRb69BT7gK7fobjQ6pV3V4x6DrOoWFhcyfP5/8/HxKS0upqalp16aoqIiwsDCWL1/O5MmTWb16NQA1NTXY7XaWLFnCggULKCwsRNd1TCYTd999N/n5+SxcuJB333237Zzr1q1j9OjRLFu2jNGjR7NunX+sSGwrmHfZ5Yihw4wOR+kGMcG97WowFNaTzhb3TmxSuovj9fH/4nieJn7wnzByDPIvzyC/CIy95zurw8RQXV1NXFwcsbGxmM1m0tLSKC8vb9emoqKCjIwMAFJSUqiqqkJKSXl5OWlpaYSEhBATE0NcXBzV1dUMGjSI4cOHA9C/f3+GDh2Kw+EAoLy8nIkTJwIwceLEs67ls/btgoP73XsKK35JRMfClVcHRWE9+epz8EW1ewZSzEVGh+OThGZC+/mvISISfWUe8kTwrI7vcCjJ4XAQFXVm9WNUVBS7du06bxuTyURoaChNTU04HA5GjDiz05PFYmlLAN+oq6tj7969JCYmAtDY2MigQYMAGDhwII2NjeeMy2azYbO5pxfm5eURHd29qaFms7nbx37bsb8WcrJvP6K/9yM0H6+N5Kk++5PO9vnk93/EsSW/ZUDtF/T18wkE5+vzyff+xbFN7xL6o7uIyJlsQGTe4/Hf7ehoWn6zkIbHHsC8+mkGzsvzuUWr3vj3bGgRlFOnTrF48WLuvfdeQkNDz3pdCHHe+dQ5OTnk5Jz5dN7dPU89sS+uPH0afdO/EWPTcDSfhOaTPTqftwX7ns8XIhNHQWgYjf/8G9rQy3ohMu85V59lzT73ENIVozl10084HWC/B1753Y6+CPHTGbS8+iyHVz+L9v1bPXv+HjJkz2eLxUJ9fX3b1/X19VgslvO2cblcNDc3ExERcdaxDoej7djW1lYWL17M9ddfz/jxZ/ZCjoyMpKHBXa+koaGBAQMGdLaPhpFbSuHUSTWMFABESB/E+InuwnoBNnQgm0+g/zEP+oej/eIhhMlkdEh+Q2RNRoy7Hrn2ZeRn24wOx+s6TAwJCQnU1tZSV1dHa2srdrsdq9Xark1ycjLFxcUAlJWVkZSUhBACq9WK3W7H6XRSV1dHbW0tiYmJSClZuXIlQ4cO5ZZbbml3LqvVysaNGwHYuHEj48b5/u28LF0PMRepgnkBQqRPglYn8sNio0PxGCkl+gvL4Mghd3G8yEFGh+RXhBCIe34FsUPcxfYa6js+yI91mBhMJhPTp09n4cKFzJkzh9TUVOLj41mzZg0VFRUAZGVlcfz4cWbPns3bb7/N1KlTAYiPjyc1NZW5c+eycOFCZsyYgaZp7Ny5k02bNlFVVcXDDz/Mww8/zNatWwGYMmUK27Zt48EHH+STTz45a2qsr5FfHYTPt7sL5gVhGYFAJIYlwLDhAVUiQ65fB1s/QPzkZ4jL1QeY7hD9+ruL7bWcdhfba201OiSvETJAti46eLB7y9d7Oiapv/Ei8p030P5Q6DclitUzho7pRW8jX3kW7f/l++1q4G/6LD/fjr54AYwZ756aGsAfYHrjd1vfvAn53JOInB+i3T7Dq9fqDEOeMSjnJ10u95z30cl+kxSUzhHjM8Ac4vcroWVjA/qziyA6Du1nDwZ0Uugt2nU3IDInI21vup8vBiCVGHpi+1ZodKBNUA+dA40IC0eMTUV+uBHpbDE6nG6RrlZ3Ujh5HO3+eWqLWQ8St02Hyy5Hf34Z8tCXRofjcSox9IBesh4iIuFq339ArnSdmJADzSeQWz8wOpRuOb76Wfi8CjH1AcTFlxodTkAR5hB3sT2z2b347fQpo0PyKJUYukkea4Bt5YjUrIDeEzeoXXk1RMX45UNoWVlG89qXETd8Dy1NbRjlDcIyGO0XD8HB/ciXnyZAHtcCKjF0m7tgnkutXQhgQtPcdw2ffow88pXR4XSarKtF//NTmBOudO8voHiNGHkt4gd3uPfy2PiO0eF4jEoM3eAumGeDhCsRF8UbHY7iRSItG4Rwb9fqB2TLafciNiEY+JuFiJA+RocU8MTk22BUMnLNc8i9uzo+wA+oxNAde3ZC7QH3p0kloImowXDVGKTdhtR9f5N4+ZdnoGYv2ow5mFRxvF4hNA1txhwYMMj9vOH4MaND6jGVGLpBltqgbz/EuHSjQ1F6gUjPAccR+NS3SyHo7/8bWWpDTL4NoSZE9CoRPgDtvnlwrAG9MN/vq/OqxNBF8tRJ5Ob3EdYJiH5nF/5TAo8YkwJhET69pkHu3+2+W7jqGsR/3GF0OEFJXDYCcfvPoWoL8p+vGR1Oj6jE0EVyix1On1S7tAURERKCSMlAVpb55DCBbD7urpgaPsBdHE9TxfGMIiZ+3/278vdXkDs+MjqcblOJoYtkyXqIGwoJVxkditKLxIQcaG1FfrjR6FDakbqO/uel4DiMdt8jiIhIo0MKakIIxF0PwEXx6M89iXQcNjqkblGJoQvkoRqo3qEK5gUhEX8ZXJKILFnvU/PV5btvwMebET+djki40uhwFED07ecuttfair7y98hWp9EhdZlKDF0gSzeApiFS1YKhYCTSc6BmH+z3jf1/5WfbkGtfRljTEVm3dHyA0mtE3MVo9z4Iez9H/nWV0eF0mUoMnSRdLuQHRTDaqmrZBylx3Q0Q0scnHkLLo/XuOkixQxA/+5W6g/VBInkCIueHyKK30TdvMjqcLlGJobM+qYDGBjT10DloidBvCuttQracNiwO2dqK/swiaDntLo6nZsf5LPGTn0HiVcgXC5AH9xsdTqepxNBJeqkNBgyEUclGh6IYSKRPgpPGFtaTb7zgftZ19yzEkGGGxaF0TJjNaDN/A336up83nPLt/eC/0anqb5WVlaxatQpd18nOzj5rVzWn00lBQQF79uwhIiKC3NxcYmJiAFi7di1FRUVomsa0adMYM2YMAE8//TRbt24lMjKSxYsXt53rtddeY8OGDW17Pd9xxx2MHTvWI53tLtn4dcG8SVNUwbxgd/koiI51DyelZPT65eUWO3L9m4jMm9HGT+z16ytdJwZFof3iIfT8x5EvFsAvHvL5ob8O7xh0XaewsJD58+eTn59PaWkpNTU17doUFRURFhbG8uXLmTx5MqtXrwagpqYGu93OkiVLWLBgAYWFhehfrwjMyMhg/vz557zm5MmTWbRoEYsWLTI8KQDuZwu6rgrmKWcK6+38BHn4UK9eWx76Ev35p+CyyxE/NX7nMKXzxFXXIKZMRZa/jyz6h9HhdKjDxFBdXU1cXByxsbGYzWbS0tIoLy9v16aiooKMjAwAUlJSqKqqQkpJeXk5aWlphISEEBMTQ1xcHNXV1QCMHDmS8PBwz/fIw6SU7hIYiVch4i42OhzFB4i0rK8L6/VeOW55+hT6yjwwm9F++QgiJKTXrq14hvjeT+Dqcci//hm5+zOjw7mgDsdFHA4HUVFntq2Miopi165d521jMpkIDQ2lqakJh8PBiBEj2tpZLBYcDkeHQb377rts2rSJ4cOHc88995wzgdhsNmw29z/MvLw8oqOjOzzvuZjN5gse2/LpNhoOfcmAX/2M/t28hq/pqM+ByKN9jo6mYcx4WsuKiZo2G2Hy7kpjKSXHlj3BqYP7GfjfS+h7RecWV6r32ffoDz+B46HpyOeexLJ4FZoHZjh6o88+N2B+4403cuuttwKwZs0aXnzxRR544IGz2uXk5JCTc2Zop7ubYXe0kbb+j9ehb3+OX3ENJ7y8yXhv6Y0N032Np/ssx09E/6iMI+/bEF6ekKBvfAdZ/A7iB3fQdHECTZ3sh3qffZP8xcPoeb/h8O/no+X+tsclTHrS5yFDhpzz+x0OJVksFurr69u+rq+vx2KxnLeNy+WiubmZiIiIs451OBxnHftdAwcORNM0NE0jOzub3buNW0wkTzUjK0oQ49IR/fobFofig66+DsIj3Nu7epHctwv56rOQdC3iltu9ei2ld4hLEhB3/tK9AdRbrxodzjl1mBgSEhKora2lrq6O1tZW7HY7Vqu1XZvk5GSKi4sBKCsrIykpCSEEVqsVu92O0+mkrq6O2tpaEhMTL3i9hoaGtv/fvHkz8fHGbYQjy0vg9ClVME85i7uwXiZUbkY2NXrlGvL4MXdxvAGD0Gb8GqGp2eWBQqRPQqRlI99eg/xki9HhnKXDoSSTycT06dNZuHAhuq6TmZlJfHw8a9asISEhAavVSlZWFgUFBcyePZvw8HByc3MBiI+PJzU1lblz56JpGjNmzED7+pd76dKl7Nixg6amJu677z5uu+02srKyePnll9m3bx9CCAYPHszMmTO9+xO4AFlqg4viYfgVhsWg+C6RPglp+zvyw2JEzg89em6p6+iF+XDUgfZIHiJigEfPrxhLCAF33ofcvwe9cAnaY0sQ0bFGh9VGSF+qCNYDBw8e7NZx5xufk7UH0P97FuLWaWg3/ain4fkUfxiH9TRv9dm18NfgbEF7fJlH56brb69Bvrkaced9aJk3d+sc6n32fbLuIPr/zYWYIWiP/L5bs80MecYQrGSJDUwmRGqG0aEoPkykT4Ivv4B9ntvrV+74CPn3vyDGT0RkfN9j51V8j4gZgjYtF76oRq55zuhw2qjEcA6ytfXrgnnjEANUwTzl/MS466FPH/cHCQ+QjsPozy2Gi+LdJS98fIWs0nPi2hTETT9CbnwHvew9o8MBVGI4t08qoKlRFcxTOiRCwxBjJyDLNyFP96ywnmx1oj/zB3A63cXx+vbzUJSKrxM/ugcuT0K+tAJZs8/ocFRiOBe9ZD1EWmCU8eU4FN/nLqzXjNxq79F55OvPw56daPfOVqvsg4wwmdzF9vqHof8xD3my2dB4VGL4Dnm0Hj7ZgkjL9PqKViVAXJ4Eg+N6tE+DvnkTcsNbiJz/QFjTPRic4i9E5CC0mQ/DkUPozy8zdKdAlRi+Q37wHkgdMUENIymdI4RwF9b7vApZ1/XZcbL2gLvqZsKViJ/c6/kAFb8hLh+F+PHPYKsdafu7YXGoxPAtUkr3Q8TLkxCx557GpSjnItKyQWju7V+7QJ46if7HPOjT110cT5V1D3rixilwbQry9VXIXTsMiUElhm/btQPqDro//SlKF4hBUTBqLNK+AelydeoYKSXypRVw6Eu0XzzkPocS9IQQaPf+F0THoj/zB+Sxho4P8jCVGL5FlqyHfv0RyROMDkXxQ1p6Dhx1wPatnWov3/sHcvMmxA/vRFx1jZejU/yJCA1Du28eNB9Hf/bJTn/Y8BSVGL4mTzYjt5QirrtBTRNUuufqcRAR6d4GtgNy92fI1/4MV49DfP/WXghO8Tci/jLEXfe7N4V6c3WvXlslhq/J8veh5bQaRlK6TZhDECkZ8PFm5LGj520nm46hP/sHGGhBmz5HFcdTzktLy0ZcfyPyX68jKz/svev22pV8nCxZD0OGwWWXGx2K4sdE+iRwuZDnWcEqdRf6nxbDsUb3IrYw39/FUDGWuGMmDEtA//PSXttOViUGQH65H/Z+7i6Fq0oQKD0ghgyD4VcgS2znnIcu31oDOz5C3DETccmFS9ArCoAI6YN23yMgQF+Zh2zp2Qr7zlCJAZCl68Fkdg8DKEoPiQk5UHsA9uxs931ZtQX5jzWI1CzE9TcaFJ3ij8TgOLQZc2H/HuQrz3r9ekGfGKTT6V7Uds11iIhIo8NRAoC7sF5f934eX5P1deh/WgJDL0FMvV/dmSpdJq4eh7j5p8iS9Z2a4NATQZ8YTleUwvFj7qmGiuIBon8oInkCcvP7yNOnkE6nexGb7kK7bx6ib1+jQ1T8lPjhnXDl1cjVK5H793jtOp1aZllZWcmqVavQdZ3s7GymTJnS7nWn00lBQQF79uwhIiKC3NxcYmJiAFi7di1FRUVomsa0adMYM2YMAE8//TRbt24lMjKSxYsXt53r+PHj5Ofnc/jwYQYPHsycOXMID/feA7qTG96CgVGQdK3XrqEEH5E+CflBEbKiFPZ9Dl9Uo93/qFpRr/SI0Exov3gI/Ylc9JV5aI8tAaI9fp0O7xh0XaewsJD58+eTn59PaWkpNTU17doUFRURFhbG8uXLmTx5MqtXu+fc1tTUYLfbWbJkCQsWLKCwsBBd1wHIyMhg/vz5Z11v3bp1jB49mmXLljF69GjWrVvniX6ek2yop+WjDxFp2QhNFcxTPGjESIgZgvzb88jifyFu/BFibKrRUSkBQAwYiPbLR8BxGH3VU14pttdhYqiuriYuLo7Y2FjMZjNpaWmUl5e3a1NRUUFGRgYAKSkpVFVVIaWkvLyctLQ0QkJCiImJIS4ujurqagBGjhx5zjuB8vJyJk6cCMDEiRPPupYnSfsG0HXEhGyvXUMJTkIIRHoONDW6a2/9+B6jQ1ICiEi8CnHrvVD5Iae7WJ+rMzocSnI4HERFnanhEhUVxa5du87bxmQyERoaSlNTEw6HgxEjRrS1s1gsOByOC16vsbGRQYPcu6YNHDiQxsbGc7az2WzYbO4HMHl5eURHd/126uTFw2id9AMiRo7u8rH+zGw2d+vn5c+M6LP+46mcEBA6+aeYDKiDpN7nwCZvn87p+MsIuz6Hfl+PxHiKT5dyFEKcd/ZGTk4OOTlnHhh3azPsa1KIzr7FrzYP9wR/2zDdEwzr8/du5bRLggHXVu9zELjiavrperf7PGTIuZ95dTiUZLFYqK+vb/u6vr4ei8Vy3jYul4vm5mYiIiLOOtbhcJx17HdFRkbS0OCuJtjQ0MCAAQM6ClFRFEXxoA4TQ0JCArW1tdTV1dHa2ordbsdqtbZrk5ycTHFxMQBlZWUkJSUhhMBqtWK323E6ndTV1VFbW0ti4oVXe1qtVjZu3AjAxo0bGTduXDe7piiKonSHkJ14pL1161ZeeOEFdF0nMzOTH//4x6xZs4aEhASsVistLS0UFBSwd+9ewsPDyc3NJTY2FoA33niD9957D03TuPfee7n2Wve00KVLl7Jjxw6ampqIjIzktttuIysri6amJvLz8zly5EiXpqsePNj1nbMgCG89UX0OFqrPwaEnfT7fUFKnEoM/UImh81Sfg4Pqc3DwRmII+pXPiqIoSnsqMSiKoijtqMSgKIqitKMSg6IoitJOwDx8VhRFUTwj6O8Y5s2bZ3QIvU71OTioPgcHb/Q56BODoiiK0p5KDIqiKEo7pt/+9re/NToIow0fPtzoEHqd6nNwUH0ODp7us3r4rCiKorSjhpIURVGUdlRiUBRFUdrx6Y16vK2yspJVq1ah6zrZ2dlMmTLF6JB67MiRI6xYsYKjR48ihCAnJ4ebb76Z48ePk5+fz+HDh9tVrZVSsmrVKj766CP69u3LAw884LdjtLquM2/ePCwWC/PmzaOuro6lS5fS1NTE8OHDmT17NmazGafTSUFBAXv27CEiIoLc3FxiYmKMDr/LTpw4wcqVKzlw4ABCCO6//36GDBkS0O/z22+/TVFREUII4uPjeeCBBzh69GhAvc9PP/00W7duJTIyksWLFwN0699vcXExb7zxBgA//vGP27Zf7hQZpFwul/zVr34lDx06JJ1Op3zooYfkgQMHjA6rxxwOh9y9e7eUUsrm5mb54IMPygMHDsiXXnpJrl27Vkop5dq1a+VLL70kpZRyy5YtcuHChVLXdblz50756KOPGhZ7T7311lty6dKl8ne/WK3qeQAABNVJREFU+52UUsrFixfLkpISKaWUzzzzjHz33XellFK+88478plnnpFSSllSUiKXLFliTMA9tHz5cmmz2aSUUjqdTnn8+PGAfp/r6+vlAw88IE+fPi2ldL+/7733XsC9z9u3b5e7d++Wc+fObfteV9/XpqYmOWvWLNnU1NTu/zsraIeSqquriYuLIzY2FrPZTFpaGuXl5UaH1WODBg1q+8TQv39/hg4disPhoLy8nIkTJwIwceLEtr5WVFRwww03IITg8ssv58SJE2076PmT+vp6tm7dSnZ2NgBSSrZv305KSgoAGRkZ7fr8zaenlJQUqqqqkH42B6O5ufn/t3f3Lq2zYRzHvxJUqNW2ieigiK9LK7Wg4uSgg5MuDoLi4KgFxdHJP0AQHazUQdDVRcFdrEMRfC3i+1CdxKIppUVraJtnKPY5fXwGe46c0vT+bE0Due78aO/kJuHi+vqa/v5+IN3ruKKiwvA5p1IpNE0jmUyiaRpWq9VwOdvt9i89aHLN9fz8HKfTidlsxmw243Q6OT8//3YNRbuUpKoqivJvg3ZFUbi/v89jRT8vFAoRDAZpbW0lEolgs9kAsFqtRCIRIH0efm2erigKqqpm9i0UGxsbjI+P8/7+DkA0GsVkMiFJEpBuP6uqKpCdvSRJmEwmotFoQbWRDYVCVFVVsbq6yuPjI83NzUxMTBg6Z1mWGRoaYmpqirKyMjo6OmhubjZ0zp9yzfW//2+/npfvKNo7BqOLx+MsLi4yMTGByWTK+q6kpISSkpI8VfbzTk5OsFgsBblm/ruSySTBYJCBgQEWFhYoLy9nZ2cnax+j5RyLxTg6OsLj8bC2tkY8Hs/pKtgo/kauRXvHIMsyr6+vmc+vr6/IspzHin5OIpFgcXGR3t5eenp6ALBYLITDYWw2G+FwOHPVJMtyVvenQjwPt7e3HB8fc3Z2hqZpvL+/s7GxwdvbG8lkEkmSUFU1M67P7BVFIZlM8vb2RmVlZZ5HkRtFUVAUhba2NiC9VLKzs2PonC8uLqipqcmMqaenh9vbW0Pn/CnXXGVZ5urqKrNdVVXsdvu3j1e0dwwtLS08PT0RCoVIJBL4/X66urryXdYf03Udr9dLXV0dg4ODme1dXV34fD4AfD4f3d3dme0HBwfous7d3R0mk6mglhcAxsbG8Hq9eDweZmdnaW9vZ2ZmBofDweHhIZB+QuMz387OTvb39wE4PDzE4XAU3JW11WpFUZRMS9uLiwvq6+sNnXN1dTX39/d8fHyg63pmzEbO+VOuubpcLgKBALFYjFgsRiAQwOVyfft4Rf3m8+npKZubm6RSKfr6+hgeHs53SX/s5uaG+fl5GhoaMj+C0dFR2traWFpa4uXl5cvjbuvr6wQCAcrKynC73bS0tOR5FL/v8vKS3d1d5ubmeH5+Znl5mVgsRlNTE9PT05SWlqJpGisrKwSDQcxmM7Ozs9TW1ua79Jw9PDzg9XpJJBLU1NTgdrvRdd3QOW9tbeH3+5EkicbGRiYnJ1FV1VA5Ly8vc3V1RTQaxWKxMDIyQnd3d8657u3tsb29DaQfV+3r6/t2DUU9MQiCIAhfFe1SkiAIgvD/xMQgCIIgZBETgyAIgpBFTAyCIAhCFjExCIIgCFnExCAIgiBkERODIAiCkOUfC6ZQWC23jhoAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"2MMOua0WwwdO","executionInfo":{"status":"ok","timestamp":1633622733338,"user_tz":-330,"elapsed":926,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"5b3a0bc1-b1f8-4157-b292-4f81795f459c"},"source":["plot_lr(triangular(250, 0.005, 'exp_range', gamma=0.999))"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deUBV1733//faBxwYRA+IRMVE0AwOEeWoQDSiEk1iBjPZzFVzmza25pI8t7c26W36u328zdPEoRFT015qJtuYSTOZGAnihAOoOCYqDkmIGISDCqJMe/3+OIaEODCdwz7D9/WXwB4+i4182WuvvZbSWmuEEEKIcwyrAwghhPAuUhiEEEI0IoVBCCFEI1IYhBBCNCKFQQghRCNSGIQQQjQSZHUAdzl69Gir9ouKiqK0tNTNabybtDkwSJsDQ1va3LNnzwt+Xu4YhBBCNCKFQQghRCNSGIQQQjQihUEIIUQjUhiEEEI0IoVBCCFEI1IYhBBCNCKFQTSbPnUCc0MWMlO7EP7Nb15wE55nvvDf8GUhqtflcEV/q+MIITxE7hhEs+jiIviy0PXv7ZstTiOE8CQpDKJZ9IdLoWMn6HU5umCT1XGEEB4khUE0SRcXofPWosZOQo2eAEe/Qn/burmphBDer1nPGAoKCli8eDGmaTJ+/HgmT57c6Ou1tbVkZGRw6NAhwsPDSU9PJzo6GoBly5aRnZ2NYRhMmzaNhISEhv1M02TWrFnY7XZmzZoFQElJCfPnz6eiooK4uDhmzpxJUJA8CrGS/vAN6NARNWEy1FSj3/g7umATauKdVkcTQnhAk3cMpmmSmZnJU089xbx589iwYQNFRUWNtsnOziY0NJQFCxYwadIklixZAkBRURG5ubnMnTuXp59+mszMTEzTbNhvxYoV9OrVq9GxXn/9dSZNmsSCBQsIDQ0lOzvbHe0UraSLv0bnrXPdLYRHoCKjoU88ert0Jwnhr5osDIWFhcTExNCjRw+CgoJISUkhLy+v0Tb5+fmkpqYCkJSUxO7du9Fak5eXR0pKCsHBwURHRxMTE0NhoesBZllZGdu2bWP8+PENx9Fas2fPHpKSkgBITU0971yifekPl567W7ij4XNq6Eg4tA99wmlhMiGEpzRZGJxOJ5GRkQ0fR0ZG4nQ6L7qNzWYjJCSEioqK8/a12+0N+7788ss8+OCDKKUavl5RUUFISAg2m+287UX700e/ct0tjJuECu/S8Hk1NBm0Ru/cYmE6IYSnWNJ5v3XrViIiIoiLi2PPnj2tOkZWVhZZWVkAPPvss0RFRbXqOEFBQa3e11c1t80nXnmBmo6dibr3EYwuXRs+ryMjKYvphW33Nrrd+aAno7qNXOfAIG120zGb2sBut1NWVtbwcVlZGXa7/YLbREZGUl9fT1VVFeHh4eft63Q6sdvt5Ofnk5+fz/bt26mpqeHMmTO88MILzJw5k6qqKurr67HZbA3bX0haWhppaWkNH7d2BSNZ8enC9NGvMDd8hrrxLpw1dfCj7c1rh1P/2Ycc//orVOcQT8Z1C7nOgUHa3DKtXsEtPj6e4uJiSkpKqKurIzc3F4fD0WibxMREcnJyANi0aRMDBw5EKYXD4SA3N5fa2lpKSkooLi6mX79+3H///SxatIiFCxeSnp7OoEGDePzxx1FKMXDgQDZtcj3YzMnJOe9con24ni10co1EugA1NAnq69C78ts5mRDC05q8Y7DZbEyfPp3Zs2djmiZjx44lNjaWpUuXEh8fj8PhYNy4cWRkZDBz5kzCwsJIT08HIDY2luTkZJ588kkMw+CRRx7BMC5dix544AHmz5/PG2+8Qd++fRk3bpx7WiqaTX/zFTp/Peqmu1FhXS68UdxVEB4BBZthxPXtG1AI4VFK+8mMaEePtu6FK7n1PJ/59+fRO/Iwnv37xQsDYL6agc5bhzH3dVRwsCeiuo1c58AgbW6ZVnclicCiS46i89ajUm+6ZFGAc91JZ8/AFzvbKZ0Qoj1IYRCN6I/fgaAg1ITbm9746iHQsTN6+0bPBxNCtBspDKKBdh5Hb1yNGnUDqku3JrdXwcGowYnogs1os74dEgoh2oMUBtFAr1wG6JbNgZQwEipOwqH9HsslhGhfUhgEAPpUOXrdp6ikVFRk92bvpwY7wBYkcycJ4UekMAgA9Kr3oa4OdePdLdpPhYTC1YPR2zfKkp9C+AkpDAJ9uhKdswLluA4V06vpHX5EDU2G48fg6FceSCeEaG9SGAR69Ydw9gzq5pbdLXxHDRnhOo6MThLCL0hhCHD67Bl01gcwZASqd99WHUN1tUP81eitUhiE8AdSGAKcXvsJnK7AuKl1dwvfUYnXQdFhdIks+SmEr5PCEMB0bQ360+Vw9bWo+KvbdCw1LNl1TLlrEMLnSWEIYDo3G06WY9x8T5uPpSKj4Yr+6K0b3JBMCGElKQwBSpv1rruFy/vB1de65ZhqWAp8WYguK3HL8YQQ1pDCEKgKNkPJUYwb72y0vGpbqMTvupNy3XI8IYQ1pDAEIK015ifvQvcYOPdswB1UdE/o3Re9TQqDEL5MCkMAqt1bAIf3oyZMRhk2tx5bJabAwS/Q5WVNbyyE8EpSGALQ6eX/hLAuqJTxbj+2SkwB5GU3IXyZFIYAo7/5ipr8Dahxt6A6dHT78dVlsXBZrDxnEMKHSWEIMPrTZdCxE2rszR47h0pMgQN70afKPXYOIYTnSGEIILq8DL15DZ3Tbmly2c62UIkpoE309s0eO4cQwnOkMAQQ/dn7YJqE3nqvZ0/U6wqIvkxGJwnho6QwBAhddRq95hOU4zpsPXp69FxKKdddwxc70ZWnPHouIYT7SWEIEHrdStfU2i1ZtrMN1LAUME30ji3tcj4hhPsENWejgoICFi9ejGmajB8/nsmTJzf6em1tLRkZGRw6dIjw8HDS09OJjo4GYNmyZWRnZ2MYBtOmTSMhIYGamhqeeeYZ6urqqK+vJykpiSlTpgCwcOFC9u7dS0hICAC//OUvueKKK9zY5MCj62rRWe/DNUNQl8e3z0kv7weR0a7RSdeltc85hRBu0WRhME2TzMxMfve73xEZGclvf/tbHA4HvXv3btgmOzub0NBQFixYwIYNG1iyZAlPPPEERUVF5ObmMnfuXMrLy/njH//IX/7yF4KDg3nmmWfo1KkTdXV1/P73vychIYErr7wSgIceeoikpCTPtTrA6Lz1cMKJ8dOZ7XZOpRRqWDI6+yN01WnXEqBCCJ/QZFdSYWEhMTEx9OjRg6CgIFJSUsjLy2u0TX5+PqmpqQAkJSWxe/dutNbk5eWRkpJCcHAw0dHRxMTEUFhYiFKKTp06AVBfX099fb3b5usRjWmt0VnvwWWxMHBYu55bJV4H9XXondKdJIQvafKOwel0EhkZ2fBxZGQkBw4cuOg2NpuNkJAQKioqcDqd9O/fv2E7u92O0+kEXHciv/nNbzh27BgTJ05stN2//vUv3n77bQYNGsQDDzxAcHDwebmysrLIysoC4NlnnyUqKqol7W4QFBTU6n19Qc3u7ZR/dYjwx35DSPfuQPu1WdtTKLVHEbwrn663tH1q77bw9+t8IdLmwOCJNjfrGYMnGIbBc889x+nTp3n++ef56quv6NOnD/fffz9du3alrq6Ol156iffee4+77z5/dbG0tDTS0r7vuy4tLW1VjqioqFbv6wvq33kNwsI5PchB1bl2tmebdUIS1WtXcvzrr1CdQ9rlnBfi79f5QqTNgaEtbe7Z88IjFJvsSrLb7ZSVfT8hWllZGXa7/aLb1NfXU1VVRXh4+Hn7Op3O8/YNDQ1l4MCBFBQUANCtWzeUUgQHBzN27FgKCwub2UTxY7qkGHZsRl1/k0emv2gONXwU1NXK6CQhfEiThSE+Pp7i4mJKSkqoq6sjNzcXh8PRaJvExERycnIA2LRpEwMHDkQphcPhIDc3l9raWkpKSiguLqZfv36cOnWK06dPA1BTU8POnTvp1asXAOXlrmkUvntGERsb6872BhSd/SEYNo9Of9GkuKuhayQ6f711GYQQLdJkV5LNZmP69OnMnj0b0zQZO3YssbGxLF26lPj4eBwOB+PGjSMjI4OZM2cSFhZGeno6ALGxsSQnJ/Pkk09iGAaPPPIIhmFQXl7OwoULMU0TrTXJyckkJiYC8MILL3DqlOulqMsvv5xHH33Ug833X7rqNHp9Fmr4aFRXe9M7eIgyDJRjFHr1R+iqSlRImGVZhBDNo7TW2uoQ7nD06NFW7eevfZLmp8vQby3G+K95qD6N311o7zbrQ/sw//Rr1NR/x7jO/VN9N4e/XudLkTYHBkueMQjfo+vr0dkfwZWDzisKluh7petlt/x1VicRQjSDFAZ/VLAJykow0m6zOglw7mU3xyj4fIfMnSSED5DC4IfMVe+51nMeMtzqKA3U8NFQX4/evsnqKEKIJkhh8DP68H44+AVq/K1uX8+5TfrEQfcYdJ50Jwnh7aQw+Bmd9T50DkFZ9JD3YpRSrruGL3ahT52wOo4Q4hKkMPgRfaIMvXUDatQNqE7WvWV8MWr4KNfKbrKAjxBeTQqDH9FrV4JpolItfKHtUnpdATG90fkbrE4ihLgEKQx+QtfVugrDoERU9GVWx7kgV3fSKNi/G33CaXUcIcRFSGHwE3prLpwsxxg3yeool6Qco0BrV14hhFeSwuAn9OqPIPoyGDDU6iiXpHr2gV6Xy8tuQngxKQx+QH950DVEdezNKMP7L6lyjILCz9HO41ZHEUJcgPf/FhFN0qs/gg4dUSneNUT1YtTw0QDSnSSEl5LC4ON05Sn0lrWo5LE+M3Op6tET+sTJy25CeCkpDD5Ob8iC2hrUWO9+6PxjyjEaDu9Hl35rdRQhxI9IYfBh2qxHr14BVw1G9brc6jgtohzXAchdgxBeSAqDL9uZ75pF1cfuFgBU9xiIvxq9eY3VUYQQPyKFwYeZqz+CblGQMNLqKK2iRo6Bb75EFx2xOooQ4gekMPgoXVwEewtQY25E2bxoFtUWUInXgWGgt6y1OooQ4gekMPgonbMCgoJQoydYHaXVVJeuMCABvWUt2jStjiOEOEcKgw/S1WfRG1ejhl3n+uXqw9SIMVBWAoe+sDqKEOIcKQw+SOetgzOnUWNutDpKm6mhI6FDB/Rm6U4SwltIYfBBes0ncFks9B9gdZQ2U51CUENGovPXo+vqrI4jhEAKg8/RXx6EIwdQY25CKWV1HLdQI66HylPweYHVUYQQQFBzNiooKGDx4sWYpsn48eOZPHlyo6/X1taSkZHBoUOHCA8PJz09nejoaACWLVtGdnY2hmEwbdo0EhISqKmp4ZlnnqGuro76+nqSkpKYMmUKACUlJcyfP5+Kigri4uKYOXMmQUHNihkQ9NpPoEMHVHKq1VHcZ9AwCAlDb16DGuywOo0QAa/JOwbTNMnMzOSpp55i3rx5bNiwgaKiokbbZGdnExoayoIFC5g0aRJLliwBoKioiNzcXObOncvTTz9NZmYmpmkSHBzMM888w3PPPcef//xnCgoK2L9/PwCvv/46kyZNYsGCBYSGhpKdne2BZvsmfabK9ctz+GifmRepOVRQMMpxHbpgM7r6rNVxhAh4TRaGwsJCYmJi6NGjB0FBQaSkpJCXl9dom/z8fFJTUwFISkpi9+7daK3Jy8sjJSWF4OBgoqOjiYmJobCwEKUUnTp1AqC+vp76+nqUUmit2bNnD0lJSQCkpqaed65ApjfnQPVZ1JibrI7idmrEGKg+i96xxeooQgS8JvtonE4nkZGRDR9HRkZy4MCBi25js9kICQmhoqICp9NJ//79G7az2+04na4lHU3T5De/+Q3Hjh1j4sSJ9O/fn1OnThESEoLt3AtbP9z+x7KyssjKygLg2WefJSoqqiXtbhAUFNTqfduT1hrn+lUQdyV2R3Kbni94Y5u1/XpKI7sTVLCJbjff6fbje2ObPU3aHBg80WbLOu8Nw+C5557j9OnTPP/883z11Vd07dr8MflpaWmkpaU1fFxaWtqqHFFRUa3etz3pg19gfnkQ9dAMysrK2nQsb22zThxFzWfvc/zIIVRYF7ce21vb7EnS5sDQljb37Nnzgp9vsivJbrc3+kVUVlaG3W6/6Db19fVUVVURHh5+3r5Op/O8fUNDQxk4cCAFBQWEh4dTVVVFfX39RbcPVHrNJ9Cxs2sEj59SI8dAfb0s4COExZosDPHx8RQXF1NSUkJdXR25ubk4HI1HjiQmJpKTkwPApk2bGDhwIEopHA4Hubm51NbWUlJSQnFxMf369ePUqVOcPn0agJqaGnbu3EmvXr1QSjFw4EA2bdoEQE5OznnnCkT6dAU6fz0qaQyqU4jVcTwnti9cFoveIjOuCmGlJruSbDYb06dPZ/bs2ZimydixY4mNjWXp0qXEx8fjcDgYN24cGRkZzJw5k7CwMNLT0wGIjY0lOTmZJ598EsMweOSRRzAMg/LychYuXIhpmmitSU5OJjExEYAHHniA+fPn88Ybb9C3b1/GjRvn2e+AD9Abs12L8fjhQ+cfUkqhRlyPfm8J2nkcZe9udSQhApLSWmurQ7jD0aNHW7Wft/dJaq0xfz8DQsKw/fY5txzTm9usS4oxn/456q6fYtx4l9uO681t9hRpc2Cw5BmDsNj+PXDsG7+YF6k5VPRlEHcVelOO1VFEO9Nnz2Au/V908ddWRwl4Uhi8nF67EjqHohJHWR2l3ajksa4FfL4+bHUU0U60aWJmzkNnvY+5ZBF+0pHhs6QweDF9uhK9LRc1cgyqY0er47Qb5RgFtiDXsxUREPTy16Bgk2tiyH274IudVkcKaFIYvJjenAN1tajRN1gdpV2psC5wrQO9eQ363NBl4b/M3Gz0x++grp+I8cR/Q7cozGWvyV2DhaQweCmtNXrdKugTj+oTb3Wcdmckj4NTJ2CvzLjqz/TBL9CvZcDV16Lu+zkquAPqlp/A4f2wU6bDsYoUBm/11UEoOowaFVh3Cw0GJ0JouHQn+TFdXob51z9BtyiMX/wGdW4WZZUyHrrHYC5fIku+WkQKg5fS6z6F4A6okf77pvOlqKBg1PDRrhlXz1RZHUe4ma6tcRWFs2cwfvk0KjS84WsqKAh1231QdFjegreIFAYvpKur0VvWohJT/Gp67ZZSyWOhtga9dYPVUYQbaa3Rr/8VDu/HmP4Eqtfl522jRlzvegv+/SXynMkCUhi8kN66Ac5UoUZNsDqKtfpeCT16oTeutjqJcCOd/SE69zPULfeihiVfcBtl2DBufwCOfeMahCHalRQGL6Q3rILoy+DKgVZHsZRSynXXsH83uvRbq+MIN9Cf70C/mQkJI1G33nvpjYclQ5949Pv/QtfVtk9AAUhh8Dr62Dewfw9q1A1+s6ZzW6ikVAB5E9oP6OPHMP/2Z+jRy9WFZFz6149SCmPyg1BWgl6f1U4pBUhh8Dp6QxYYBipZJg8EUJHRcOUg9MbVMq7dh+nqs5gv/glM0/WwuXMzZwkeNAzir0Z/tBRdU+3ZkKKBFAYvouvq0LmfwWAHqqusQ/EdlTwWSo66xrYLn+N62PwifHME42f/gepx4YnbLkQphXHHQ3DC6VqTRLQLKQzeZFc+nDqBMTrAHzr/iEq8DoI7yENoH6XXfILelIO67T7UoMQW76+uGgzXDEGveEuGLrcTKQxexFy/CiLs0Ir/PP5MdQ5BDU1C561D18pDSF+iD+9HL/07DEpE3Tyl1ccx7ngYKk+hP13mxnTiYqQweAldXga7tqJSxqFsNqvjeB2VPBZOV7juqoRP0BWnMBf9P4iwY/zbk00+bL4U1bc/yjEK/ely9MlyN6YUFyKFwUvoTatBm6jr0qyO4p2uSYAuXTGlO8knaLMe83/nwKlyjMdmNXqzubXU5Aehvg794VI3JBSXIoXBC2it0bnZ0G9Aix7MBRJls6FGjoFd+eiKU1bHEU3QHy6FvdtR9z2KuryfW46pevREjZ6IXrcS/W3rVmwUzSOFwRsc3g/HilApMkT1UlTKeNdfjFvWWB1FXILetRX94VJU8jjU6IluPba65ScQFIxe/rpbjysak8LgBXTuZ9Chg2v0jbgo1fsKuLyfvOzkxXRZCWbmXOh1OeqBx9z+kqaK6Ia64XZ0/nr04QNuPbb4nhQGi+naGnTeOtTQZFRIqNVxvJ66Ls016+ZXB62OIn5E19Vh/u05qK9zPVfw0KqDasIdENYF891X5KVHD5HCYDFdsAWqTks3UjOpEde7uhLkrsHr6PeWwKF9qId/hYr23LMy1TnE1aX0xU5ZyMlDpDBYTG/Mhm5RcPW1VkfxCSo0zPVOw+Y16Noaq+OIc/TubehP3kGNnoAxfLTHz6euvxGiemC+87Is5uMBQc3ZqKCggMWLF2OaJuPHj2fy5MmNvl5bW0tGRgaHDh0iPDyc9PR0oqOjAVi2bBnZ2dkYhsG0adNISEigtLSUhQsXcuLECZRSpKWlcfPNNwPw5ptv8tlnn9GlSxcA7rvvPoYNG+bONnsNfcIJu7ehbroLZci7C82lrktzvexWsAU1fJTVcQKePuHE/Mc813OFn/ysXc6pgoNRkx9E/+8cV1fsyDHtct5A0WRhME2TzMxMfve73xEZGclvf/tbHA4HvXv3btgmOzub0NBQFixYwIYNG1iyZAlPPPEERUVF5ObmMnfuXMrLy/njH//IX/7yF2w2Gw899BBxcXGcOXOGWbNmce211zYcc9KkSdx2222ea7WX0JtzXO8uJI+1OopvueZasEehc7NACoOltFnvethcfQbj0dkee65wIWr4aPTKd9HLXkMPS0EFB7fbuf1dk11JhYWFxMTE0KNHD4KCgkhJSSEvr/Ei3fn5+aSmpgKQlJTE7t270VqTl5dHSkoKwcHBREdHExMTQ2FhId26dSMuLg6Azp0706tXL5xOp/tb58Ua3l2IvxoV07vpHUQDZdhcs8/u2Y52llodJ6DpFW/BFztR9/0c1bNPu55bGQbGXVNd03LnrGjXc/u7Ju8YnE4nkZGRDR9HRkZy4MCBi25js9kICQmhoqICp9NJ//79G7az2+3nFYCSkhIOHz5Mv37fvwSzcuVK1q5dS1xcHA8//DBhYecvb5mVlUVWlusB5LPPPktUVFRz2nueoKCgVu/bFrWFn+M8+hXhv/hPQtr5/Fa12Z3qJt1N2UdvErJzM6F3/7TJ7f2hzS3l6TbX7NlO+Qdv0On6CXS5/V5r1g8ZcwPlOR9Ru+JN7LfcI9fZXcd069Fa6OzZs8yZM4epU6cSEuKan33ChAncfffdACxdupRXX32VGTNmnLdvWloaaWnfTx9RWtq6vxyjoqJavW9bmCvegaBgTl+TQFU7n9+qNrtVcCe4chCVq96naszNTf5S8os2t5An26wrT2E+/3uI6kHNPdMpKyvzyHmaleX2B9H/nU7pqy8S/ctZcp1boGfPC48ea7IryW63N7roZWVl2O32i25TX19PVVUV4eHh5+3rdDob9q2rq2POnDmMHj2akSNHNmzTtWtXDMPAMAzGjx/PwYP+N15d19ait6xDDU1ChZx/NySaR12XBiXFcGCv1VECitYa85UMqDyJ8eh/ojo1c9EdD1G9r0CNSkOvXkHd0a8tzeIvmiwM8fHxFBcXU1JSQl1dHbm5uTgcjkbbJCYmkpOTA8CmTZsYOHAgSikcDge5ubnU1tZSUlJCcXEx/fr1Q2vNokWL6NWrF7fcckujY5WXfz9z4pYtW4iNjXVDM73Mzjw4XSHvLrSRSkyBTp1dq96JdqPXr4KCTag7HkJdHm91HADU7Q9AUDCVr75odRS/0GRXks1mY/r06cyePRvTNBk7diyxsbEsXbqU+Ph4HA4H48aNIyMjg5kzZxIWFkZ6ejoAsbGxJCcn8+STT2IYBo888giGYfDFF1+wdu1a+vTpw69//Wvg+2Gpr7/+OkeOHEEpRffu3Xn00Uc9+x2wgJn7GXS1w4AEq6P4NNWxk2tkypa16Pt+ZvlfroFAf3sU/cbf4ZohqLTbrY7TQEV0Q914J9XvLcG4fiLqykFWR/JpSvvJO+VHj7ZutsX27nvWp05g/noq6obJGHdPbbfz/pA/9bfrg19gPvufqJ/OxBh1w0W386c2N5e726zr6jD/32+gpBjjmRdQdu96yKurq+GZGZhhERhPPd+m9R98iSXPGIR76bz1YJqu4Zai7eKugphe6A2fWZ3E7+kP34AjBzAemuF1RQFAdexI2AO/gC8L0VvWWh3Hp0lhaGd6cw7E9kX1at8x3/5KKeV6CF24F32syOo4fksf2Ite8bZrKm2H975U2GnMROgTj172Krqm2uo4PksKQzvSx76Bw/tRI1OtjuJXVPI4sNnQ61ZZHcUv6arTrrebI7uj7vPuZ37KMDCmTAdnKTrrfavj+CwpDO1Ib14DSrlmCBVuoyK6wbXD0Ruz0XW1VsfxO/pffwNnKcYjT6I6e/8DfnXVYBgyAv3x2+hTsj50a0hhaCdaa1c30tXXorpFNrm9aBlj9ESoOAkFm62O4lfMvPXoTatRk6ag+l1jdZxmM+6eCrU16Pf+aXUUnySFob0c2gfHj0k3kqcMTAB7d8x1n1qdxG/ok+XoJX+FK/qjJk2xOk6LqJjeqNSb0es+RX91yOo4PkcKQzvRm3MguANqWLLVUfySMmyoUTfA3gL08WNWx/F5WmvM1xZC9VmM6emoIEtnz2kVdet9EBqO+cbfZKW3FpLC0A50XZ1rzviEkT7RR+ur1HVpoAzXm7miTfTG1bBjC+qOB1GX+ebsAyo0DDX5QTiwF52/3uo4PkUKQ3vYsx0qK6QbycOUPQoGDUNv+AxdX291HJ+lnaWut5v7DUCl+fa6KGr0DRDbF/32YtcLcKJZpDC0A705B8LCYeBQq6P4PeP6iXDSCbvymt5YnEdrjfnqAqivw5j27z6/sqAybBj3PuoavrryHavj+AwpDB6mz1ShCzajHKN9sp/W5wx2QFc75lp5CN0aet2nsGc76u6pqOjLrI7jFurKga45tT55F11WYnUcnyCFwcP09o1QW4NKSrU6SkBQNhsqJQ12b0M7j1sdx6fo0m/Rb/7DNaR6zE1Wx3ErdfdUUKDfWmx1FNipEk0AACAASURBVJ8ghcHD9KYc6B7jmtNHtAs1Kg20iV4v03E3lzZNzJdfAAXG1Mf9bgI6Ze+OuvFu9NYN6H27rY7j9fzr6nsZfaLMtR7uyFRrlj0MUKp7DAwYit6wCm3KQ+jm0DkrYN8u1JRHUJHRVsfxCDXhDte7Lm/8XX4umiCFwYP0lrWgNWrkGKujBBzj+gngLHWNCBOXpEu/Rb/7Kgwc6noXxE+pjh0x7pkGRYfR8gzqkqQweJDevMb11mhML6ujBJ4hIyA8Qh5CN8H1ItuLgMJ46Jf+f2ebeB1cOQj93uvoylNWp/FaUhg8RB8rgq8OoUbKhHlWUEHBrqVTd25Bn3BaHcdr6Y2rYe921F0P+20X0g8ppTDuexSqTqOXvW51HK8lhcFD9JZ1rplUvXjuen+nRk8E05Q3oS9CnypHL/1f6HeN341CuhTV+wrUuFvR61aiDx+wOo5XksLgAVprdN46uHIQqqvMpGoV1aMnXDPE9QtA3oQ+j/7n36CmGuOnM/1uFFJT1G33QZdumEv+Kg+iLyCwfhray9eH4VgRavhoq5MEPCP1JnCWUr011+ooXkVv24jeugF1y09QMb2tjtPuVOcQ1D3TXMuAynOo80hh8ACdtw5sNtSwFKujiCEjoaudM5+8a3USr6FPV2L+cxH07ouaeKfVcSyjRlwPVw1GL3sNXXHS6jheRQqDmzV0I12TgArvYnWcgKdsNtToidRs34wuKbY6jlfQby+GipOuF9kCeJoWpRTG/T+H6jPod162Oo5XadZPRUFBAYsXL8Y0TcaPH8/kyZMbfb22tpaMjAwOHTpEeHg46enpREe7RjgsW7aM7OxsDMNg2rRpJCQkUFpaysKFCzlx4gRKKdLS0rj55psBqKysZN68eRw/fpzu3bvzxBNPEBYW5uZme9ChfVBWgrrtfquTiHPU6Anoj95Er/0Edfc0q+NYSn++A71+FerGu1CXx1sdx3KqZx9U2u3ole+iR92A6jfA6kheock7BtM0yczM5KmnnmLevHls2LCBoqKiRttkZ2cTGhrKggULmDRpEkuWLAGgqKiI3Nxc5s6dy9NPP01mZiamaWKz2XjooYeYN28es2fPZuXKlQ3HXL58OYMHD+aFF15g8ODBLF++3APN9hydtw6CglFDk6yOIs5R3SLpOGI0ekMWurbG6jiW0TXVrsV3oi9D3Xqv1XG8hrrlJ9AtCnPJSzJI4ZwmC0NhYSExMTH06NGDoKAgUlJSyMtrPKVxfn4+qampACQlJbF792601uTl5ZGSkkJwcDDR0dHExMRQWFhIt27diIuLA6Bz58706tULp9M11jwvL48xY1xvCo8ZM+a8c3kzbda7FgS51iEL8niZzjfeAZUV6K0brI5iGb3iLTh+DOPBGagOHa2O4zVUp84YP3nE9UZ0zgqr43iFJruSnE4nkZHfD7mMjIzkwIEDF93GZrMREhJCRUUFTqeT/v37N2xnt9sbCsB3SkpKOHz4MP369QPg5MmTdOvWDYCuXbty8uSFHwplZWWRleWaJO3ZZ58lKiqqycZeSFBQUKv3/bGanfmUnywnYtwkOrnpmJ7gzjb7Clt0NBU9+2CsX4X9lnusjtMufnid674+QtnKd+k0ZiIRo8dbnMxzWvuzrSfcxolNq6l9/590u+FWbHbf+f/hif/Plj55Onv2LHPmzGHq1KmEhJz/F7ZS6qKv6KelpZGWltbwcWlpaasyREVFtXrfHzOzPoSOnam44ioq3XRMT3Bnm31FVFQU5qgbqH8zk+Pb81Cxfa2O5HHfXWetNWbG/0CHTtTc9oBfX/u2/Gzru6ej//AryhY9h/Hor92czHPa0uaePXte8PNNdiXZ7XbKysoaPi4rK8Nut190m/r6eqqqqggPDz9vX6fT2bBvXV0dc+bMYfTo0YwcObJhm4iICMrLywEoLy+nSxffGNmj62rR2zaiEkagOsptujdSKeMguAM652Oro7QrnZsN+3ej7vopqktXq+N4LdWjJ+qme9B569C7t1odx1JNFob4+HiKi4spKSmhrq6O3NxcHA5Ho20SExPJyckBYNOmTQwcOBClFA6Hg9zcXGpraykpKaG4uJh+/fqhtWbRokX06tWLW265pdGxHA4Ha9asAWDNmjUMHz7cTU31sL0FcLoCNVzmRvJWKjTctZLX5hz0mSqr47QLXXEK/fY/XNNe+PHMqe6ibrobYnpjvv5XdPVZq+NYxvaHP/zhD5fawDAMYmJiWLBgAZ988gmjR48mKSmJpUuXcvbsWXr27EmfPn1Yv349//znPzly5AiPPvooYWFhREREUFlZyUsvvcT69euZPn06PXv2ZN++fbz88stUV1ezatUqVq1aRVRUFJdddhlxcXG89957vPPOO1RWVjJt2jQ6dOjQZEMqKipa9Q0ICQmhqqrtvyT0h0vBWYp68DGvXyfXXW32JQ1t7mp3PWDsFoXq27/pHX1YSEgIpzPnwpEDGDP/CxXRzepIHtfWn21ls6F6X47Oeh/q61ADvH+d9ra0OTw8/IKfV1pr3ZZQ3uLo0aOt2s8d/e26phrzyYdRI0ZjPPyrNh2rPQTqM4aG/vb/+yTU1WL8YYFfTzPd5duvKf/dL1E33oVx10+tjtMu3PWzbb6agd6QhfH0XFSfODck8xxLnjGIZtiVD9VnZG4kH6CUQqXeBEe/ggN7rY7jMbqullOLnoPIaNQt8s5CS6m7pkJouKtABOAke1IY3MDMWwfhEXDlIKujiGZQI8ZASCh69UdWR/EYvXIZ9UVHMB74hQyGaAUVGob6yb+5JtlbHXjvNkhhaCNdfRZ25aMSU1A27362IFxUx46oUTegt+Winf7XpabLStAr3qRjcipqsKPpHcQFqRHXw8Ch6GWvo53HrY7TrqQwtNWufKipQSVeZ3US0QIq9WbQGr3mE6ujuJ35ZiagCJ/2uNVRfJpSCuOBx0DXY/7rb1bHaVdSGNpI528414000OooogVU9xi4drhrER8/mj9J79kO2zaibr4HW/cYq+P4PNU9BnXrfVCwGb1to9Vx2o0UhjbQ1dXoXfmoYcleP0RVnM8YdwtUnHRNfOgHdF0t5ht/c02SN+EOq+P4DZV2O/S+AvNfL6GrTlsdp11IYWiL3flQUy3dSL7qmiFwWSw6+yP8YdS2znofjn2Dce+jqOBgq+P4DRUU5BqGfvJEwKzbIIWhDb7vRpLRSL5IKYUaNwm+LHSto+HDdHmZ6yXLISNQgxOtjuN3VN8rURNuR69dif58h9VxPE4KQyvp6mr0zjzU0GQZjeTDVNJY6ByK/uwDq6O0iX7rH1Bfj/GTf7M6it9St90P0T1d7zacPWN1HI+SwtBau7e6upEc0o3ky1Snzqjr0lxDV0+UNb2DF9L7dqHz1qFuusv1UF14hOrQEWPq41BWgl7+utVxPEoKQyvprRsgrIt0I/kBNfZmME30mpVWR2kxXVfnGkoZGY268S6r4/g91X8AauwkdPaHaD9+c14KQyvomnPdSMOkG8kfqOjLYFAieu0n6Npaq+O0iM75CL75EuPef5NV2dqJuuMhsHfHfGUBuqba6jgeIYWhNXZvheqzMhrJjxjjb4VTJ9Bb11sdpdl0xUn0+/+CgUNhyMimdxBuoTp1do1S+vYb1/ffD0lhaAWdf64b6arBVkcR7nLNEIjphc72nfmT9HtLoPosxk9+5tezxHojNSABNXoC+tPl6MMHmt7Bx0hhaKGGbqShSdKN5EeUYaDGToLD+9E+MHRVFx1Br/0UNXYS6rLeVscJSOruaRDRDfOVF9B1vtUF2RQpDC21e5urG0lGI/kdlTIOOod4/dBVrbVrPqTOIahbZUptq6iQUIwHZ8A3X6I/esvqOG4lhaGF9NZcCAuHq661OopwM9UpxDXrav56755Nc2cefL4Dddv9qNALr8Al2ocaMhyVNBb98VvoLwutjuM2UhhaQNfWoHdukZfa/JgafytovPZZg66rxXzzHxDTGzXmRqvjCEDd+zMI74qZOc9vJmSUwtASe7bB2TMyGsmPqchoVGKKa+oDL3y7Va9eASVHMaY8ggoKsjqOwLWoj/HTmVD8td+8+CaFoQX0to0QEiajkfycSrsNzpxGb/jM6iiN6IpT6A/egEHDZD4kL6MGDUONuRG96j30/t1Wx2kzKQzNpOvq0Du2uPoU5S81v6bir4b4q9Gfve9V6/3q9/8J1Wcw7pludRRxAeruaRDVA3PxX9Bnq6yO0yZSGJpr/26oOo0almx1EtEOjBtuh+PHYEee1VEA0N98iV7zCSr1ZlTPPlbHERegOnXGmJ7umkvpzX9YHadNpDA0k96+ETp0hAFDrY4i2kNCEkRGY65abnWSc8NT/yHDU32A6jcANeEO9LpP0bvyrY7Tas3qEykoKGDx4sWYpsn48eOZPHlyo6/X1taSkZHBoUOHCA8PJz09nejoaACWLVtGdnY2hmEwbdo0EhISAHjxxRfZtm0bERERzJkzp+FYb775Jp999hldunQB4L777mPYsGFuaWxradNEb98MgxJlPpoAoWw21Phb0W9moo8cQF3R37owe7bD3u2onzyCCutiXQ7RLOr2B9C7t2K+koHx/y3wySHFTd4xmKZJZmYmTz31FPPmzWPDhg0UFRU12iY7O5vQ0FAWLFjApEmTWLJkCQBFRUXk5uYyd+5cnn76aTIzMzFNE4DU1FSeeuqpC55z0qRJPPfcczz33HOWFwUADu+Hk07pRgowatQN0KkzetX7lmXQZj3mOy9D9xhU6s2W5RDNp4KDXV1KlSfRSxZZHadVmiwMhYWFxMTE0KNHD4KCgkhJSSEvr3G/a35+PqmpqQAkJSWxe/dutNbk5eWRkpJCcHAw0dHRxMTEUFjoeglkwIABhIWFub9FHqC3bQRbEGqww+oooh2pziGoURPQW9ejnaWWZNAbc6DoCOqOh1FBslynr1B94lG33IvOW4fpg2uKN9mV5HQ6iYyMbPg4MjKSAwcOXHQbm81GSEgIFRUVOJ1O+vf//hbcbrfjdDqbDLVy5UrWrl1LXFwcDz/88AULSFZWFllZWQA8++yzREVFNXncCwkKCrrkvlprynZuwXatg259Lm/VObxNU232R61tc/09D1Oa/QGdNmUT/vAMDyS7OF1dTekH/8TW7xrsN97e4ony5DpbSz/0c8o/L6BuySK6OZKxeWgRJU+02evGXU6YMIG7774bgKVLl/Lqq68yY8b5/yHT0tJIS0tr+Li0tHV/0UVFRV1yX110GPPYN5g3TG71ObxNU232R61usxEMQ5OoWrmMs+NuRXXq7P5wF2F+/Da67DhMe4KyspavLifX2Xr6p4+j/zud0uf/C+P//F+U4f4ZE9rS5p49e17w8012Jdnt9kY/lGVlZdjt9otuU19fT1VVFeHh4eft63Q6z9v3x7p27YphGBiGwfjx4zl48GBTET1Kb9sESqESRliaQ1jHuGEyVJ1G57bfC2+64hT647dhyAjUVbJKoK9S0Zeh7n8U9u9Bf/yO1XGarcnCEB8fT3FxMSUlJdTV1ZGbm4vD0bivPTExkZycHAA2bdrEwIEDUUrhcDjIzc2ltraWkpISiouL6dev3yXPV15e3vDvLVu2EBsb24pmuY/evhH6XYPq0s3SHMI6DS+8rXoPXd8+L7zpj5bC2bMYdz7cLucTnqOSx6GGj0Z/8C/04f1Wx2mWJruSbDYb06dPZ/bs2ZimydixY4mNjWXp0qXEx8fjcDgYN24cGRkZzJw5k7CwMNLT0wGIjY0lOTmZJ598EsMweOSRRzAMVy2aP38+e/fupaKigl/84hdMmTKFcePG8frrr3PkyBGUUnTv3p1HH33Us9+BS9Alxa4Hf1MesSyD8A7GxDsxX/wf9LZc1PDRHj2XLjmKzlmBGn2DvMzmB5RS8OBj6INfYP79eYzf/6VduyRbQ2mttdUh3OHo0aOt2u9S/XPmymXotxdj/OnvqKgebYnnVbytH7Y9tLXN2jQxf/9L6NgJ43dzPbpimvnSn9E78zBmv4Tqeumu10uR6+xd9P7dmM//DpUyFmPqv7vtuJY8YwhkevtG6BPnV0VBtI4yDNTEO+Crg/DFTo+dRx/ah85fj5pwR5uKgvA+6spBqJvuRm/4DJ3v3WuLS2G4CH3CCQe/QA2Vl9qEi0oa61rK8RPPPETUWrteZguPQE2c3OT2wveoW++FvldivrbQNeLMS0lhuAhdsBlA3nYWDVRwMGr8bbC3AP2VB0bL7cqH/XtQt96H6hTi/uMLy6mgIIx/exLqTcx/zPWq2Xt/SArDRejtG6FHL7jM2lFRwruoMRNd02SsXObW42rTxFz2umvqi9E3uPXYwruo6J6o+84NYfXStaKlMFyAPl0J+3ahhiV59CGj8D0qJAx1/Y2udaGPH3PbcXX+eig67FrHWaa+8HsqZRxq5Bj0B2+g9+2yOs55pDBcgN6ZB/X18nxBXJAafysoA53lnsn1dF0d+r0l0Oty1AjPDoUV3kEphXrwMYi+DPPvc9CnTlgdqREpDBegCzZD10i4/NIv44nApOxRrr/21n+KrjjV5uPp3M+gpBhj8oMemTJBeCfVKQTj5/8JpyswM+ehz8087Q2kMPyIrq2BPdtQCSNQhnx7xIWpiXdATQ06Z0WbjqNra1zrOMddBUNk2pVAo2L7ou79Gezdjl75rtVxGshvvh/7YhdUn0XJf1JxCapnHxgyAp39Ibq6utXH0atXwIkyjDsekudZAUpdP9E1Zcby19EH9lodB5DCcB5dsBk6doarrrU6ivByxsQ7ofIUesOqVu2vz1ShP34LBiSgrpaft0CllEI99EvXUrJ/fx5d2fbuybaSwvAD2jTRO7fAoKGoYBkZIprQ7xrX5HqfLkfX1bV4d73qPaiswJj8kAfCCV+iOodg/Pw3UHEC8x/zsXqmIikMP/TVQTjhRA0ZaXUS4QOUUhg33wNlJegta1q0r644hV61HIYlo/pauJ608Brq8njUPdNhV77rZ8NCUhh+QBdsBsNADU60OorwFYMdENsX/fHbLXqLVX/yNlRXY9z+gAfDCV+jxk6CYSnod1+19HmDFIYf0Du2QL8BqLAuVkcRPqLhruHYN7BtY7P20eVl6OyPUEmpMq22aEQphfHTmRDZwzXL7snypnfyACkM5+jjx1xrL8hoJNFSw5IhphfmR281q29Yf/wWaNM1oZoQP6JCQjEemwVnKjH/9udWPb9qKykM5+ideQCoBHm+IFpGGTbUTXdD0WHXRHiXoJ3H0es+RaWMR3locXjh+1TvK1AP/co1n9KyV9v9/FIYztE7tsBlsajoy6yOInyQGjHGNdxwxaXvGvTHb4MGNWlKO6YTvshISkWNneQa9bZ1Q/ueu13P5qV0VSXs341KkG4k0ToqKAg18U44+AVcZFI0XXYcvW4ValQaKjK6nRMKX6SmTIf4qzEXv4AuLmq380phAPSura5J82SYqmgDNSrNtZDPigtPpaxXvAkK1M33tHMy4atUUDDGo/8JHTpg/vVP6LNn2uW8UhgAdmyB8Ajoe6XVSYQPU8EdUDdMhs93oA/ta/Q1XfotekMWatQElL27RQmFL1L2KIyf/Qcc+wb9yoJ2efkt4AuDrq1F796KGiKT5om2U2NuhNDw8+4a9Iq3QCnXQ2ohWkhdMwR1x0OudUDcNN37pQT8b8KavQVwpkpGIwm3UJ06u9Zr2LEFXXQYcA2F1rmfoa6/EWWPsjih8FXqxjthaBL67cXoz3d49FwBXxiqt6yDDh3g6iFWRxF+Qo27xbX854q3AdAfLQXDhrrpLouTCV+mlMKYng49ernebyj91mPnCmrORgUFBSxevBjTNBk/fjyTJ09u9PXa2loyMjI4dOgQ4eHhpKenEx3tGnWxbNkysrOzMQyDadOmkZCQAMCLL77Itm3biIiIYM6cOQ3HqqysZN68eRw/fpzu3bvzxBNPEBYW5q72NqK1pjpvHQwYiurY0SPnEIFHhYa5hhl+8g56ZCp642rU2EmorpFWRxM+TnUKwfjV05iz/w/mwv/BmPVnj5ynyTsG0zTJzMzkqaeeYt68eWzYsIGiosbDprKzswkNDWXBggVMmjSJJUuWAFBUVERubi5z587l6aefJjMzE/PcKkWpqak89dRT551v+fLlDB48mBdeeIHBgwezfLkHJ5P6+jDm8W/lbWfhduqGydChI+aiP4EtCHWj3C0I91DRPV0Po785gn7lBY88jG6yMBQWFhITE0OPHj0ICgoiJSWFvLy8Rtvk5+eTmpoKQFJSErt370ZrTV5eHikpKQQHBxMdHU1MTAyFhYUADBgw4IJ3Anl5eYwZMwaAMWPGnHcud9I7trgeCF7r8Ng5RGBS4V1QqTdBXR0q9SZUV7vVkYQfUYMSXQ+j89ZRveEztx+/ya4kp9NJZOT3t8CRkZEcOHDgotvYbDZCQkKoqKjA6XTSv//3Uwrb7XacTuclz3fy5Em6desGQNeuXTl58uQFt8vKyiIrKwuAZ599lqiolj/UOxN7OXVptxIeF1jTHgcFBbXq++XLrGizef+jVAbZCJvyCEaXiHY9N8h19nf6wZ9z9vI4QkffQCc33zU06xmDVZRSF13uMC0tjbS0tIaPS0tLW36ChGSi0m5t3b4+LCoqStrcXiY/jLOmFiw4t1znADBgGJ21bnWbe/bsecHPN9mVZLfbKSsra/i4rKwMu91+0W3q6+upqqoiPDz8vH2dTud5+/5YREQE5eWuqWbLy8vp0kWmwBZCiPbUZGGIj4+nuLiYkpIS6urqyM3NxeFo3CefmJhITk4OAJs2bWLgwIEopXA4HOTm5lJbW0tJSQnFxcX069fvkudzOBysWeNaDWvNmjUMHz68lU0TQgjRGko345H2tm3beOWVVzBNk7Fjx3LnnXeydOlS4uPjcTgc1NTUkJGRweHDhwkLCyM9PZ0ePXoA8O6777J69WoMw2Dq1KkMHToUgPnz57N3714qKiqIiIhgypQpjBs3joqKCubNm0dpaWmLhqsePXq0Vd+AgLv1RNocKKTNgaEtbb5YV1KzCoMvkMLQfNLmwCBtDgyeKAwB/+azEEKIxqQwCCGEaEQKgxBCiEakMAghhGjEbx4+CyGEcI+Av2OYNWuW1RHanbQ5MEibA4Mn2hzwhUEIIURjUhiEEEI0YvvDH/7wB6tDWC0uLs7qCO1O2hwYpM2Bwd1tlofPQgghGpGuJCGEEI1IYRBCCNGIVy/U42kFBQUsXrwY0zQZP348kydPtjpSm5WWlrJw4UJOnDiBUoq0tDRuvvlmKisrmTdvHsePH280a63WmsWLF7N9+3Y6duzIjBkzfLaP1jRNZs2ahd1uZ9asWZSUlDB//nwqKiqIi4tj5syZBAUFUVtbS0ZGBocOHSI8PJz09HSio6Otjt9ip0+fZtGiRXz99dcopXjsscfo2bOnX1/nDz/8kOzsbJRSxMbGMmPGDE6cOOFX1/nFF19k27ZtREREMGfOHIBW/f/Nycnh3XffBeDOO+9sWH65WXSAqq+v17/61a/0sWPHdG1trf6P//gP/fXXX1sdq82cTqc+ePCg1lrrqqoq/fjjj+uvv/5av/baa3rZsmVaa62XLVumX3vtNa211lu3btWzZ8/Wpmnqffv26d/+9reWZW+rDz74QM+fP1//6U9/0lprPWfOHL1+/XqttdYvvfSSXrlypdZa608++US/9NJLWmut169fr+fOnWtN4DZasGCBzsrK0lprXVtbqysrK/36OpeVlekZM2bo6upqrbXr+q5evdrvrvOePXv0wYMH9ZNPPtnwuZZe14qKCv3LX/5SV1RUNPp3cwVsV1JhYSExMTH06NGDoKAgUlJSyMvLszpWm3Xr1q3hL4bOnTvTq1cvnE4neXl5jBkzBoAxY8Y0tDU/P5/rr78epRRXXnklp0+fblhBz5eUlZWxbds2xo8fD4DWmj179pCUlARAampqozZ/99dTUlISu3fvRvvYGIyqqio+//xzxo0bB7jWOg4NDfX762yaJjU1NdTX11NTU0PXrl397joPGDDgvDVoWnpdCwoKuPbaawkLCyMsLIxrr72WgoKCZmcI2K4kp9NJZGRkw8eRkZEcOHDAwkTuV1JSwuHDh+nXrx8nT56kW7duAHTt2pWTJ08Cru/DDxdPj4yMxOl0NmzrK15++WUefPBBzpw5A0BFRQUhISHYbDbAtfys0+kEGl97m81GSEgIFRUVPrWMbElJCV26dOHFF1/kyy+/JC4ujqlTp/r1dbbb7dx666089thjdOjQgSFDhhAXF+fX1/k7Lb2uP/799sPvS3ME7B2Dvzt79ixz5sxh6tSphISENPqaUgqllEXJ3G/r1q1ERET4ZJ95a9XX13P48GEmTJjAn//8Zzp27Mjy5csbbeNv17myspK8vDwWLlzISy+9xNmzZ1v0V7C/aI/rGrB3DHa7nbKysoaPy8rKsNvtFiZyn7q6OubMmcPo0aMZOXIkABEREZSXl9OtWzfKy8sb/mqy2+2NVn/yxe/Dvn37yM/PZ/v27dTU1HDmzBlefvllqqqqqK+vx2az4XQ6G9r13bWPjIykvr6eqqoqwsPDLW5Fy0RGRhIZGUn//v0BV1fJ8uXL/fo679q1i+jo6IY2jRw5kn379vn1df5OS6+r3W5n7969DZ93Op0MGDCg2ecL2DuG+Ph4iouLKSkpoa6ujtzcXBwOh9Wx2kxrzaJFi+jVqxe33HJLw+cdDgdr1qwBYM2aNQwfPrzh82vXrkVrzf79+wkJCfGp7gWA+++/n0WLFrFw4ULS09MZNGgQjz/+OAMHDmTTpk2Aa4TGd9c3MTGRnJwcADZt2sTAgQN97i/rrl27EhkZ2bCk7a5du+jdu7dfX+eoqCgOHDhAdXU1WuuGNvvzdf5OS69rQkICO3bsoLKyUQmWAAAAAShJREFUksrKSnbs2EFCQkKzzxfQbz5v27aNV155BdM0GTt2LHfeeafVkdrsiy++4Pe//z19+vRp+E9w33330b9/f+bNm0dpael5w90yMzPZsWMHHTp0YMaMGcTHx1vcitbbs2cPH3zwAbNmzeLbb79l/vz5VFZW0rdvX2bOnElwcDA1NTVkZGRw+PBhwsLCSE9Pp0ePHlZHb7EjR46waNEi6urqiI6OZsaMGWit/fo6v/nmm+Tm5mKz2bjiiiv4xS9+gdPp9KvrPH/+fPbu3UtFRQURERFMmTKF4cOHt/i6Zmdns2zZMsA1XHXs2LHNzhDQhUEIIcT5ArYrSQghxIVJYRBCCNGIFAYhhBCNSGEQQgjRiBQGIYQQjUhhEEII0YgUBiGEEI38/6cUaXkSH5aaAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"gdwJu6i3wwdP","executionInfo":{"status":"ok","timestamp":1633622733339,"user_tz":-330,"elapsed":21,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"3501bcf8-9895-44e1-e6bd-c6b2051e69f7"},"source":["plot_lr(cosine(t_max=500, eta_min=0.0005))"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU1f3H8fe5kwWyEJgJIYBRIcSFTSpBQhQJJKCCKCJal7oArQuKjdRfq9haa0ulWhYlWKiNiIiCVXYVNcZgYYwEEWRRJOAWDQYyCAlhSXLP74+R1JQl2yR35s739Tx9nk5y753PyR38zjn33nOU1lojhBBC/MiwOoAQQgj/IoVBCCFELVIYhBBC1CKFQQghRC1SGIQQQtQihUEIIUQtIVYH8JXvvvuuUfvFxsayb98+H6fxb9Lm4CBtDg5NaXOnTp1O+nPpMQghhKhFCoMQQohapDAIIYSoRQqDEEKIWqQwCCGEqKVedyVt2rSJefPmYZom6enpjBo1qtbvKysrycrKYvfu3URHR5OZmUlcXBwAS5cuJTc3F8MwGDt2LH369AHgmWeeYePGjcTExDBt2rSaY5WXlzNjxgz27t1L+/btuf/++4mKivJVe4UQQtShzh6DaZpkZ2czefJkZsyYwbp16ygqKqq1TW5uLpGRkcyaNYsRI0awcOFCAIqKinC73UyfPp2HH36Y7OxsTNMEIC0tjcmTJ5/wfsuWLaNXr148/fTT9OrVi2XLlvminUIIIeqpzsJQWFhIfHw8HTp0ICQkhNTUVAoKCmpts2HDBtLS0gBISUlh69ataK0pKCggNTWV0NBQ4uLiiI+Pp7CwEIDu3buftCdQUFDAoEGDABg0aNAJ7+VL5gfvUf7KPMycFZjud9Gfb0Mf/AGZiVz4gj5Ujvnmq5hvLcV8fzV6ywZ0STHarLY6mhCnVedQksfjweVy1bx2uVzs3LnzlNs4HA4iIiIoKyvD4/GQlJRUs53T6cTj8Zz2/Q4cOEC7du0AaNu2LQcOHDjpdjk5OeTk5AAwdepUYmNj62rKCfZv/pBDH7lrXh8vB4YrjtBeFxLWux/hFw3EiLTXUFZISEij/l6BzIo2H95awMElL9S8Pv75Uq1aE3J+b8J69SW8/yBCOiU0y/vLeQ4OzdFmv37yWSmFUuqkv8vIyCAjI6PmdaOe/LvrQeLatmXft9/AoTL4vhi9pwi96zOObHBzJG81hITCBf0wLr0Mzu9zyjyBRJ4ObRnmgYMAGI/OgtaRUFqC3lME3+zm2GdbOPbxh5S/8AycnYRKTUelDkGFt/LZ+8t5Dg7N8eRznYXB6XRSWlpa87q0tBSn03nSbVwuF9XV1VRUVBAdHX3Cvh6P54R9/1dMTAz79++nXbt27N+/nzZt2tQVsUlUSAgqMhoioyGuE6pXXxh6Ndo04cud6PXvo9e/j/mRGxK6oC6/FpV8CcqQG7pEXX7sI7RqjXLGgjMWldT9v7/17ENv+A86Pw/90hz08oWowcNRGVejbNZLFYGlzv+6JSYmUlxcTElJCVVVVbjdbpKTk2tt07dvX/Ly8gDIz8+nR48eKKVITk7G7XZTWVlJSUkJxcXFdOvW7bTvl5yczJo1awBYs2YN/fr1a2TTmkYZBqrruRg3/Arjb8+hbpsIVVXoZ/+OOeU36B1bLMkl7EM5YzGGXYPjkacwfjcVkrqjX38F8+E7MXNWoKsqrY4ogpTj0UcfffR0GxiGQXx8PLNmzWL16tUMHDiQlJQUFi9ezJEjR+jUqRNnnnkma9eu5aWXXuLLL7/kjjvuICoqipiYGMrLy5k7dy5r165l3LhxNV2XmTNnsnjxYkpLS8nJySEiIoIuXbrQtWtXli9fzmuvvUZ5eTljx44lLCyszoaUlZU16g8QERFBRUXFabdRDgfqzETUoCugQ0f4ZD363VXw/bdwTg+fdv9bQn3abDdWtFl/sxs2fejtAUREnnZb5WyPcdGlqD4p6G+/hLw30R+5UWclensbjSDnOTg0pc3R0dEn/bnSNrkFpyVnV9XHjqJXv4Z+41VoHYFx812o5Esa9f5WkHHYlmGufQc9fxbG1GyUq32999Naw5YNmAvnwP59qPSRqNG3okLr/oL0U3Keg4PMruonVFg4xlU3YfxhBrjiMOc+gbnwH+jKY1ZHEzaglEL17ofxp1moQVegc1ZgPv5/6JLGffkRoqGkMDSB6nwWxoNPoC67Bp33JubU36L37rE6lvAXxzvjjbyRTbXy9kaNiX+A0r2Yf74f/dE63+UT4hSkMDSRCgnBGDMW497fw74SzL8+gC781OpYwkZU734YjzwFnc7EnPM3zNdfkYcwRbOSwuAj6oKLMCb/HSIiMaf9HrNgrdWRhN9o+rMvytUe44G/olLS0MteRM9/Wu5aEs1GCoMPqQ6dMB58Es5OQv/zCcx3V1kdSdiICg1FjbsfNfIG9Lp3MbP+gj561OpYwoakMPiYim6DMenP8LMU9KJ/Yq5+zepIwkaUUhhX3eR9rmb7JsxZj6GPHLY6lrAZKQzNQIWGYtzxW1S/gejX5mOueMnqSMIKNReffT+NinHJUNT4SbBzG+bMP6IrDvn8PUTwksLQTFRICOqXk1Cp6eiVizDffNXqSMJmjP6DMO78LXy5E3PWn2VYSfiMFIZmpAwH6raJqIsuRS95ATPvTasjCSs047yL6sJUjF/+BnZ9hjnncbkgLXxCCkMzU4aBGpsJvZLRL83B/HCN1ZFEi2mZW0pV8iWoWybA1o3of02X9R5Ek0lhaAEqJATjrt9BUg/0vJnoTzdbHUnYjDFwGOq6seiP1qEXPWt1HBHgpDC0EBUWjnHPw9ChM+Y/pqKLv7E6kmhuNR2GllnDwxh2DWrYKPR7b8it0qJJpDC0IBUR6Z3eICTEe7Gw7KDVkYTNqGtvgwsuQi/+F0d/sjqhEA0hhaGFqdgO3p7DDx7MZ6agK+Vioe214Kp/ynB4L0YnnM2Bvz+CLvqyxd5b2IcUBguoxPO8F6QLP0UvlvFg27JoPiPVqjXGvX9AtY7AfOav6IpyS3KIwCWFwSJGv0u8s7KuWY3pzrU6jrAZ1c5FzG+ngGcv5nMzvUvVClFPUhgspK65Fc7thX7xGfQ3X1gdRzSXlhtJqiXsvF6o68bD5vVoecBSNIAUBgsphwPjjgcgMgrzH49Ll992rJ8aWw0Z4X3AcvlC9PaPrY4jAoQUBoupNu0w7noQPPsw58+SefaFTymlULfeCx0TMJ+dhv7BY3UkEQCkMPgBlXge6ppbYOMH6LXvWB1H+EoLP8dwKiq8lXdOpWNHMOc9JdcbRJ2kMPgJNfRqOP8C9KJn0cVFVscRNqM6nYm6/pew/WN0znKr4wg/J4XBTyjDwBh3P4SFYz77pDzfYCct+BzD6ahLL4M+KeglC9Bf7bI6jvBjUhj8iGrrxLj9PvjmC/TSF6yOI5rMv64XKaUwbrsXottgPvt39NEjVkcSfkoKg59RF1yEShuOfmc5+vOtVscRNqOi2nh7pt9/i14iXz7EyUlh8ENqzO3QPh7z+aflW10gq7nDzD+Gko5T51+AGnIlOncVeod8+RAnksLgh1R4K++Q0t496NfmWx1H2JAafav3y8f8p2XNaHECKQx+Sp3TE5U+Ev3e6+gdW6yOI5rCvzoMwPEvH7+Gfd+jl8iXD1GbFAY/pq65BeI6eu89l291gce/rj2fQJ3Twzuk9N4b6M8+sTqO8CNSGPxYzbc6z1700gVWxxE2pK651fvlY/4s9NGjVscRfkIKg59TSd1RaVd4h5S+2Gl1HNEgP3YZ/OQ5hpNR4eEYt070Dim9vsjqOMJPSGEIAGrULRDTDnNBFrpaFnoXvqXO7Ym6OAP99jJZ2EcAEFKfjTZt2sS8efMwTZP09HRGjRpV6/eVlZVkZWWxe/duoqOjyczMJC4uDoClS5eSm5uLYRiMHTuWPn36nPaYW7duZcGCBVRVVdGlSxfuvvtuHA6HL9sccFREJMYNd2DOmYp+dyVq2Ki6dxJ+xH97DMepMbejN6/HfPEZjN9ORRnynTGY1Xn2TdMkOzubyZMnM2PGDNatW0dRUe25fHJzc4mMjGTWrFmMGDGChQsXAlBUVITb7Wb69Ok8/PDDZGdnY5rmKY9pmiazZ8/m17/+NdOmTaN9+/asWbOmeVoeaC4cAL37eadPLi2xOo2ojwCaKVdFtUFdPx52fYZ+/y2r4wiL1VkYCgsLiY+Pp0OHDoSEhJCamkpBQUGtbTZs2EBaWhoAKSkpbN26Fa01BQUFpKamEhoaSlxcHPHx8RQWFp7ymOXl5YSEhNCpUycAevfuzYcffuj7VgcgpRTGTXcCYL40V6bnFj6nUtLgvN7oJS/I9NxBrs6hJI/Hg8vlqnntcrnYuXPnKbdxOBxERERQVlaGx+MhKSmpZjun04nH46k5zv8eMzo6murqanbt2kViYiL5+fns27fvpLlycnLIyckBYOrUqcTGxta3zbWEhIQ0et8WFxvLoZt+RfnzWUQXbqPVgLRGHSag2uwjVrT5UGQk5YAr1oURGd2i7w2Na3PVvZMpvf9WQpcvoO1v/txMyZqPfLZ9dEyfHq2JlFJkZmYyf/58KisrueCCCzBOMdaZkZFBRkZGzetTFZC6xMbGNnpfK+iUdMhZxYF/zaDszCRUeHiDjxFobfYFK9psHvKuyFdaWoo63PK3gjaqzeERqMuv5ejKl9nbfzDqvN7NE66ZyGe7YY6PzvyvOoeSnE4npaWlNa9LS0txOp2n3Ka6upqKigqio6NP2Nfj8eB0Ok97zHPOOYfHHnuMxx9/nPPPP5+OHTs2oJn2pxwOjBvv8D7bsPo1q+OIevH/i88/pS4fDa44zEXPyl1wQarOwpCYmEhxcTElJSVUVVXhdrtJTk6utU3fvn3Jy8sDID8/nx49eqCUIjk5GbfbTWVlJSUlJRQXF9OtW7fTHvPAgQOA906n5cuXM2zYMB83OfCpc3qi+g1Er34NvXeP1XHEqQToZSAVFo5x/Xj49it03ptWxxEWqHMoyeFwMG7cOKZMmYJpmgwePJiEhAQWL15MYmIiycnJDBkyhKysLCZOnEhUVBSZmZkAJCQkMGDAACZNmoRhGIwfP75maOhkxwRYsWIFGzduxDRNhg0bRs+ePZux+YFLjRnrvb3w38/hmDDZ6jjCbn6WAt37oFcsRF80EBUdY3Ui0YKUtsntLd99912j9gvkMUnzjX+jly7AuP9PqO4/q/d+gdzmxrLkGsPby9D/fg7j6UWo1hEt+t7Q9Dbr4m8w/3QfKjUd49Z7fZis+chnu2EafY1B+C81dJR36uSXn0VXVVkdR9iM6pjgnWRv7TvoL2U6lmAihSGAqdBQjJ//CvYUoXNXWR1H2JC68gaIjsF8+Z9o07Q6jmghUhgCnLqgH/Tsi161GF120Oo4opbjk+hZm6IpVESkd1Gf3TvQBf+xOo5oIVIYbMC4biwcPYxeJbNjCt9TA4ZAQhf00gXoymNWxxEtQAqDDahOZ6IGDkOveRO951ur44jjam7rCOAuA6AMA+O6cVBagn53pdVxRAuQwmAT6qqbIDQM87XnrY4ibEidf4F3Esc3/o0uO2B1HNHMpDDYhGrTFnX5tbDpQ/SOrVbHETZkjLkdjh5Br5QhS7uTwmAjaujV4IzF/PdzcgeJX/D/FdwaQnVMQF16Gfr91eg9RXXvIAKWFAYbUWHhqGtuga8K0etlHQvhe2rkjd4hy1eftzqKaEZSGGxGXTQIzurmvYPkmCzubqmaSQXs0WOAH4csrxgDm9ejd2yxOo5oJlIYbKbmDhLPPvS78tCb8D2VcdWPQ5bzZMEom5LCYEPq3J7QKxm9+lX0j2sCCOErKiwcddXN8FUhbPzA6jiiGUhhsCnjmlug4hD6rSVWRwle9htJqqEGpEHHBMxlC2TNBhuSwmBTKqEL6qJB6HdXyPq9wueU4fB++djzLdr9rtVxhI9JYbAxdfVNUF2Nfn2x1VGCm01uVz1Bn/7Q5Rz0ykVyo4PNSGGwMRXXETXwMvR/3kaXNG69CtEU9r4wq5TCuPY22L8PnfeG1XGED0lhsDk14npwhKCXv2R1FGFD6txe0ONn6DdeRVccsjqO8BEpDDan2jpRGVeh17+P/nq31XGCiw2fYzgZ45pb4VAZ+u2lVkcRPiKFIQioy66BiCjMZS9aHUXYkDorEdVvIPqd5eiD+62OI3xACkMQUBFRqOFjYMsG9OcywV6Ls3eHAQB19c1QVYleJTc62IEUhiChBo+Atk7MZS/K06otJYj+zqpDJ9QlQ9Hvv40uLbE6jmgiKQxBQoWFo4ZfDzu3c2xzgdVxhA2pEdeDAv36K1ZHEU0khSGIqEuGgjOWQy8/K72GFhUEY0mAcrb33h7tfhe9d4/VcUQTSGEIIio0FDXi51R+vg22fmR1HGFDavgYMBxyrSHASWEIMio1HUeHTpjLX5Jeg/A51daFGnQF+oP3ZP3xACaFIciokBAirx/rnRlz84dWx7E3ba8V3OpLXTEaQkPRq2QJ0EAlhSEItRp0GcR19PYaZAlQ4WOqTTvU4BHehyq/+9rqOKIRpDAEIeUIQY28AYq+hI9lPv3mc7zHYG0KK6jLRkNYK/RK6TUEIikMQUpddCnEn/Fjr0Hm0xe+paLboDJGojesRRd9YXUc0UBSGIKUMhyoq26E4m/QBWutjiNsSA0dBa0jMVe8bHUU0UAh9dlo06ZNzJs3D9M0SU9PZ9SoUbV+X1lZSVZWFrt37yY6OprMzEzi4uIAWLp0Kbm5uRiGwdixY+nTp89pj7llyxZefPFFTNOkVatW3HPPPcTHx/uyzeJHqu/F6M6veOfTT74E5XBYHcleam76CsKxJEBFRqGGXo1e8RL6q12osxKtjiTqqc4eg2maZGdnM3nyZGbMmMG6desoKiqqtU1ubi6RkZHMmjWLESNGsHDhQgCKiopwu91Mnz6dhx9+mOzsbEzTPO0x//WvfzFx4kSefPJJLrnkEl577bVmaLYAUIaBcdWN8P236A/XWB1H2JBKH+mdwHH5QqujiAaoszAUFhYSHx9Phw4dCAkJITU1lYKC2lMqbNiwgbS0NABSUlLYunUrWmsKCgpITU0lNDSUuLg44uPjKSwsrPOYhw8fBqCiooJ27dr5sLniBD8bAGd2Ra9ahK6qsjqNvQTp7ao/pSIivbP7btmA3r3D6jiinuocSvJ4PLhcrprXLpeLnTt3nnIbh8NBREQEZWVleDwekpKSarZzOp14PJ6a45zsmHfddRePP/44YWFhtG7dmilTppw0V05ODjk5OQBMnTqV2NjYejX4f4WEhDR630D1v20+cvOdHHj8d0Rt30jrIcMtTNZ8rDjP5ZERHAJiY2NRFhQHf/lsm9fdxr6cFYS8vZR2v/97s76Xv7S5JTVHm+t1jaElvf766zz00EMkJSWxYsUKXnjhBe66664TtsvIyCAjI6Pm9b59+xr1frGxsY3eN1D9b5t1l/MgoQsHF2dT3qOvLa81WHGezUMVgPezaUVh8KvPdvpIji17kb0bPkCdnVTn5o3lV21uIU1pc6dOnU768zqHkpxOJ6WlpTWvS0tLcTqdp9ymurqaiooKoqOjT9jX4/HgdDpPecyDBw/y1Vdf1fQyUlNT2bFDup/NTSmFceUNUFKMLnjf6jg24h1KsqIo+Bs15ErvtQaZQykg1FkYEhMTKS4upqSkhKqqKtxuN8nJybW26du3L3l5eQDk5+fTo0cPlFIkJyfjdruprKykpKSE4uJiunXrdspjRkZGUlFRwXffeReu/+STT+jcubPvWy1O1Kc/dD4L/for8lyD8DnVOgI19CrYvB799S6r44g61DmU5HA4GDduHFOmTME0TQYPHkxCQgKLFy8mMTGR5ORkhgwZQlZWFhMnTiQqKorMzEwAEhISGDBgAJMmTcIwDMaPH49heGvRyY4JcOeddzJt2jQMwyAyMpK77767GZsvjlOGgTHyBsw5f0MXrEX1H2R1pMAncxTWooaMRL+9HHPVYhwTJlsdR5yG0jaZYvN4L6OhZEzyv7RpYv7pPtAa49GnUYZ9rjVYco1h+UvoVYtwPLuiRd/3OH/8bJsrXkKvXITxx6dQZ3Tx+fH9sc3NzZJrDCJ4KMNAXflz79PQH7mtjiNsSKVfBa0jMFfKtQZ/JoVB1KL6pkLHBPSqxTLzapPpoH6G4WRUZJT3obeNbvS3X1kdR5yCFAZRizIc3rV7v/taZl4VzUJlXAWtWssqb35MCoM4gep3CcR3xpReQ9NoTbDOk3Q6KjIaNeRK9EfrZL0GPyWFQZygptdQ9CVsklXehO+pjKshLBz9+itWRxEnIYVBnJTqd6l3lbdVi2RtaOFzKrqNd5W3gv+gi4vq3kG0KCkM4qSU48dewzdfwOb1VscJTDKSdFpq2CgIDUO/Ib0GfyOFQZyS6p8G7eMxV0qvQfieio5BpQ1Hf/g+es+3VscRPyGFQZyScjhQw6+Dr3fBlg1WxxE2pC4bBaEh6Df+bXUU8RNSGMRpqZTBENtBeg2NIs8x1EW1aYe69Ar0h3nokmKr44gfSWEQp6VCQry9hi93wraNVscRNqQuuwYc0mvwJ1IYRJ3UgMHgbC+9hoaS5xjqRbV1oi69DJ3/HnrvHqvjCKQwiHpQIaHeXsPuHbB9k9VxhA2py0eDMtBvvmp1FIEUBlFPKjUd2sVirnxZeg3C51RbF2rgULT7XfS+762OE/SkMIh6UaGhqCvGwK7P4LNPrI4TILSMJDWAunwMKIV+8zWrowQ9KQyi3tQlQ6GtS3oNolkoZyzqkqHodTno0r1WxwlqUhhEvXl7DdfCzu2wY4vVcfyfBukyNIy6fAwAerVca7CSFAbRIGrgMIhxYq5cZHUUYUPK1R51cQZ67TtoT3CtxOZPpDCIBlGhYd47SD7fit6x1eo4wobUFdeC1tJrsJAUBtFg6tLLIKYd5irpNZyWliefG0PFdkClpqP/8zZ6f6nVcYKSFAbRYCosHHXZaPjsE/TO7VbHETakrhjj7TW8tcTqKEFJCoNoFHXp5RAdg7nyZauj+DG5XbWxVPt4VMpg9PtvoX/wWB0n6EhhEI2iwn/sNXy6GV34qdVxhA2p4ddBdZX0GiwghUE0mkq7wttrkGsNohmouI6o/mnoNavRB/ZbHSeoSGEQjabCW3lX4dr2MXr3Dqvj+B95jqHJ1IjroUp6DS1NCoNoEpU2HKKi5bkG0SxUh06o/oPQa95EH/zB6jhBQwqDaBLVqjVq6CjY+hH6i8+tjuNn5HZVX1AjroPKKvTbS62OEjSkMIgmU0NGQGQ05qrFVkcRNqTiz0D1G4h+7w102QGr4wQFKQyiyVSrCNTQq+GTAvRXhVbHETakrrweKo+h315mdZSgIIVB+IQaPAIiIuVaw0/JCm4+ozomoJIvQb/3OrrsoNVxbE8Kg/AJFRGJyrgaNq9Hf73L6jjChtSVP4djR9E5y62OYnsh9dlo06ZNzJs3D9M0SU9PZ9SoUbV+X1lZSVZWFrt37yY6OprMzEzi4uIAWLp0Kbm5uRiGwdixY+nTp89pj/nII49w+PBhAA4ePEhiYiK//e1vfdZg0XxU+pXod5ZjrlqMY8Jkq+NYT5as8CnV6UxU34vRuavQw0ahIqOtjmRbdfYYTNMkOzubyZMnM2PGDNatW0dRUVGtbXJzc4mMjGTWrFmMGDGChQsXAlBUVITb7Wb69Ok8/PDDZGdnY5rmaY/52GOP8eSTT/Lkk0+SlJRE//79m6HZojmoiChUxkj4OB9d9IXVcfyDjCT5lBpxPRw5jH5Heg3Nqc7CUFhYSHx8PB06dCAkJITU1FQKCgpqbbNhwwbS0tIASElJYevWrWitKSgoIDU1ldDQUOLi4oiPj6ewsLBex6yoqGDbtm3069fPd60VzU6lXwWtIzBXyh1KwvfUGWfDhaneXsOhcqvj2FadQ0kejweXy1Xz2uVysXPnzlNu43A4iIiIoKysDI/HQ1JSUs12TqcTj8dTc5zTHbOgoICePXsSERFx0lw5OTnk5OQAMHXqVGJjY+tqykmFhIQ0et9A1axtjo2l/MrrOfTv54k5dIDQsxKb530ayIrzXNa6FYcNw7LPl10/25W33IXn/ltp7c4h6sZf1vqdXdt8Os3R5npdY7DCunXrGDJkyCl/n5GRQUZGRs3rffsat9pTbGxso/cNVM3dZn3xUFi5mP0L5mDc9btme5+GsOI8m4cPo7W27PNl2892VFv4WQqHVi7m8MUZqIioml/Zts2n0ZQ2d+rU6aQ/r3Moyel0Ulr638UySktLcTqdp9ymurqaiooKoqOjT9jX4/HgdDrrPObBgwcpLCzkwgsvrGfzhD9RkdGoIVeiN7rR335tdRzraLn63FyMK38Ohw+h311ldRRbqrMwJCYmUlxcTElJCVVVVbjdbpKTk2tt07dvX/Ly8gDIz8+nR48eKKVITk7G7XZTWVlJSUkJxcXFdOvWrc5j5ufnc+GFFxIWFubb1ooWozKuhrBw9OvBfq1Brj43B3VmIlxwETpnObrikNVxbKfOoSSHw8G4ceOYMmUKpmkyePBgEhISWLx4MYmJiSQnJzNkyBCysrKYOHEiUVFRZGZmApCQkMCAAQOYNGkShmEwfvx4DMNbi052zOPcbvcJt8SKwKKi26CGjECvXoIefp33oqEQPmSMvBHzL/ejc5ajrrrJ6ji2orS2R3/3u+++a9R+MibZfPShMsyHfgXn9bb8uQZLrjEsehb9QS6Op6xZ5S4YPtvV/3gctm/CePxZVFSboGjz/7LkGoMQjaUio71DSh/nyxxKolkYV90ER4+g35KZV31JCoNoVirjKu/Mq8tfsjpKy7NHZ9yvqc5nofpd6n2u4aCs8uYrUhhEs1IRkd61obdsCNK1oeXic3NTI2+Aqkr0m69ZHcU2pDCIZqeGjPCuDb18odVRhA2p+M6oAUPQeW9Sva/E6ji2IIVBNDsV3go1/Dr47BP0p5utjtNytKzg1lLUlT8HrTn06nyro9iCFAbRItSgy6GtC3P5QmxyI5zwIyq2A2rgMA7nrEDv3WN1nIAnhUG0CKO8+P8AABcTSURBVBUa5v1Wt+sz2LrR6jjChtSI68DhQMsSs00mhUG0GHVxOsR2wFz2YpD0GrRce25Bqq2LiMtHoz94D72nqO4dxClJYRAtRoWEokbeCF/vgo/zrY4jbCjyml9AWBh6hTUPFdqFFAbRolTKIIg/w3utway2Ok7z0iBdhpZltHWi0keiC/4ji0U1gRQG0aKU4fDOa/Pd1+j171sdR9iQGnYNtI7EXCa3RzeWFAbR4lTfVDgzEb1sIbqy0uo4wmZUZBTq8tGweT36821WxwlIUhhEi1OGgXHtbVBagl7zhtVxmpE8x2AVlX4VtHViLpkfJDc6+JYUBmEJ1b0PdP8Z+vVXZD594XMqPNw7ZLnrM9j0odVxAo4UBmEZ49pbobwM/dYSq6M0D/mmaimVmu690WHJC+hqm9/o4GNSGIRl1JmJqIsGeVfh+qG07h2EaADlcGCMvhX2FKHX5VgdJ6BIYRCWUqNuhmoTvXKR1VGEHfXpD4nnoVe8jD561Oo0AUMKg7CUah+PSrsCvfYddLHNnlbVyMVniymlMK69HQ540DnLrY4TMKQwCMupEddDWDjm0hesjiJsSCV1hwsuQr+1BF120Oo4AUEKg7Ccio7xLubzcb7NFvORi8/+whh9Kxw5gn7jFaujBAQpDMIvqKFXQ4wT85Vse913LkNJfkF1OhN1SQb6vTfQ339ndRy/J4VB+AUV3go1+hb44nOZKkM0C3X1zRASivnqPKuj+D0pDMJvqJTBcFY39Gvz7XEHidbIJHr+Q8W0867ZsOnD4FpJsBGkMAi/oQwD4/rxsH8f+u2lVscRNqQyrgJXnHfI0u6z+zaBFAbhV9Q5PVB9L0avfg29Xx56E76lQsMwxtwORV+i18pDb6cihUH4HXXtbWCaaDvcviojSf6n78XQrTt62YsyT9cpSGEQfke1j0cNvcq7ROMXO62OI2xGKYXx8/FQdgD9xr+tjuOXpDAIv6SuuA7atMVc/Gzg3r4qF5/9ljo7CTVgCPrdFeiSYqvj+B0pDMIvqdYRqGtugV2fofPzrI4jbEiNvgUcoZiL/2V1FL8jhUH4LZWaDl3PRf/7OXRFudVxGi5QezpBQrV1oUbeAJ8UoGXNhlpC6rPRpk2bmDdvHqZpkp6ezqhRo2r9vrKykqysLHbv3k10dDSZmZnExcUBsHTpUnJzczEMg7Fjx9KnT5/THlNrzaJFi8jPz8cwDIYOHcrw4cN92WYRIJRhYNx0F+aU36CXLUTddKfVkRpOnnz2ayp9JHpdDuaiZzHO74MKD7c6kl+os8dgmibZ2dlMnjyZGTNmsG7dOoqKas+CmZubS2RkJLNmzWLEiBEsXOhdhLuoqAi328306dN5+OGHyc7OxjTN0x4zLy+P0tJSZsyYwYwZM7j44oubodkiUKizElFpl6Pz3kR/vcvqOMJmVEgIxs13eZeZfVMuRB9XZ2EoLCwkPj6eDh06EBISQmpqKgUFBbW22bBhA2lpaQCkpKSwdetWtNYUFBSQmppKaGgocXFxxMfHU1hYeNpjvv3224wZMwbD8EaLiYnxcZNFoFGjfgFR0ZgL56BN0+o4DSBDSYFAndsL1X+Qd/ZVmUcJqMdQksfjweVy1bx2uVzs3LnzlNs4HA4iIiIoKyvD4/GQlJRUs53T6cTj8dQc52TH/P7773G73axfv542bdowduxYOnbseEKunJwccnK8D6hMnTqV2NjYejf6p0JCQhq9b6AKvDbHcvj2iRyc9ReiPvmQ1hkjG3wEK9p8ILwVxxwOy/7WgXeem66xba6+8wFK791AyKvP0faRGagAGgJsjvNcr2sMLamyspLQ0FCmTp3Khx9+yD/+8Q8ee+yxE7bLyMggIyOj5vW+ffsa9X6xsbGN3jdQBWKbda9+0K07B+dnUd6tByqqTYP2t6LN5pEjaLPasr91IJ7npmpSm0feyLHF/2Lf2ytQfQNnCLspbe7UqdNJf17nUJLT6aS09L9TE5SWluJ0Ok+5TXV1NRUVFURHR5+wr8fjwel0nvaYLpeL/v37A3DRRRfx1Vdf1beNwsaUUhg33wkVh9CvPGd1nAYInG+ewU4NHgEJXTBffjYw74LzoToLQ2JiIsXFxZSUlFBVVYXb7SY5ObnWNn379iUvLw+A/Px8evTogVKK5ORk3G43lZWVlJSUUFxcTLdu3U57zH79+rF161YAtm/ffsqKJoKPOqML6vJr0R/kord9bHUcYTPK4cC4bSIc/AH96vNWx7FUnUNJDoeDcePGMWXKFEzTZPDgwSQkJLB48WISExNJTk5myJAhZGVlMXHiRKKiosjMzAQgISGBAQMGMGnSJAzDYPz48TUXlU92TIBRo0bx9NNP8/rrr9OqVSvuvDMAb1EUzUZd+XP0RjfmgtkYj85CtWptdaRTk+cYAo46qxtq2NXot5aiL7oUdV5vqyNZQumAnW+gtu++a9zdBDIOG3j0zu2YTzyISh+JccOv6rWPJdcYnpuJ/nwrjqnWPFkb6Oe5MXzRZn30KOafJgJg/HGW3z/bYMk1BiH8jUrqjkobjs5dhd71mdVxTsMW37mCjgoPx7j1Xti7B73yJavjWEIKgwhIavSt0NaFOX8WurLS6jjCZtR5vVEDh6HfXo7+Mvhm+JXCIAKSah2BccsEKP4maL/ViealxtwOMW29Q4KVx6yO06KkMIiApXole7/VrV6C3rnd6jgnkpGkgKYiojBuu8/75WPJAqvjtCgpDCKgqevHedfwfW4G+kiF1XFOFEBP0IoTqZ4XogYPR+csR3+62eo4LUYKgwhoqlUExvj7oXQvenG21XGEDalrx0KHzpjPPxU0D75JYRABT3Xrjrr8GvTad9Cb8q2O8xNaegw2oMLDvV8+fvCgX/6n1XFahBQGYQvqqpu80xnMz0LvL617ByEaQHU5BzXi5+j8PMwgWFFQCoOwBRUSivGr/4PKY5jPPomurrY6kjz5bDNqxPXQrTv6xWfQe4rq3iGASWEQtqE6noH6xd2wczt6hdzCKnxLORwYv3oAQsMw5/wNfeyo1ZGajRQGYStGymDUJUPRb76K3rrR6jjCZpQz1nu94duv0IuetTpOs5HCIGxH3XAHdDoTM3s62rPXuiAaufhsQ6pnX9QVY9D/eRvzg/esjtMspDAI21Hh4Rh3/g6qKjGfedzWXX5hDXX1zXBOT/SC2bacMkMKg7Al1fEMjF/+Br7ehZ4/C2smEZaLz3alHA6Mu34Hbdpizv4r+geP1ZF8SgqDsC11wUWoUb9Ar3+fCsumNJChJLtS0TEY9zwMFeWYc6baajJHKQzC1tQVY1D9BlK+cC5683qr4wibUQldMMbdD7s+897GapNblKUwCFtTSqFuu4+Qrudi/vMJ9O4dLffmNvmPhDg91TcVNfJGtPtd9IqXrY7jE1IYhO2p8HDa/v7vEOPEnPVYyz6cJHclBQU18gbUxRnoVYsw8960Ok6TSWEQQcHR1omR+SgoA3Pmo+gfZNoM4TtKKdQt90DvfuiX5qI3fmB1pCaRwiCChorrhHHfI1B+EHP6I+iDP7TAmzb/Wwj/oBwOjDv+D87uhvns39FbP7I6UqNJYRBBRZ2dhDHxD1D6Pea037dMcRBBQ4W3wvj1H6FTgvc21gB9+l4Kgwg66txeGBMfgX17MKf/AV12oHneSC4+ByUVGY0x6c/Q8QzM2VMCsjhIYRBBSZ3XG+PeP8DeYswnHkKXljTXOzXTcYU/O6E4fOS2OlKDSGEQQUudfwHGr/8EB/djTv0tuugL376B9BiCmopqg/Gbv8BZiZhz/4b53utWR6o3KQwiqKlzemD8diqgvD2HbR/7+A2kxxDManoOP96tZL76PNr0g7VC6iCFQQQ91fksjIeeAGd7zKf+hPnmq7Z5glVYT4WFY9z9ECrtCvRbSzCffgx9qMzqWKclhUEIQDnbYzz0JCr5YvSSF7wLsRxq2sLvWibREz9SDgfGzXejbr0XdmzB/Mskv56VVQqDED9S4a1Qv3oAdd1Y2JSP+ehE3w8tiaBmDByG8X+PQ3U15uP/h7niJXRVldWxTiCFQYifUEphDLsG46EnoXUE5sw/Yi6YjS4/2PCDSYdBnITqei7GH59GXXQpeuUi740PftZ7kMIgxEmos5Mwfj8dNWwUeu07mA/fhZm7quHf7uTiszgJFRmFMX4Sxt0Pwv59mH99APP5p/xmXYcQqwMI4a9UWDjqunHo1HTMRc+iX/4n+q2lqMtHoy7OQIWFWx1RBDh1YSrG+X3Qqxaj312JXv8f1CVDUZeNRrnaW5arXoVh06ZNzJs3D9M0SU9PZ9SoUbV+X1lZSVZWFrt37yY6OprMzEzi4uIAWLp0Kbm5uRiGwdixY+nTp89pjzl79my2b99OREQEAPfccw9nn322r9orRIOpzmd5bznc+hHm6694J0lb/hIqJQ2VOgQSuqJO2jOQsSRRN9U6AnXdWPSgy9BvvoZ+fzV6zZvQKxnj4gzo2RcVGtqimeosDKZpkp2dze9//3tcLhcPPfQQycnJnHHGGTXb5ObmEhkZyaxZs1i3bh0LFy7k/vvvp6ioCLfbzfTp09m/fz9//vOfeeqppwBOe8xbbrmFlJSUZmqyEA2nlPL+Q+3ZFz7fil7j/cer310J7WJRPS+EbuejzjgbOiagQsOO72hpbhE4VFwn1G0T0VfegM57A/3Be5ib10N4Kzi3F+r83qiErtD5LIiMPsWXEd+oszAUFhYSHx9Phw4dAEhNTaWgoKBWYdiwYQPXXXcdACkpKTz33HNorSkoKCA1NZXQ0FDi4uKIj4+nsLAQoM5jCuGPlFLef6Tn9kIfKkNv/AC9dSN6w1r4z9v/7SO0joDDFdDpTCvjigCkXO1R196GHvUL2L4JvaXA+xn7pOC/n6+QUIhqA60jqPrDNAht5dMMdRYGj8eDy+Wqee1yudi5c+cpt3E4HERERFBWVobH4yEpKalmO6fTicfjqTnOqY758ssv8+qrr9KzZ09uvvlmQk/SjcrJySEnJweAqVOnEhsbW68G/6+QkJBG7xuopM0+EhsLZ3WBa25CV1dTvaeIqi93UfXtV5gHf0CXHSCsV19aW/S3lvNsAx0ug8GXAVDt2UfV17uo+moX5g/7vZ+xwxWERkQSG9POp2/rdxefb7rpJtq2bUtVVRVz585l+fLljBkz5oTtMjIyyMjIqHm9b9++Rr1fbGxso/cNVNLmZhIeCef29v7vR5XAIYv+1nKebeiMRO//fkLHtGt0mzt16nTSn9d5u6rT6aS09L+rXZWWluJ0Ok+5TXV1NRUVFURHR5+wr8fjwel0nvaY7dq1QylFaGgogwcPrhl6EkII0TLqLAyJiYkUFxdTUlJCVVUVbreb5OTkWtv07duXvLw8APLz8+nRowdKKZKTk3G73VRWVlJSUkJxcTHdunU77TH3798PUHONIiEhwcdNFkIIcTp1DiU5HA7GjRvHlClTME2TwYMHk5CQwOLFi0lMTCQ5OZkhQ4aQlZXFxIkTiYqKIjMzE4CEhAQGDBjApEmTMAyD8ePHYxjeWnSyYwI8/fTTHDzofcr0rLPO4o477miutgshhDgJpW0yjeR3333XqP1sPyZ5EtLm4CBtDg5NaXOjrzEIIYQILlIYhBBC1CKFQQghRC1SGIQQQtRim4vPQgghfCPoewwPPvig1RFanLQ5OEibg0NztDnoC4MQQojapDAIIYSoxfHoo48+anUIq3Xt2tXqCC1O2hwcpM3BwddtlovPQgghapGhJCGEELVIYRBCCFGL3y3U05I2bdrEvHnzME2T9PR0Ro0aZXWkJtu3bx+zZ8/mhx9+QClFRkYGw4cPp7y8nBkzZrB3717at2/P/fffT1RUFFpr5s2bx8cff0x4eDgTJkwI2DFa0zR58MEHcTqdPPjgg5SUlDBz5kzKysro2rUrEydOJCQkhMrKSrKysti9ezfR0dFkZmYSFxdndfwGO3ToEHPmzOGbb75BKcXdd99Np06dbH2eV61aRW5uLkopEhISmDBhAj/88IOtzvMzzzzDxo0biYmJYdq0aQCN+vebl5fHkiVLABg9ejRpaWn1D6GDVHV1tb733nv1nj17dGVlpX7ggQf0N998Y3WsJvN4PHrXrl1aa60rKir0fffdp7/55hu9YMECvXTpUq211kuXLtULFizQWmv90Ucf6SlTpmjTNPWOHTv0Qw89ZFn2plq5cqWeOXOmfvzxx7XWWk+bNk2vXbtWa6313Llz9VtvvaW11nr16tV67ty5Wmut165dq6dPn25N4CaaNWuWzsnJ0VprXVlZqcvLy219nktLS/WECRP00aNHtdbe8/vee+/Z7jxv27ZN79q1S0+aNKnmZw09r2VlZfqee+7RZWVltf5/fQXtUFJhYSHx8fF06NCBkJAQUlNTKSgosDpWk7Vr167mG0Pr1q3p3LkzHo+HgoICBg0aBMCgQYNq2rphwwYuvfRSlFKcc845HDp0qGaxpEBSWlrKxo0bSU9PB7wLPW3bto2UlBQA0tLSarX5+LenlJQUtm7dig6wezAqKir49NNPGTJkCOBd6zgyMtL259k0TY4dO0Z1dTXHjh2jbdu2tjvP3bt3JyoqqtbPGnpeN23aRO/evYmKiiIqKorevXuzadOmemcI2qEkj8eDy+Wqee1yudi5c6eFiXyvpKSEL774gm7dunHgwAHatfMuGN62bVsOHDgAeP8OP1083eVy4fF4arYNFM8//zy/+MUvOHz4MABlZWVERETgcDgA7/KzHo8HqH3uHQ4HERERlJWV0aZNG2vCN0JJSQlt2rThmWee4auvvqJr167cfvvttj7PTqeTkSNHcvfddxMWFsYFF1xA165dbX2ej2voef3f/7799O9SH0HbY7C7I0eOMG3aNG6//XYiIiJq/U4phVLKomS+99FHHxETExOQY+aNVV1dzRdffMGwYcN44oknCA8PZ9myZbW2sdt5Li8vp6CggNmzZzN37lyOHDnSoG/BdtES5zVoewxOp5PS0tKa16WlpTidTgsT+U5VVRXTpk1j4MCB9O/fH4CYmBj2799Pu3bt2L9/f823JqfTWWv1p0D8O+zYsYMNGzbw8ccfc+zYMQ4fPszzzz9PRUUF1dXVOBwOPB5PTbuOn3uXy0V1dTUVFRVER0db3IqGcblcuFwukpKSAO9QybJly2x9nrds2UJcXFxNm/r378+OHTtsfZ6Pa+h5dTqdbN++vebnHo+H7t271/v9grbHkJiYSHFxMSUlJVRVVeF2u0lOTrY6VpNprZkzZw6dO3fmyiuvrPl5cnIya9asAWDNmjX069ev5ufvv/8+Wms+//xzIiIiAmp4AeCmm25izpw5zJ49m8zMTHr27Ml9991Hjx49yM/PB7x3aBw/v3379iUvLw+A/Px8evToEXDfrNu2bYvL5apZ0nbLli2cccYZtj7PsbGx7Ny5k6NHj6K1rmmznc/zcQ09r3369GHz5s2Ul5dTXl7O5s2b6dOnT73fL6iffN64cSPz58/HNE0GDx7M6NGjrY7UZJ999hmPPPIIZ555Zs0/ghtvvJGkpCRmzJjBvn37TrjdLTs7m82bNxMWFsaECRNITEy0uBWNt23bNlauXMmDDz7I999/z8yZMykvL6dLly5MnDiR0NBQjh07RlZWFl988QVRUVFkZmbSoUMHq6M32JdffsmcOXOoqqoiLi6OCRMmoLW29Xl+5ZVXcLvdOBwOzj77bO666y48Ho+tzvPMmTPZvn07ZWVlxMTEcP3119OvX78Gn9fc3FyWLl0KeG9XHTx4cL0zBHVhEEIIcaKgHUoSQghxclIYhBBC1CKFQQghRC1SGIQQQtQihUEIIUQtUhiEEELUIoVBCCFELf8PrUnKrhV08RQAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"agK_98cWwwdQ"},"source":["### Training Loop\n","\n","Now we're ready to start the training process. First of all, let's split the original dataset using [train_test_split](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) function from the `scikit-learn` library. (Though you can use anything else instead, like, [get_cv_idxs](https://github.com/fastai/fastai/blob/921777feb46f215ed2b5f5dcfcf3e6edd299ea92/fastai/dataset.py#L6-L22) from `fastai`)."]},{"cell_type":"code","metadata":{"id":"216J9e39wwdR"},"source":["X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n","datasets = {'train': (X_train, y_train), 'val': (X_valid, y_valid)}\n","dataset_sizes = {'train': len(X_train), 'val': len(X_valid)}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fFgH5tuLwwdR","executionInfo":{"status":"ok","timestamp":1633622734195,"user_tz":-330,"elapsed":34,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"4e64cc2c-8377-46a1-d220-96fbee9a59bb"},"source":["minmax = ratings_df.RATING.astype(float).min(), ratings_df.RATING.astype(float).max()\n","minmax"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(1.0, 5.0)"]},"metadata":{},"execution_count":35}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WlAmBo9Jys0m","executionInfo":{"status":"ok","timestamp":1633622734196,"user_tz":-330,"elapsed":25,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"59234538-cd74-426f-8543-72cac11d7ee0"},"source":["n_users = ratings_df.USERID.nunique()\n","n_movies = ratings_df.ITEMID.nunique()\n","n_users, n_movies"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(943, 1682)"]},"metadata":{},"execution_count":36}]},{"cell_type":"code","metadata":{"id":"KHeaux79wwdS"},"source":["net = EmbeddingNet(\n"," n_users=n_users, n_items=n_movies, \n"," n_factors=150, hidden=[500, 500, 500], \n"," embedding_dropout=0.05, dropouts=[0.5, 0.5, 0.25])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"onmMNylKwwdS"},"source":["The next cell is preparing and running the training loop with cyclical learning rate, validation and early stopping. We use `Adam` optimizer with cosine-annealing learnign rate. The rate is decreased on each batch during `2` epochs, and then is reset to the original value.\n","\n","Note that our loop has two phases. One of them is called `train`. During this phase, we update our network's weights and change the learning rate. The another one is called `val` and is used to check the model's performence. When the loss value decreases, we save model parameters to restore them later. If there is no improvements after `10` sequential training epochs, we exit from the loop."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4mRf0N9kwwdT","scrolled":false,"executionInfo":{"status":"ok","timestamp":1633622852592,"user_tz":-330,"elapsed":116159,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"50f21317-af76-4dd8-a8f0-5ab7b61cc726"},"source":["lr = 1e-3\n","wd = 1e-5\n","bs = 50\n","n_epochs = 100\n","patience = 10\n","no_improvements = 0\n","best_loss = np.inf\n","best_weights = None\n","history = []\n","lr_history = []\n","\n","device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n","\n","net.to(device)\n","criterion = nn.MSELoss(reduction='sum')\n","optimizer = optim.Adam(net.parameters(), lr=lr, weight_decay=wd)\n","iterations_per_epoch = int(math.ceil(dataset_sizes['train'] // bs))\n","scheduler = CyclicLR(optimizer, cosine(t_max=iterations_per_epoch * 2, eta_min=lr/10))\n","\n","for epoch in range(n_epochs):\n"," stats = {'epoch': epoch + 1, 'total': n_epochs}\n"," \n"," for phase in ('train', 'val'):\n"," training = phase == 'train'\n"," running_loss = 0.0\n"," n_batches = 0\n"," \n"," for batch in batch_generator(*datasets[phase], shuffle=training, bs=bs):\n"," x_batch, y_batch = [b.to(device) for b in batch]\n"," optimizer.zero_grad()\n"," \n"," # compute gradients only during 'train' phase\n"," with torch.set_grad_enabled(training):\n"," outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n"," loss = criterion(outputs, y_batch)\n"," \n"," # don't update weights and rates when in 'val' phase\n"," if training:\n"," scheduler.step()\n"," loss.backward()\n"," optimizer.step()\n"," lr_history.extend(scheduler.get_lr())\n"," \n"," running_loss += loss.item()\n"," \n"," epoch_loss = running_loss / dataset_sizes[phase]\n"," stats[phase] = epoch_loss\n"," \n"," # early stopping: save weights of the best model so far\n"," if phase == 'val':\n"," if epoch_loss < best_loss:\n"," print('loss improvement on epoch: %d' % (epoch + 1))\n"," best_loss = epoch_loss\n"," best_weights = copy.deepcopy(net.state_dict())\n"," no_improvements = 0\n"," else:\n"," no_improvements += 1\n"," \n"," history.append(stats)\n"," print('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n"," if no_improvements >= patience:\n"," print('early stopping after epoch {epoch:03d}'.format(**stats))\n"," break"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["loss improvement on epoch: 1\n","[001/100] train: 0.9756 - val: 0.8986\n","loss improvement on epoch: 2\n","[002/100] train: 0.8512 - val: 0.8780\n","[003/100] train: 0.8775 - val: 0.8851\n","loss improvement on epoch: 4\n","[004/100] train: 0.8071 - val: 0.8705\n","loss improvement on epoch: 5\n","[005/100] train: 0.8338 - val: 0.8697\n","loss improvement on epoch: 6\n","[006/100] train: 0.7598 - val: 0.8624\n","[007/100] train: 0.7931 - val: 0.8698\n","[008/100] train: 0.7192 - val: 0.8733\n","[009/100] train: 0.7555 - val: 0.8743\n","[010/100] train: 0.6720 - val: 0.8844\n","[011/100] train: 0.7104 - val: 0.8882\n","[012/100] train: 0.6229 - val: 0.9149\n","[013/100] train: 0.6686 - val: 0.8936\n","[014/100] train: 0.5796 - val: 0.9359\n","[015/100] train: 0.6257 - val: 0.9201\n","[016/100] train: 0.5433 - val: 0.9525\n","early stopping after epoch 016\n"]}]},{"cell_type":"markdown","metadata":{"id":"EGrWJQGnwwdT"},"source":["### Metrics\n","\n","To visualize the training process and to check the correctness of the learning rate scheduling, let's create a couple of plots using collected stats:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":282},"id":"y3vAtcy9wwdU","executionInfo":{"status":"ok","timestamp":1633622852595,"user_tz":-330,"elapsed":64,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"612b10fe-440e-4422-90ca-cef19cc5ce36"},"source":["ax = pd.DataFrame(history).drop(columns='total').plot(x='epoch')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAEJCAYAAACE39xMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUVfrA8e+5M+mVSUhCIHQQEAExdOkoCoigAiuC8mOLiKu7rmXtnbWytrUDKqwoFkBREEXpUQywKFVAQwkESIFUUib3/P4YCEQS0qZl8n6exyeZmTv3fWcw75w559xzlNZaI4QQot4zPJ2AEEII55CCLoQQPkIKuhBC+Agp6EII4SOkoAshhI+Qgi6EED7C6snghw8f9mT4cqKjo8nIyPB0GpXy9vzA+3P09vzA+3P09vzA93OMj4+v9DFpoQshhI+Qgi6EED5CCroQQvgIKehCCOEjpKALIYSPkIIuhBA+Qgq6EEL4iHpX0PVvv2AufM/TaQghhNepfwV9/6/oZZ+iU1M8nYoQQniVelfQVeKlYLGgf1jt6VSEEMKr1L+CHhYOF3ZH/7gGbZqeTkcIIbxGvSvoAKrXQDieAXu2ezoVIYTwGvWzoHftBQFB6A3S7SKEEKfVz4IeEIDq3hu9cT26pNjT6QghhFeolwUdQPUaBCfzYetGT6cihBBeod4WdDp0gfBITOl2EUIIoB4XdGWxoHoOgJ+T0QV5nk5HCCE8rt4WdDg128VuR29K8nQqQghRLXlFpS47d70u6LRoC7FNZbaLEKJeSE7NY9qS31i286hLzl+vC7pSytFK370NnZXu6XSEEKJCJaUmszYe5cnVqUQHW+kUG+aSOPW6oMOpbhet0T+u8XQqQghxjkM5xdyzfD9LfjnOqAsa8ezwFrSwBbskltUlZ3UjFdMEWl/g6Ha54lpPpyOEEGW++y2bN5OP4GcxeGBgU3o2c03L/LR630KHU6301H3oQ/s9nYoQQlBQUsoL6w/z0vdptLUF8tKIli4v5uArBb1HfzAM9IZVnk5FCNHA7ck8yR1L97Fmfw4Tu0Tz+NDmRAX7uSW2bxT0sAjHCowbVssKjEIIjzC1ZvHOTO79ej92UzNjWHMmXBSNxVBuy8EnCjqc6nbJyoC9OzydihDCQ9Jyi9mceoJSU7s17olCO0+sTOWdzekkNg3lxRGt6BTjmoHP86n3g6KnqW690AGB6A2rUe07ezodIYSb7c44yWMrD5JXbNIoyMqAFmEMbBVB60YBKOW6VvKWtHxeTDpMXrHJtB6xXNEu0qXxzsd3CnpAIOri3uiN69B/+AvKzz19VkIIz9t+tIDHV6USGWjh9oFt+HZnGl/uPs5nu47TLNyfgS3DGdgqnNhQf6fFtJua+T+ls3BHFk3D/Xl0SAItGwU67fy14TMFHRzdLvqHVbBtE1zc29PpCCHcYPPhPJ5ac4iYED8eH5rABc1j6RVjIbeolPUHclidksP7P2fw/s8ZdIgOYmCrcC5tHkZ4YO3L39G8YmauP8wvGYVc3jaCP10SS4DV8z3YPlXQ6dgNwiIwN6zCIgVdCJ/3/cFcnl93iISIAB4bkkDEWUU6LMDCFe0acUW7RhzLK2HNvhxW78vmzeSjzNp4lO7xIQxoGUGvZqE1Ksbr9ufw6oYjKODuS+O5tEW4C15Z7fhUQT+9AqNe/RW6IB8VHOLplIQQLrIqJZuXvk+jXVQgDw9OINTfUumxMaF+XNc5imsvtLHvRBGrUnJYuy+H5EOHCbQa9EkIZWCrCLrEBlc6K6XIbvL2xqN882s2F0QHcme/eKd24TiDTxV0ONXt8u0S9OYk1KWXeTodIYQLLN9zgtd/PELn2GAeGNiMIL/qtbCVUrRqFEirRoHc2K0x248VsHpfDkkHclmZkkNkoIX+LcMZ2DKctrbAssHNfccLeW7dYQ7lFHPdhVFc3yUaqxunI1aXzxV0WraDmCaOpQCkoAvhcz7bmcWczce4JD6Ef/ZvWuu+a4uh6BIXQpe4EG7uEcvGQ3ms3pfDst0nWLLrOE3D/RnQMpwgq8G8LemE+hs8OiSBbk2895u/zxV0xwqMg9BffIg+nolqFOXplIQQTqC15qNtmcz/OYO+zcP4R994/CzOaSX7Wwz6Ng+nb/Nw8opKSTqYy+qUbD74OQOAS+JDuL1PEyLrMJDqDt6dXS2p3gPRSz5A/7gGNXysp9MRQtSR1pr3/pfOop1ZDGkdzl97NXHZFZihARYubxvJ5W0jSc8vIS23mM6xwRgemlteE56fZ+MCKiYeWrWXtV2EcKKNh/J4eNkuVqVku/VKTFNr3kw+yqKdWVzZLpLberuumP9e4xA/usSF1ItiDj7aQgcc3S4fvoU+dADVtLmn0xGi3jqaV8zsTcfYkJpHoNXg290m83/O4JpONoa2jsDP4rp2YampeeWHNFam5HBNJxs3dmvssasw6wOfbKEDqB6XygqMQtRBcanJgq0Z/PWLFH46ks9N3Rqz7Obe3D+gKeEBFl7/8Sh/+ew3PtuZRaHd+YvilZRqnl9/mJUpOdzQJVqKeTX4bgs9PBI6dUP/uAY9ZhLK8NnPLiGcbuOhPN7eeJQjeSX0ax7G1EtiiA72w99q0CshjJ7NQvnpSAGfbM9kzuZjfLw9k9EXNGLEBY3OOx+8uorsJs+sPcSmw/lM7R7D1R1tTnhVvs9nCzqc6naZ/W/YuxPaX+jpdITwemd3rzQL9+exSqbpKaXo1iSEbk1C2JV+kk+2Oy6tX7gjixHtIxndwUZkUO3KS0FJKf9afYhtRwuY3jOO4e0i6/qyGoxqveNbtmzhnXfewTRNhg4dypgxY8o9np6ezuuvv05OTg6hoaHcdtttREV5frqg6tYL7R9wagVGKehCVKa41GTRjiw+2Z6JoeCmbo25qoOtWtMCOzQO4sFBCaQcL+TjbZks3JHFkl+Oc1nbSMZ2tNE4pPoL5eUVlfL4qoPsySzk732bMKhVRF1eVoNTZUE3TZPZs2fz4IMPEhUVxX333UdiYiLNmjUrO2bevHkMGDCAQYMGsW3bNubPn89tt93m0sSrQwUGobr1Rm9aj77+zyirrMAoxO9V1r1SU60aBXJP/6Ycyinm0+2ZfLX7OMv3HGdQqwiu7RRFfPj5L5PPLrTzyHcHOZhdzD39m9InwfVbtvmaKjuW9+7dS1xcHLGxsVitVvr27UtycnK5Y1JTU+nc2bEG+YUXXsjGjRtdky2QWVDC5sN5aF29aVOq90DIz4Vtm12WkxD10dG8Yv61OpUnVqViMRSPDUngnv5Na1XMz9Y03J/b+zThzavbMLxtJGv25XDrF7/x3LpD7DteWOFzMgtKuP+bAxzKKeaBgVLMa6vKFnpWVla57pOoqCj27NlT7pgWLVrw448/MmLECH788UdOnjxJbm4uYWHl/1FWrFjBihUrAHj66aeJjo6uccILk/bzXnIq7RqHcMMlzRjc7vxrKuj+w0h/92X8/vc9kcNGVnqc1WqtVT7u4u35gffn6O35gXtyLLKbzN+UytzkVCwGTO/XkvEXx1dr+mFN8ouOho4tmjAtv5gF/zvMwq1prNufS79WjbixRwKdmzhWKUzLKeTBL7ZxorCUF8Z2plvTunWzNOR/Z6cMik6ePJk5c+awatUqOnbsiM1mw6hgVsmwYcMYNmxY2e2MjIwax7qqTRDhljgW7cji0a9+4bW1vzGmo41hbSIqX9Phkn4UrfuG9IMHUEEVbwsVHR1dq3zcxdvzA+/P0dvzA9fn+Pvulf/rHkPjED+yj2e5NL9xHUK5slVrvtx9nCW7srg55ThdYoMZ2iaCuVvSKbKbPDY4gWYBJXV+/b7+7xwfH1/pY1UWdJvNRmZmZtntzMxMbDbbOcfcddddABQWFrJhwwZCQlyzgI2fxWBYm0iGtI4gOTWPT3dk8dbGo3ywNYNR7R3TpsIDyk+bUr0Gold+id78ParfUJfkJYQ3O3v2StPzzF5xpdAACxMuimZ0Bxtf7z3Bop1ZvJCURkSghRnDmnt8tx9fUGVBb9OmDWlpaRw7dgybzUZSUhK33357uWNOz24xDINFixYxePBglyV8mqFU2XzYneknWbgjkw+2ZrBwRybD2kYypoONmNBTfYGtL4DGcY6LjKSgiwbk7NkrCrixW2NGV3P2iqsE+Rlc3dHGiPaRJB3I5YLoIOLCvGtd8fqqyoJusViYOnUqM2bMwDRNBg8eTEJCAgsWLKBNmzYkJiayY8cO5s+fj1KKjh078sc//tEduQOO+bCdYoLpFBPM/hNFLN7pGF1ftvs4/VuEc00nGy0bBTrmpH+5AH0iExXp+SmVQjhLqanJLS7lxEk72UWlZBeWcqLQTnZhKev255zTveIt/CwGA2VaolMpXd3pIi5w+PBhl5w3Pb+EJbuyWL43m0K7SfcmIYxtYtLp37dhjJuKcfmYc57jrH43rTW5RaUE+RlOXePC1/sF3cHb84MzORbZzbKifHaBPudnkZ3colIqWivLoqB5ZABTLo5xWvdKfXoPvZnH+tDro8Yhfky9JJbxnaNZuuc4X+w6zkNppbTrcxdjtv2PPsN0nVdrM7UmPb+Eg9nFpOYUcTC7mEM5xaRmF5FbbBLsZ9CzaSh9m4fRrUmIV2wgK7xPXnEpuzNOsjP9JL9knCTj5D4y84srXRslyGoQEWghMtBKXJgfHRoHld2ODLQQEWgtux3ib9SbVQKFc/hkQT8tNMDC+M7RXN3Bxne/ZbN4czHPBQwnfvFuxnaJZVCrcPyraEWXlGrScos5mFNEanYxqdmO3w/lFFNceqZZFBFgoVmEP32bh9M03J8D2UVsOJjLqn05BFoNejQNoW/zMC6Jr9mGtMJ3aK05nFvCrvQCdmWcZFf6SQ5mF6MBQ0HLyAA6xYYTaNjPFOgAK5FBjp8RgRb5f0ecl08X9NMCrAZXtm/EZTGa7599gUUXjeHVDZr5P6UzqoONK9pFElRsZ0+m4w8sNbuI1JxiDmYXcySvuNzX2ZgQK83CA7goNpiEiACahfvTLCLgnJk1ALf0jGPb0QKSDuTy/cFc1u7PJcCiSDzVcr8kPrTaeyGK+qfIbrIns5Bd6SfZlVHAroxCcotKAQjxN+gQHUT/FuF0aBxEu6gggvyMetFdILxXgyjop1kjbfSNMejz0+ts/9uLLNx5nHlb0vng53Ts5pmLpSwKmoT50yLSn37Nw2gW4U9CRABNw/0JrKKFpPNz0csXodp2xNqlR9kCRjf3iGX7sTPFff2BXPwtikviQ+jbPJzEpiEE+9V9lTrhOen5JaeKt6P1nXK8kNNf4pqF+9OrWSgdooPo0DiIpuH+0h0inK5BFXRwrMDInBfokn+ArkM68VtWIatSsomzhWOz2mkW4U9cqH+Nd/TWpole9w160VzIy0UHhWA8/ioq0jFn/+wNaf+cGMuu9JOsP5jL9wdy+f5gHn6G4uL4EPo1D6NH01BCnLAEqag9U2vs5un/cPwsPfs+zUm7ya9Zp1rg6SfJPGkHIMCiaBcdxNhOUXRsHET76KAKv8EJ4WwNr6Bf3Avt7+9YgbFtJ1rbAmltC6zTV12dshtz/puwbw+064Qx7GrMt59HL5iFuvmec463GIoLY4O5MDaYP10Swy+ninvSgVx+TM3DakC3OEefe69mYYRKMTgvrTVFpZrsQjs5RaXkFJY6fpb955gRUqwPU1BUUq4o/75Ql5z6WZMd1mJCrFwYE0yHxo7Wd8vIALdtkSbE2RpeQQ8MRnXthU5eh57wpzqtwKhzs9EL56LXfQMRNtSf7kT1HIBSCjVyPPqz99F9h6AuSqz0HIZSdIwJpmNMMFO7x7Ans5CkA7kkHchh4w/5WNQRusaF0Kt1ISWFBRhKYTUUFoMzvyvHh4RFOe63GAqrUhgGpx5Xpx4/c5y/VRERYPHaHWAKSkrJyHcU6Owi+7lFuvD0Y6XkFpWWG6A+m6EgPODUoGJIAIFWhdUwsBqO987PUFgtqux2ufsNhdVCpY/5WRQtIgOIquNiVkI4S4Mr6ACq9yB08lrY/j/o2rPGz9elpejVy9CfvQ9FhajLx6KumoAKPLNOjLriGvSPazDffwPj0VdQgUFVntdQiguig7ggOogpFzdmb9bp4p7L6+v31TjPqgT7GSScGh9IiPAnITyAhIgAokOsbuvfzSsuJTW7mAPZRRzMdkz/PJhdREaBvcLjQ/wMwgIshAdYiAq20rJR4KmCbSE80HF/eIDV8TPQQoifUfahJQOOwtc1yIJOp4shNNzR7VLDgq737HB0r6SmQMeuGNf/BdUk4ZzjlNUPY/KtmM/ei/58Pmp8za6eVUrRLsox++HGbo0JibBxLD2DUn2mS8Buakq1ptR0XC34+9/Pd9zJEpPDuY6ZPBsP5bHi19Ky2IFWRbPwgPLFPiKAmBC/Wncl5BaVlivYB079nnXyTOH2tygSIvzpHBNMQmQAsSF+RJwu0oFWwvwtHr1kXQhv1yALurJaUYmXotevQJ8sqHQFxrPpE1noT99F/7AKbNEY0+6F7n3O22Wh2nVCDRiOXrEE3WsQqkWb2uWrFMH+Fpf2pecUlZJ6VsE9mF3Ez0cKWJmSU3aMv0XRNPx0S/5MsY8LOzOInFNo52AFLe7jhed+YHRrElz2raB5pD+NQ/xk5ocQddAgCzqc6nZZtRT9v+9RfStfsEvb7ejvlqCXfAj2EtTI8agrr0MFVG9lOHXtTeiffsSc9yrGfc+hLN45wBkeYClbE+ds+cWlp+bknynOuzIKWLP/TKG3GhAX6k9eya+cOFlSdn+Q1dGl0z0+tOwDoLmbu3SEaEgabEE/swLjaqikoOudP2F+8BakHYSLEjH+8CdUTOXrKFREBYeiJvwZ/daz6O++QF12tTOyd5sQf0tZv/7ZTpaYHMo505pPzSmmcUQIMQG6rHhHB1u9dtBVCF/UYAu6UgrVcwB66SfoE1mO7VVO0Vnp6I/moDeth8ZxGH99CNW1R+1jJfZDf5/omPXSvS8qqrEzXoJHBfkZtI0KpG3UmW8qMugohGc16OvOVa9BoE3HjBdAl5RgfvkR5kPT0T8no66eiPHYf+pUzMHx4WHcMA20xpz/RrX3QxVCiJposC10ANWkGbRoi96wmqL2nTDfeh6OpUH3PhjjpqKiY50XKyoGdfUN6I/nwOYkuKSf084thBDQwFvo4Niejv17OfHknWAYGH9/DMst9zm1mJfFGnoVNG+D+cFb6II8p59fCNGwSUHvPQjadiT0xukYj7yMuvBi18WyWDBuvBVyHFeYCiGEM0lBD4vA8s9nCBk7qU7LAFQ7Xou2qKFXoVd/hd67w+XxhBANR4Mv6J6grp4ItsaYc19F20uqfoIQQlSDFHQPUIFBGBOnQdpB9PJFnk5HCOEjpKB7iOraA3VJP/QXC9BHXbNZthCiYZGC7kHqD38GP3/Mea/K3HQhRJ1JQfcgFWlDXXMj/LIV/f13nk5HCFHPSUH3MDVgOLTpgP54Djo32y0x9e5tmO+/js7PdUs8IYR7SEH3MGUYGJP/CidPoj+a49JYOvs45ux/Yz53P3rVMvTCeS6NJ4RwLynoXkA1bY4afg36h5XoHVucfn5dWoq54nPMh25Bb1znWAJ40JXotcvR+/Y4PZ4QwjOkoHsJNXIcxDTB/O9r6OIip51X79mB+eQd6AWzoPUFGI/+B2PMJNTYGyEsAnP+m2jTdFo8IYTnSEH3Eso/AGPSdEg/gv7yozqfT+ccx5zzIuaz90JBHsYt92L87VFUrGM9dxUcghr3f5CyG71+RZ3jCSE8r0GvtuhtVMeuqD5D0MsXonv0RzVrWeNzlG1gvfh9KC5y7K40cnyFOyypXoPQa5ajF76H7t4HFRLmhFchhPAUaaF7GTVuKgQFO7peatgVovfuxJzxD/QHb0GrdhiPvoxxzY2VbpenlMKYeDMU5KMXyQCpEPWdFHQvo8LCUeP+CL/uQq/5qlrP0TknMN99CfOZf0JeLsa0f2L8/TFUXLOq4zVrhRo80tFSlwFSIeo1KeheSPUZDB27ohfORZ/IrPQ4bZZirlrqmL3ywyrUFddiPP4q6pJ+NdrLU42eKAOkQvgAKeheSCmFMekWsNsxP3i7wmP0b79gzrgL/f4b0LwNxiMvY1x7EyowqMLjzxsvOAR1nQyQClHfSUH3UiomHjVyPGxOQm/ZUHa/zs3BnPsfzKfuhpzjqL/cg/GPJ1BNEuoWr/cgaNvJMUAqV5AKUS9JQfdiavhYaNoCc/6bmPl5mKu/wnxwGjrpW9TlYzGeeA2jx6U16l6pNJZSGDfIAKkQ9ZkUdC+mrH4Yk2+FE5lkTLsO/d/XIKEVxkMvYYz7P1RgsHPjnT1Aun+vU88thHC9as1D37JlC++88w6maTJ06FDGjBlT7vGMjAxeffVV8vPzMU2TiRMn0r17d5ck3NCoNh1Ql41BbVoPf/gzqucAp7TIK403eiI6eS3m+29g3PssypDPfCHqiyr/Wk3TZPbs2dx///288MILrF+/ntTU1HLHfPrpp/Tp04dnn32Wv//978yePdtlCTdE6ropRL+9CKPXQJcWc5ABUiHqsyoL+t69e4mLiyM2Nhar1Urfvn1JTk4ud4xSioKCAgAKCgpo1KiRa7JtoJRSLi/k5eLJAKkQ9VKVBT0rK4uoqKiy21FRUWRlZZU7Zty4caxdu5Zp06bx1FNPMXXqVOdnKtxGBkiFqJ+cspbL+vXrGTRoEFdddRW7d+/mlVdeYebMmRi/639dsWIFK1Y4vsY//fTTREdHOyO8U1itVq/K5/fcnl90NLkjrqPgy4+JuGo8fm06VPkUeQ/rzttz9Pb8oGHnWGVBt9lsZGaeuVoxMzMTm81W7pjvvvuO+++/H4D27dtTUlJCbm4uERER5Y4bNmwYw4YNK7udkZFRp+SdKTo62qvy+T1P5KcvGwtrvibr1aerNUDqzBy11qC1Uwdlvf3fGLw/R2/PD3w/x/j4+Eofq/KvpU2bNqSlpXHs2DHsdjtJSUkkJiaek9y2bdsASE1NpaSkhPDw8FolK7yHpwZI9dHDmDPuxJz5gCxFIEQNVNlCt1gsTJ06lRkzZmCaJoMHDyYhIYEFCxbQpk0bEhMTufHGG3nzzTf58ssvAZg+fbpbB/GE66je7l1i19ywGj3vNdAmFBeh133j2HdVCFElpbXWngp++PBhT4U+h7d/TfNkfjo1BfOJO1ADhmPccEulx9UlR11chF4w27HCZNuOGH++C3P2C3BoP8aTr6NC6/6Nz9v/jcH7c/T2/MD3c6xTl4sQZVeQrv7KJVeQ6iOHMJ+6B73mK9SV12LcOQNla+xYq/2kzLQRorqkoItqUaOvdyyx+/4bTu3XNjesxnzyH3AiA+P2RzCuuQlldfQEqqYtUEOvQq/9Gp0ia7ULURUp6KJaVHAo6topThsg1cVFmPNeRc+aCQktMR56CXXRJefGvep6CG+E+f7raLO0znGF8GVS0EW1qT6DT11BOrdOV5DqI6mYT92NXrPc0cVy179Qtorn5KqgYMdm1vv3otd9U+uYQjQEUtBFtZXtQZqfh17831qdw9HFciecyDzTxWKxnD9uzwHQvjN64Tx0bk6t4grREEhBFzWiElqhhtR8gLR8F0urSrtYKoypFMbEaacGSOfWNnUhfJ4UdFFjNR0gLd/Fch3GXTMq7WKpNGbT5qhho9HrvkGn7K5t6kL4NCnoosbKDZAmfXveY8/MYsnE+NsjGNfcWGUXS6Vxr/oDRDQ69UEiA6RC/J4UdFErZQOkn1a8xK4uLsKc+59TXSytHV0snavXxVJpzMBgx1IE+/ei18oAqRC/JwVd1Mr5Bkj1kVTMf92FXvt1rbtYKo3bcwBccJFjpo0MkApRjhR0UWsVDZCaP6xydLFkH69zF0uFMZXCuP5mKDopA6RC/I4UdFEnavT1EBqO+f4b5Lz2NHr2v091sbxY5y6WSmM2bY4aemqA9LdfXBJDiPpICrqoExUcWrbE7slvPnd6F0ulca+a4Bggnf+mDJAKcYpTdiwSDZvqMxiyjhHRrQe5zdq4J2ZgMGrcVPTbz6PXfI0adKVb4grhzaSFLupMKYUx6g8EdOvl3rg9+jsGSBe5/wpS/esuSp++B713h1vjCnE+UtBFvVU208bNA6Rm0reYz98Pv+7C/OBt2VVJeA0p6KJeU/GnBkjXfo3+dZdLY+nSUsyPZqPfeQnadkJN+BMc+BW9KcmlcYWoLinoot5TV02ASJtLB0h1fh7mK4+jv/kMNWQUxt8eRQ0ZCU1boBf/F223uySuEDUhBV3UeyowGDX+j47W8prlTj+/TnNcKMWuragb/4px/V9QVivKsGCMnQzHDqOT3LeJthCVkYIufIJKvBQ6dDk1QJrttPPqrZswn7oLTuZj3PkkRv/Lyx/QpQe06YBe8iG6qMhpcYWoDSnowiecGSAtRC+s+wCp1hpz+ULMVx6H6FiMB/6Natep4rjX3AQnstArv6hzXCHqQgq68BmqScKZJXbrMECqi4vQc15Af/IuqntfjH8+g4pqXHnc9hfCRYnoZZ+g8/NqHVeIupKCLnyKGjUBIqMw59duiV19PBPzufvRP6xCjZmEuvkeVEBglc8zxk6GkwXo5Qtrk7YQTiEFXfiUMwOkv6FX12yAVP/2C+aMOyEtFePW+zFGjkcpVb24Ca1QPQegv/0cfSKrNqkLUWdS0IXPUYn9HAOki6s/QGomfYf53P3g54dx37Oobr1rHvfqG6C0FP3FhzV+rhDOIAVd+JxyA6SfvnfeY7VZivnxHPQ7L0KbDhgPzEQ1bVG7uI3jUAOGOy5yOnq4VucQoi6koAuf5BggvRq9fkWlA6S6IA/zlSfQXy9GDR6J8ffHUKHhdYs7cgJY/dCfvV+n8whRG1LQhc8qGyB9//VzBkgduyrdDTt/Rk2+FWPizShr3RcfVRGNHB8kyWvRB36t8/mEqAkp6MJnqcAgxwDpwRT06q/K7tdbNzmKeUEexj+ewBgw3Llxh4+FkDDMRfOcel4hqiIFXfg0ldgPOnZFL/4v5okszOWLMF95AqJjHP3l7S90fszgENSV18G2zehftjr9/EJURpEiG10AAByQSURBVAq68Gln9iAtIvOu/0N/8g50733qYqEY18UdPMLR3bNwLlprl8UR4mxS0IXPU02aoS4fg5mZjrp6IsbN/6zWxUJ1iukf4Nhv9bdf4KcNLo0lxGlS0EWDoMZMIvqthRij/lDti4XqHLPvUIhrirlwnux7KtxCCrpoEJRhYGkc596YFgvGmMmQdhD9/Sq3xhYNkxR0IVypex9o0Rb9+Xx0SbFbQ+uiInRWultjCs+Sgi6ECymlMK69CbLS0auXuS2uTj+C+eQdmA/fis4+7ra4wrOkoAvhYqpjV8fUyS8/Rp8scHk8/dsvmE/dDTnHoaQY/ZWsANlQVOvSuC1btvDOO+9gmiZDhw5lzJgx5R5/99132b59OwDFxcVkZ2fz7rvvOj1ZIeorY+yNmP+6E/3NYtToiS6LozcnYc76N0Q0wrj9EfTyT9Grl6GHj0FFRrksrvAOVRZ00zSZPXs2Dz74IFFRUdx3330kJibSrFmzsmOmTJlS9vuyZctISUlxSbJC1FeqVTu4pC/668/Qg0agwiOden6tNfqbzxzz7Fu2w/jrg44YIyegf1iFXvYp6vq/ODWm8D5Vdrns3buXuLg4YmNjsVqt9O3bl+Tk5EqPX79+PZdeeqlTkxTCFxhjJkFJEXrpx049ry4tRc9/E/3xHLi4D8ZdM8o+MFTjOFTfoeg1X6GzMpwaV3ifKlvoWVlZREWd+aoWFRXFnj17Kjw2PT2dY8eO0blz5wofX7FiBStWOHZHf/rpp4mOjq5Nzi5htVq9Kp/f8/b8wPtz9Hh+0dHkDBnJyVVf0Wj8FCwxTc45pKY5micLyJ75MMWbkgi+eiKhN05HGeXbaaWTbibj++8IWLmE8JvvrtNL8Ph7WA0NOce6Ly93lvXr19O7d28Mo+KG/7Bhwxg2bFjZ7YwM72kxREdHe1U+v+ft+YH35+gN+enLxsLq5WS++x+MqXec83hNctQnMh3r0hzch7phGkWDRlCUVcFuSYYfqt8wTn6zhKJBo867P2pVvOE9rIqv5xgfH1/pY1V2udhsNjIzM8tuZ2ZmYrPZKjw2KSmJfv361SJFIRoGZYtGDRnp6Nc+tL/W59Gp+xwrRh49jHHbgxiDRpw/7ohxoEAv/ajWMYX3q7Kgt2nThrS0NI4dO4bdbicpKYnExMRzjjt06BD5+fm0b9/eJYkK4SvUlddBYHCtl9fV2/+H+cw/QZsY9zyFuujcv8dzYtoao/oPd2z4kX6kVnGF96uyoFssFqZOncqMGTO444476NOnDwkJCSxYsICNGzeWHbd+/Xr69u3rtnUyhKivVEiYY830n35E791Zo+eaa5ZjvvwYRMdi3Pc8qnmb6scdcR0oA/2ltNJ9VbX60Lt370737t3L3TdhwoRyt8ePH++8rITwcWrYaPR3X2AufA/j7qeqbAhp03Rser3sU7jwYseKkUHBNYsZGYUadCX6uy/QI65DxVTeFyvqJ7lSVAgPUAGBqFF/gD07YNvm8x6rS4rRs2Y65pIPGI7x14dqXMzL4l5xLVit6C8W1Or5wrtJQRfCQ1T/y6BxnGMTDNOs8Bidm4P574fQyWtR196EmjS9TnufqohGqEEj0D+sRh9JrfV5hHeSgi6EhyirH+rqGyA1BZ289pzH9dHDmE/fDfv2ov5yD8YV1zpljEoNvwb8/NBLpJXua6SgC+FBqkd/aNYK/dn7aHtJ2f167w5HMS/Iw7jzSYwezrv6WoVHooaMQievQacddNp5hedJQRfCg5RhYFxzI6QfQa/9BgAzeS3mzIcgOAzjvudQbTs6P+7lY8E/EL3kQ6efW3iOU68UFULUQufu0P5C9Bcfkmfa0R/OgradMG69HxUa7pKQKiwcNfQq9LKP0SPGoZq1dEkc4V7SQhfCw5RSGGNvhJwT5H84C9VzAMY/HndZMS+Le/nVEBiEKa10nyEtdCG8gGrbETVyPCGNbBT0v+KcBbZcEjMkzDEffsmH6AO/oZq3dnnM39NFReDvLxckOom00IXwEsaYSYRce6NbivlpathoCArBXPKB22Kepn/dhXn3TehP3nV7bF8lBV2IBkwFhzq6XrZsQO/f67a4ev9ezJceg8KTjitXT2RW/SRRJSnoQjRwauhoCA7F/Nw9rXR9MAXz3w9DcAjGXTNAm7LvqZNIQReigVNBwY7Fwn5ORqfsdmksffgA5r8fgoBAjDufRLXvjOozBL36K2mlO4EUdCEEashICA3H/Hy+y2LoI4ccxdxidRTzxnGO2CPGSSvdSaSgCyFQgcGoK66BbZtrvKRvdehjaZgzHwTTxLjzCVTsmZUeVeM4VO/B0kp3AinoQggA1KAREBbh9Fa6zjzmKOYlxRj/eALVJOHc2CPHg1kqrfQ6koIuhABOLel7xbWw8yf07m1OOac+nuko5oUFGHc8XukVqapxnKMvfc1y9IkK9kUV1SIFXQhRRg28EiIaOWXGi84+7ijmudkYf38M1eL8uyupkeOh1I7+6tM6x26opKALIcqogADHnqe/bEXv+rnW59G52Y5ifiIT42+PoFpVvdewo5U+WFrpdSAFXQhRjhowHCJtmJ/PR2td4+fr/FzHPPOMoxi3PYRq26n6sUecaqUvl7702pCCLoQoR/n5Owrrnh2w86caPVcX5GO+8AgcOYhx6wOoCy6qWeyYJmfNeJFWek1JQRdCnENdehnYomvUSteFBZgvPQqp+zBuuQ914cW1iz1y3KlW+qJaPb8hk4IuhDiH8vNzDFL+ugu2n38TawBdVIj58uOwbw/GX+5GdelR+9gx8ada6cvQ2cdrfZ6GSAq6EKJCqu9QiIrB/Oz8rXRdXIT5nydh7y7Un+5Ede9T99inW+kyL71GpKALISqkrKda6fv2wM8bKzxGl5Rgvv4U/LIV9X9/w+jR3zmxY+JRvQZJK72GpKALISql+gyBxnEV9qVrewnmm8/Ats2oybdi9Bns3NijZMZLTUlBF0JUSlmtqFET4MCv8NOGsvt1aSnm2zPhpx9RE6dh9L/c+bFj4lG9BkorvQakoAshzkv1GgQx8ZiffYA2TbRZip7zImxOQo3/I8bgEa6LPXIClEgrvbqkoAshzktZLKir/gCpKRR9vwr93n/QP65GXXMjxmVXuzZ2bDyq96lWeo77W+n6RBbm+hVos9TtsWtDCroQokqqZ3+Ia0b2S4+jk75FXXU9xpXXuSd2WSvdvfPSdfZxzOcfQL/7MnrDGrfGri0p6EKIKinDgnH1RCgpRl15naPF7q7Ysaf60lctdVsrXefmODbjOJEJMU3Qn72PLilxS+y6kIIuhKgWlXgp0W8vQo2djFLKvbFHjj/VSl/s8lg6Pw/zxYch/QjGXx/EuGEaZB5Dr17q8th1JQVdCFFtluhYtxdzABXX9FQr/Ut0zgmXxSlbvuDQAYzp96E6dEF1uhg6dkV/+RG6IN9lsZ1BCroQol4400p3TV+6LipyLF+wfy/GzfegOl9S9phx7U2Ql+v168tIQRdC1AuOVvoAl7TSdUkx5qtnLV9wce/ysVu0RfXoj16x2KtXgZSCLoSoN8pa6V87r6Ws7SWYrz8NO39CTbmt0uUL1JhJUFqKXvKh02I7mxR0IUS9oeKaoXr2R69c6pRWetkVr1s3oiZNx+g7tPLYMU1QA65Ar/safeRQnWO7ghR0IUS94piXXlLnVnq5K14n/BFj4BVVxx41Hvz8MRfPq1NsV7FW56AtW7bwzjvvYJomQ4cOZcyYMecck5SUxMcff4xSihYtWvC3v/3N6ckKIYRqclYrffg1qLCIGp9DmyZ63muOK17HTsYYVr0rXlV4I9TlY9BLPkT/9guq9QU1ju1KVRZ00zSZPXs2Dz74IFFRUdx3330kJibSrFmzsmPS0tJYvHgxTzzxBKGhoWRnZ9cqGa01hYWFmKbp9qlRR48epaioyOVxtNYYhkFgYKBHpn8J4QvUyAnoH9egly9CXTelRs/VWqM/fAu97hvUqAkYI8bVLPblY9CrlmF++h7GXTO86u+4yoK+d+9e4uLiiI2NBaBv374kJyeXK+jffvstw4cPJzQ0FICIiJp/YgIUFhbi5+eH1VqtLw5OZbVasVgsbollt9spLCwkKCjILfGE8DWqSTNUjwHolV+ih4+tditda43+5F30yqWoy8eiRk+seezAYNSoCegP3oJtm+GiS6p+kptUWTmzsrKIiooqux0VFcWePXvKHXP48GEAHnroIUzTZNy4cXTr1u2cc61YsYIVK1YA8PTTTxMdHV3u8aNHjxIQEFDzV+Ek7vogsVqtKKXOef1VPacmx3uCt+fo7fmB9+foTfnZJ99MZvIaAtcuJ+zG6WX3ny/HvA9mkf/1IoJGXEfYn+6odetaj72BzO++QH3+PraBl6GMmg1Huup9dEoFM02TtLQ0HnnkEbKysnjkkUd4/vnnCQkJKXfcsGHDGDZsWNntjIyMco8XFRW5rZX8e1arFbvd7rZ4RUVF57z+84mOjq7R8Z7g7Tl6e37g/Tl6VX6Boage/SlY+gmF/YeXtdIry9Fc9gl64VzUpZdRdPUkijMz6xTeHD0R/fbzpC/9FKN3zTb3qMv7GB8fX+ljVX6s2Gw2Ms964ZmZmdhstnOOSUxMxGq1EhMTQ5MmTUhLS6tVskIIUV1q1AQoLkJ/ff41XswVnzmKec+BqMnTa9yirjB24qXQvDV6sfcs3FXlq2rTpg1paWkcO3YMu91OUlISiYmJ5Y7p2bMn27dvByAnJ4e0tLSyPvf6JDs7m3fffbfGz5s8eXKtB4KFELWnmiQ4ruBc+SU6N6fCY8zVX6EXzIbufVFT/44ynNMLoAzDsSSAFy3cVWVBt1gsTJ06lRkzZnDHHXfQp08fEhISWLBgARs3OjaO7dq1K2FhYdxxxx089thjTJo0ibCwMJcn72w5OTnMnTv3nPur6oqZN29erQeChRB1U9ZK/+bceelm0rfo/74GXXpg/PlOlJO7dL1t4S6lf7/zqxudHkw9raCggODgYADMD99GH0xxajyV0ArjD3+u8DGr1cqf//xnvv76a1q3bo2fnx8BAQFERESwd+9e1q1bx9SpUzl8+DBFRUX88Y9/ZNKkSQD06tWLZcuWkZ+fz6RJk+jZsycbN24kLi6OOXPmVDib5ezXWh1e1XdZCW/P0dvzA+/P0VvzM996Dv1zMsZTs2jcqjUZGRmYyWvRb8+EDhdh3PYQys/fJbH1/r2YT/4DNWI8xthJ1XqOx/rQG5L777+fFi1a8M033/Dggw+ydetWHn/8cdatWwfAzJkz+eqrr1i6dClz5swhK+vcRXpSUlK46aabWLlyJeHh4Sxd6h1fxYTwZb9vpestP6BnzYS2HTBufcBlxRzOXrjrM48v3OX+Cd/VVFlL2p26detG8+bNy27PmTOHZcuWAY5vFykpKecMECckJNC5c2cAunTpwsGDB92XsBANlIpvjkq8FP3dUk5e0BnzzWehRVuM2x9GBQS6Pv6YSejNSegvPkRNml71E1xEWujncXaXSFJSEmvXrmXJkiWsWLGCzp07V3hl6dnz6C0WC6Wl9WNzWSHqO0crvZCcFx+D+OYYf38UFVj9bs06xY5pghowHL3Wswt3SUE/S0hICHl5eRU+lpubS0REBEFBQezdu5fNmze7OTshxPmo+OaoAcOxtuuE8ffHUcGh7o0/aoLHF+7y2i4XT7DZbPTo0YMhQ4YQGBhY7kquQYMGMW/ePAYOHEibNm3o3r27BzMVQlRE3XALtujoctfOuC22Fyzc5bWzXNzN3VeKyiwX9/P2/MD7c/T2/MCzOerCAsz7b3Z0+dz5ZKVLC8gsFyGE8HKnF+7il62w3f3dslLQhRDCidSA4dA4DvPT99Cm6dbYUtCFEMKJlNXPsf9o6j70j6vdGlsKuhBCOJmnFu6Sgi6EEE7mqYW7pKALIYQLeGLhLinoddCuXTtPpyCE8GLGtTdBXi7663NXgnRJPLdEEUKIBqhs4a5v3LNwl9deKTpr41FSjhc69ZytGgXyp8TKN97417/+RXx8PFOmTAEcqytaLBaSkpLIzs7Gbrdzzz33MHz4cKfmJYTwXe5cuEta6GcZPXo0S5YsKbu9ZMkSxo0bx+zZs1m+fDkff/wxjz/+OB68uFYIUc+4c+Eur22hn68l7SqdO3cmIyODI0eOkJmZSUREBDExMTz66KNs2LABpRRHjhwhPT2dmJgYt+cnhKif1KgJ6KTvMBfPwzLtXpfF8dqC7imjRo3iyy+/5NixY4wePZqFCxeSmZnJsmXL8PPzo1evXhUumyuEEJUpt3BXym44a+E/Z5Iul98ZPXo0n332GV9++SWjRo0iNzeX6Oho/Pz8WL9+PampqZ5OUQhRD6nLx0BYhGNJABd120pB/50LLriA/Px84uLiiI2N5ZprruGnn35i6NChfPLJJ7Rt29bTKQoh6qGzF+4q/t8Gl8SQLpcKfPvtt2W/22y2cgOlZ9uzZ4+7UhJC+AA1YDh622aU1TWlVwq6EEK4ibL6Ybn9Yfyjo8EFa7ZLl4sQQvgIryroDWl+d0N6rUII9/Cqgm4Yhlu3gfMUu92OYXjVWy+E8AFe1YceGBhIYWEhRUVFle7F5yoBAQFumV+utcYwDAIDA10eSwjRsHhVQVdKERQU5JHY9WHzWyGEOB/53i+EED5CCroQQvgIKehCCOEjlJb5c0II4ROkhX7Kvfe6bklLZ/D2/MD7c/T2/MD7c/T2/KBh5ygFXQghfIQUdCGE8BGWRx999FFPJ+EtWrdu7ekUzsvb8wPvz9Hb8wPvz9Hb84OGm6MMigohhI+QLhchhPARUtCFEMJHeNVaLu6WkZHBq6++yokTJ1BKMWzYMEaMGOHptM5hmib33nsvNpvNK6dk5efn88Ybb3Dw4EGUUtxyyy20b9/e02mV88UXX/Ddd9+hlCIhIYHp06fj7+/v0Zxee+01Nm/eTEREBDNnzgQgLy+PF154gfT0dBo3bswdd9xBaGio1+Q3b948Nm3ahNVqJTY2lunTpxMSEuKR/CrL8bQlS5Ywb948Zs2aRXh4uFflt2zZMpYvX45hGHTv3p1JkyY5J6BuwLKysvSvv/6qtda6oKBA33777frgwYMezupcS5Ys0S+++KJ+6qmnPJ1KhV555RW9YsUKrbXWJSUlOi8vz8MZlZeZmamnT5+ui4qKtNZaz5w5U69cudKzSWmtt2/frn/99Vf9j3/8o+y+efPm6UWLFmmttV60aJGeN2+ep9KrML8tW7Zou92utXbk6sn8tK44R621Tk9P108++aS+5ZZbdHZ2toeyqzi/rVu36scff1wXFxdrrbU+ceKE0+I16C6XRo0alY00BwUF0bRpU7KysjycVXmZmZls3ryZoUOHejqVChUUFLBz506GDBkCgNVq9WiLrTKmaVJcXExpaSnFxcU0atTI0ynRqVOnc1rfycnJDBw4EICBAweSnJzsidSAivPr2rUrFosFgPbt23v876WiHAHee+89brjhBrcvw/17FeX39ddfc/XVV+Pn5wdARESE0+I16C6Xsx07doyUlBTatm3r6VTKeffdd5k0aRInT570dCoVOnbsGOHh4bz22mvs37+f1q1bM2XKFK9a791ms3HVVVdxyy234O/vT9euXenataun06pQdnZ22YdNZGQk2dnZHs6oct999x19+/b1dBrnSE5Oxmaz0bJlS0+nUqG0tDR27drFhx9+iJ+fH5MnT3Za3WnQLfTTCgsLmTlzJlOmTCE4ONjT6ZTZtGkTERERXj2ntrS0lJSUFC6//HKeffZZAgICWLx4safTKicvL4/k5GReffVV3nzzTQoLC1mzZo2n06qSUsrjLczKLFy4EIvFQv/+/T2dSjlFRUUsWrSICRMmeDqVSpmmSV5eHjNmzGDy5Mm88MILTtuSssEXdLvdzsyZM+nfvz+9evXydDrl/PLLL2zcuJFbb72VF198kW3btvHyyy97Oq1yoqKiiIqKol27dgD07t2blJQUD2dV3tatW4mJiSE8PByr1UqvXr3YvXu3p9OqUEREBMePHwfg+PHjHhvMO59Vq1axadMmbr/9dq/7wDl69CjHjh3j7rvv5tZbbyUzM5N//vOfnDhxwtOplbHZbPTs2ROlFG3btsUwDHJzc51y7gbd5aK15o033qBp06aMGjXK0+mcY+LEiUycOBGA7du3s2TJEm6//XYPZ1VeZGQkUVFRHD58mPj4eLZu3UqzZs08nVY50dHR7Nmzh6KiIvz9/dm6dStt2rTxdFoVSkxMZPXq1YwZM4bVq1fTo0cPT6dUzpYtW/jss8947LHHCAgI8HQ652jevDmzZs0qu33rrbfy1FNPedUHY48ePdi+fTudO3fm8OHD2O12wsLCnHLuBn2l6K5du3j44Ydp3rx5WUvj+uuvp3v37h7O7FynC7o3Tlvct28fb7zxBna7nZiYGKZPn+6xqXaV+eijj0hKSsJisdCyZUumTZtWNijlKS+++CI7duwgNzeXiIgIxo8fT48ePXjhhRfIyMjw+LTFivJbtGgRdru9LKd27drxl7/8xSP5VZbj6QF68HxBryi/AQMGlI05Wa1WJk+eTOfOnZ0Sr0EXdCGE8CUNvg9dCCF8hRR0IYTwEVLQhRDCR0hBF0IIHyEFXQghfIQUdCHq4NixY4wfP57S0lJPpyKEFHQhhPAVUtCFEMJHNOhL/4VvysrKYs6cOezcuZPAwEBGjhzJiBEj+Oijjzh48CCGYfC///2PJk2acMstt5StypeamsqsWbPYt28fNpuNiRMnkpiYCEBxcTEffvghP/zwA/n5+TRv3pyHHnqoLObatWtZsGABxcXFjBw5kmuuucYTL100cNJCFz7FNE2eeeYZWrZsyZtvvsnDDz/M0qVL2bJlCwAbN26kT58+zJkzh379+vHcc89ht9ux2+0888wzdOnShVmzZjF16lRefvllDh8+DMDcuXP57bffePLJJ3nnnXeYNGlSuYWpdu3axUsvvcRDDz3EJ598Qmpqqkdev2jYpKALn/Lrr7+Sk5PDddddV7ZN2tChQ0lKSgKgdevW9O7dG6vVyqhRoygpKWHPnj3s2bOHwsJCxowZg9VqpXPnznTv3p1169ZhmiYrV65kypQp2Gw2DMPgggsuKLcWzLhx4/D396dly5a0aNGC/fv3e+otEA2YdLkIn5Kens7x48eZMmVK2X2madKxY0eio6OJiooqu98wDKKiosqWq42OjsYwzrRxGjduTFZWFrm5uZSUlBAXF1dp3MjIyLLfAwICKCwsdOKrEqJ6pKALnxIdHU1MTEyF68Z/9NFHZGZmlt02TZPMzMyyHYIyMjIwTbOsqGdkZNCkSRPCwsLw8/PjyJEjXrsLjhAgXS7Cx7Rt25agoCAWL15McXExpmly4MAB9u7dC8Bvv/3Ghg0bKC0tZenSpfj5+dGuXTvatWtHQEAAn3/+OXa7ne3bt7Np0yb69euHYRgMHjyYuXPnkpWVhWma7N69m5KSEg+/WiHKk+Vzhc/Jyspi7ty5bN++HbvdTnx8PBMmTGDXrl3lZrnExcUxbdq0si3+Dh48WG6Wy/XXX0/Pnj0BxyyX+fPn8/3331NYWEjLli154IEHOHHiBH/961/54IMPyjZPfvTRR+nfv7/XbuwtfJcUdNFgfPTRRxw5csTrdn0Swlmky0UIIXyEFHQhhPAR0uUihBA+QlroQgjhI6SgCyGEj5CCLoQQPkIKuhBC+Agp6EII4SP+H1yyjenPc+ZxAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"8SNXU2CKwwdU","executionInfo":{"status":"ok","timestamp":1633622853651,"user_tz":-330,"elapsed":1109,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"c4ba4191-0a03-4cff-8a2c-afc478a1658e"},"source":["_ = plt.plot(lr_history[:2*iterations_per_epoch])"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxVdf7H8df3ey8uLC4XBNKhJhFnCi0TTKRFFKopmxlqrKlfy7g0Zm6R2ZQyU7ZQtiiOYjtZmc1YU1RTTTZItkgUZtjYKtrGiJJcUxRL4Hx/f9xivAmieOHcA5/n49GjLpx7eJ/T1Tfn+z2LMsYYhBBCiB9ouwMIIYQILlIMQggh/EgxCCGE8CPFIIQQwo8UgxBCCD9SDEIIIfy47Q4QKFu2bGnV+6Kioti+fXuA07QfJ+d3cnZwdn4nZwdn5w+m7H379m3y63LEIIQQwo8UgxBCCD9SDEIIIfxIMQghhPAjxSCEEMLPIZ2VVFZWxtKlS7Esi/T0dDIzM/2+X1dXR15eHps3byYiIoKsrCyio6MBKCgooKioCK0148ePZ8iQIQDce++9rFu3jp49ezJ//vzGde3evZvc3Fy++eYb+vTpwzXXXEN4eHigtlcIIUQLWjxisCyL/Px85syZQ25uLmvWrKGiosJvmaKiIsLCwli8eDFjxoxh+fLlAFRUVFBcXMyCBQvIzs4mPz8fy7IASEtLY86cOQf8vOeee47BgwezaNEiBg8ezHPPPReI7RRCCHGIWiyG8vJyYmNjiYmJwe12k5qaSmlpqd8ya9euJS0tDYCUlBQ2bNiAMYbS0lJSU1MJCQkhOjqa2NhYysvLATj++OObPBIoLS1l5MiRAIwcOfKAnxVIVslr7F6Rj/Xy01ivPodV9CLWGysxa9/CfLwe89VmjPcbTN2+NssghBCtYSq+wHp+OaZmZ8DX3eJQktfrJTIysvF1ZGQkGzdubHYZl8tFaGgoNTU1eL1eEhISGpfzeDx4vd6D/rydO3fSu3dvAHr16sXOnU1vdGFhIYWFhQDMmzePqKioljblADvKStjz3tsHfL2pB1ToXh50n1hcP/7TNw730f1xxx2LDrNvqMvtdrdq24OBk7ODs/M7OTs4O3+gsu/9eB27XlyB51eZuAO8L4L6ymelFEqpJr+XkZFBRkZG4+tWXUk4eTbRHg/bt22D+jqor4e6fbB3D+yugT27MLtrYNe3GO831FdXUV/+Cbz7pm/5H3mioO8xqJ8noPr/AvoPRIVFHH6eVgimqygPl5Ozg7PzOzk7ODt/oLJbu2oA2LFjB6prWKvW0dyVzy0Wg8fjobq6uvF1dXU1Ho+nyWUiIyNpaGigtraWiIiIA97r9XoPeO9P9ezZkx07dtC7d2927NhBjx49Wop4RJTWqJAQCAnZ76v/a9+maslYFlRXwZavMFu+8v37688xH76PMb45FGL6oeJ/CcediDruRFTP3m26HUIIESgtFkN8fDyVlZVUVVXh8XgoLi5mxowZfsskJSWxevVqBg4cSElJCYmJiSilSE5OZtGiRZx77rns2LGDyspKBgwYcNCfl5yczOuvv05mZiavv/46w4YNO7ItbANKa+gTC31iUSee3Ph1891e+LIcs/lTzKZPMOvfheJVvqGpfsf4CmJQEvxiEMod0uz6hRDCTi0Wg8vlYsKECeTk5GBZFqNGjSIuLo4VK1YQHx9PcnIyo0ePJi8vj+nTpxMeHk5WVhYAcXFxjBgxgpkzZ6K1ZuLEiWjtm+9euHAhH330ETU1NUyePJkLL7yQ0aNHk5mZSW5uLkVFRY2nqzqF6tYdfjEY9YvBwA9HFl9/jvm4zDeZ/formMIXoHsYanAy6qThMCjJ9z4hhAgSyhjT1Fyr4zjh7qpm3/fw8XrM+yW+o4ndu8AdAicko1NG+Uoi5PCOJGSs1T5Ozu/k7ODs/AGbY3jndczD89G33ouK/Vmr1tHqOQYROKpLVzjxZNSJJ2MaGmDTx5h1b2PefQNr3dsQGo4adioqJQ3ij2t24l0IIdqSFINNlMsFAwehBg7CXDABPirDlKzGvF2Eef0VOCoOlXY2KmUUKrR1ZxwIIURrSDEEAeVyweAk1OAkzHe1mLVrfPMRf3sQ88xjqOEjfSVxdLzdUYUQQSfwIwtSDEFGdQtFnXoGnHoG5styzOp/Yd5ZjXnzVRg4CH3Web65CC33PxRCtA0phiCmjhmA+sN0zAXjMW8VYla9gLX4Vt8w01nnoU4eaXdEIUQHJL92OoAKDUefmYnOeRA18RpwuTCPLsKa80dq/7nCd7aTEEIEiBwxOIhyu1EpozDD0+CjMqx//YOaR/4KzyxDnTMWddqZqJAudscUQjicFIMDKaUg8SRciSfRY+tX7Fh2n2+i+l/PoM65AHXaGXJltRCi1WQoyeG6DBqKnnU7euatEBWDefJ+rBunYpW+RQe5dlEI0c7kiKEDUErBcSeif3kCfPg+1jOPYh68C/PvgegLJqASjrc7ohAi0NrwFz8phg5EKQWDhqKPPxHz9mrMc8uw7roBhqSgf/cHVGw/uyMKIRxAhpI6IKVd6FPS0bc9gMq8FD5ejzV3Otazj2G+/87ueEKIQGqDW+dIMXRgqmtX9JgL0bffjzr5dMy/nsH6yxTfo0tl/kEI0Qwphk5A9eiNnpCFvn4ehEdgPXAXVu6NmMoKu6MJIYKQFEMnogYcj85egLp4EnxRjnXzDKznn8TU1bX8ZiFEpyHF0Mkolws9+lz0bfehkk/BvPh3rFuzMJs+sTuaECJISDF0UqpHL/QV16Jn3ATf78W683qsvz3oezypEKJTk2Lo5NTgJPTNeai0czCvvYR10zTMhnV2xxJC2EiKQaC6haL/70r0n+ZB125Yf52L9cS9cmqrEEGt7c4slGIQjdSA49B/yUWdmYl5YyXWzTNk7kGIYNcGTwCWYhB+VEgX9AUT0NfmgGVh3XkDVsEyTL2cuSREZyHFIJqkfjEIfdMiVOpozMtPY90+C/Pfr+yOJYRoB1IMolmqeyh63Az01DnwrRcrZybWGyvlqmkhOjgpBtEiNSQFfdMiSDges2wJ5sG7MbV77I4lhGgjUgzikKievdFXz0WdfzlmXTHWLVdjNn9qdywhRBuQYhCHTGmNPnus77RWwLrrBqyVz2Isy+ZkQohAkmIQh03F/xJ940I4cTjmH49iLb4Vs6fG7lhCdC5tONUnxSBaRYWGoydfj7pksu95D7deg/lyk92xhOiE5HkMIogopdBp56D/dIfvmod5f8J66992xxJCHCEpBnHEVP9foP+S6ztr6bHFWI/nYer22R1LCNFKUgwiIFRET3TWXNQ5F2DefBXrzhsw27fZHUsI0QpSDCJglHahz7sMPTUbqiqxbpuJ+Xi93bGEEIdJikEEnBoyHP3n+dCjF9bCm7Bee0mulhbCQaQYRJtQ0X3Rs++GwcmYJx/APHGv3IhPCIeQYhBtRnUPRU+Z45t3eGMlVu6NmJqddscSomNow6Nw96EsVFZWxtKlS7Esi/T0dDIzM/2+X1dXR15eHps3byYiIoKsrCyio6MBKCgooKioCK0148ePZ8iQIQdd53/+8x+eeOIJLMuiW7duTJ06ldjY2EBus2hHSmvUeZdh9T3ad8ZSzrXoadmonx1rdzQhOgZlw3UMlmWRn5/PnDlzyM3NZc2aNVRUVPgtU1RURFhYGIsXL2bMmDEsX74cgIqKCoqLi1mwYAHZ2dnk5+djWdZB1/nwww8zffp07r77bk499VSeeeaZgG+0aH96+Ejf9Q4NDVjzrsese9vuSEKIZrRYDOXl5cTGxhITE4Pb7SY1NZXS0lK/ZdauXUtaWhoAKSkpbNiwAWMMpaWlpKamEhISQnR0NLGxsZSXl7e4zr17fQ+kr62tpXfv3gHcXGEn9fMEdPZ86HcM1v3z2PP8kzIpLUQQanEoyev1EhkZ2fg6MjKSjRs3NruMy+UiNDSUmpoavF4vCQkJjct5PB68Xm/jeppa5+TJk7njjjvo0qUL3bt3Jycnp8lchYWFFBYWAjBv3jyioqIOaYN/yu12t/q9wcBx+aOiMLffz85Ft7L70Ty6V/6XiD9eg3Id0qhmUHHcvt+Pk7ODs/MHKvveiAh2Ab1798Yd4H0RdH8aX3rpJWbPnk1CQgIvvPACjz/+OJMnTz5guYyMDDIyMhpfb9++vVU/LyoqqtXvDQZOzW/+MIPQmL7UFjzB3v9+hb7yOlS3ULtjHRan7ntwdnZwdv5AZbdqfDeu3LFjB8rdtVXr6Nu3b5Nfb3EoyePxUF1d3fi6uroaj8fT7DINDQ3U1tYSERFxwHu9Xi8ej6fZde7atYsvv/yy8SgjNTWVTz+Ve/53REprIi6fgrpsKnz0vu9Kaa8z/6AL0dG0WAzx8fFUVlZSVVVFfX09xcXFJCcn+y2TlJTE6tWrASgpKSExMRGlFMnJyRQXF1NXV0dVVRWVlZUMGDCg2XWGhYVRW1vLli1bAPjggw/o169f4LdaBA19+lnoGTfB9m1Yd8zCfCV3aBXCbi0OJblcLiZMmEBOTg6WZTFq1Cji4uJYsWIF8fHxJCcnM3r0aPLy8pg+fTrh4eFkZWUBEBcXx4gRI5g5cyZaayZOnIjWvi5qap0AV155JfPnz0drTVhYGFdddVUbbr4IBirxJPT1d2ItvgXrrtnoSdehThhmdywhglzbnbihTAc5LeTHo4zD5eSxSnB2/p9mN99WYy2+DSo+R102FX3qGTama1lH2vdO4+T8AZtjKF6FWfpX9O0Povq07lqvVs8xCNFeVK9I9HW3w3En+i6Ge+kpOZ1VCBtIMYigorp1R0/7MyolDfPcE5i/PYCxGuyOJUSnEnSnqwqh3CEwPgt69sasLMDs+hY9cSYqpIvd0YToFKQYRFBSWqPGjsfq6cE8lY9Vsws9NRsVGmZ3NCE6PBlKEkFNn/Fb1BXXwqZPsO6ejfm2uuU3CSGOiBSDCHp6+Ej0jBvhm22+G/BtrWj5TUKIVpNiEI6gjh/iO2Np3/dYd82WC+GEaENSDMIx1DHx6OvvhJAuWPdkYzZ+ZHckIezThmdySzEIR1ExfdHXz4OevbEW3ojZ8J7dkYSwlx0P6hEi2ChPH/R1d0BMP6y8HMzat+yOJESHIsUgHEn16IWelQPHJmA9eA/Wm6/aHUmIDkOKQTiWCg1HZ90CiUMwj+dhvfqc3ZGE6BCkGISjqa5dfRe+JZ2CefoRrOeXy/2VhDhCcuWzcDzlDoFJs2BZKObFFVC7B35/BUrL7z1CtIYUg+gQlHbB5dOgeyjm389D3T64dIqUgxCtIMUgOgylFFwwAUK6Yl5+CurrYNwMX2kI0eG03ZCpFIPoUJRSqPMuxQoJwTy/HOrrYcI1KLd81EUH1QbXMcifFtEh6XN/7yuHfzyKqa/zPS7UHWJ3LCEcQQZgRYelzzofddEf4f0SrHvvwNTtszuSEI4gxSA6NJ3+a9SlU+A/a7HybsN8/73dkYQIelIMosPTI3+FGnc1fLwea9HNmO/22h1JiKAmxSA6BX1KOmriTCj/CGvhTZjaPXZHEiJoSTGITkMPH4me9Cf4YiPWgr9g9tTYHUmIoCTFIDoVlZSKvmo2/PcLrAU3SjkI52rDW79IMYhOR514MnrKHNjypZSD6ADkeQxCBIQanLxfOciwkhD7k2IQnZavHLJhy1dSDkLsR4pBdGpqcNIP5fC1lIMQP5BiEJ2eGpyEnjpHykGIH0gxCAGoQfuVw/w/Y3bvsjuSELaRYhDiB43lUFnhO3KQchCdlBSDEPvxlUO2lIMIfnIdgxDtRw0aKuUgOjUpBiGa4FcO86UcRBAL/PVtUgxCNEcNGoqe9mfYKuUgOpdDeoJbWVkZS5cuxbIs0tPTyczM9Pt+XV0deXl5bN68mYiICLKysoiOjgagoKCAoqIitNaMHz+eIUOGHHSdxhj+/ve/U1JSgtaaM844g3POOSeQ2yzEIVOJJ6Gn/Rkr7zas3BvRM29DhYXbHUuINtXiEYNlWeTn5zNnzhxyc3NZs2YNFRUVfssUFRURFhbG4sWLGTNmDMuXLwegoqKC4uJiFixYQHZ2Nvn5+ViWddB1rl69murqanJzc8nNzeWUU05pg80W4tCpxJN+uH3GV1i5N2Jqd9sdSYg21WIxlJeXExsbS0xMDG63m9TUVEpLS/2WWbt2LWlpaQCkpKSwYcMGjDGUlpaSmppKSEgI0dHRxMbGUl5eftB1vvrqq4wdOxatfdF69uwZ4E0W4vCpwUnoybOh4gushXMxe2vtjiREm2lxKMnr9RIZGdn4OjIyko0bNza7jMvlIjQ0lJqaGrxeLwkJCY3LeTwevF5v43qaWue2bdsoLi7m3XffpUePHowfP56jjjrqgFyFhYUUFhYCMG/ePKKiog55o/fndrtb/d5g4OT8jsuefjbfhYex8+5sXPfmoG9e5Kz8+3Hcvv8JJ+cPVPa9ERHswvf3qivA++KQ5hjaU11dHSEhIcybN4933nmH++67j1tuueWA5TIyMsjIyGh8vX379lb9vKioqFa/Nxg4Ob8js8cfj/7jddQ9eBfbb86iYUo2qms3u1MdNkfu+/04OX+gsls1vlu3eL1eFK5WraNv375Nfr3FoSSPx0N1dXXj6+rqajweT7PLNDQ0UFtbS0RExAHv9Xq9eDyeg64zMjKS4cOHA3DyySfz5ZdfHuo2CtEuVFIq6oprqfvkP1iLb8V8/73dkYQIqBaLIT4+nsrKSqqqqqivr6e4uJjk5GS/ZZKSkli9ejUAJSUlJCYmopQiOTmZ4uJi6urqqKqqorKykgEDBhx0ncOGDWPDhg0AfPTRR802mhB20sNOo8fVN8JnH2ItuQ2zT8pB2CXwFzK0OJTkcrmYMGECOTk5WJbFqFGjiIuLY8WKFcTHx5OcnMzo0aPJy8tj+vTphIeHk5WVBUBcXBwjRoxg5syZaK2ZOHFi46RyU+sEyMzMZNGiRbz00kt069aNK6+8MuAbLUQgdD/9TGq+/Rbz6F+x7r0dPTUbFdLF7lhCHDFlTBvecKMdbdmypVXvc/JYJTg7v5Ozw//yW2/9G/PYYhicjL5qNiokxO5oLeoo+96JAjbH8OarmMfz0Hc+gvK0bvK51XMMQoiD06eegbpsCvxnLdYDd2Lq6+yOJMQRkWIQIgD06b9CXTIZ1r+L9eDdmPp6uyMJ0WpSDEIEiE47B3XRJHi/BOvhezANDXZHEqJVpBiECCCdfi7qwonwXjEmf4GUg3CkoLvATQin02f8FstqwPzjUXC5YPzVKN26C5CEaFYbnjckxSBEG9BnnY9VX4957glQGsbNQGk5QBdtQNlwHYMQonX0mAuxLAvzwpO+I4fLpko5CEeQYhCiDelfX4TVUI956SnQLrj0KlQb/IYnRCBJMQjRxtRvLwGrAfOvZ8Cl4eIrpRxEUJNiEKKNKaXgvMuhwcK8WuA7cvj9FVIOImhJMQjRDpRSMHYcNNRjVv3TN+cwdryUgwhKUgxCtBOlFPz+Ct+w0qvP+c5W+t0fpBxE0JFiEKIdKaXg4ivBGMzKZ0H7hpmkHMThk+sYhOgwGsvBMr4JaaUh81IpB9E6bfCxkWIQwgZKa7hkMhgL8/LTvouUfnuJlIMIClIMQthEaQ2XTvENK730FGiN+s3/2R1LCCkGIeyktIbLpvqOHP75dywU+jcX2x1LdHJSDELYTGkNl08HA+aff8PSCn3uRXbHEp2YFIMQQUBpDX+Y5jtyeP5JLKXRYy60O5bopKQYhAgSSrtg3AzfnMNzT2AphT7nArtjiU5IikGIIKK07/kNWAZTsMx35HD27+yOJYJR213GIMUgRLBR2gUTsgCDefYx35zDWefbHUt0IlIMQgQh5XLBhGt8w0r/eNQ3rHTmeXbHEkFJHtQjRKehXC6YOBMsC/P0Ut+w0hm/tTuW6ASkGIQIYsrlgiuuxWAwT+X7jhwyfmN3LNHByXMGhQhyyu1GXzELho7ArHgYa9WLdkcSHZwUgxAOoNxu9B+vg5NSMH9/EKtIykG0HSkGIRxCud3oSdfBkOGYvz2IteqfdkcSHZQUgxAOotwh6Cv/9MORw0NYrxbYHUl0QFIMQjiMcoegJ/0JlXSK72yll5+2O5Kwg5EH9Qgh9qPcbvjjLHC5fVdINzSgfy033uuU2uAZHlIMQjiU7zqHLHC5MC88idVQj5KH/YgAkGIQwsEab7zndvse9lNfD7/7g5SDOCJSDEI4XOOT4FwuzMpnoaEeLpwo5SBaTYpBiA5AaQ3/N9k351D4gq8cLprk+7oQh+mQiqGsrIylS5diWRbp6elkZmb6fb+uro68vDw2b95MREQEWVlZREdHA1BQUEBRURFaa8aPH8+QIUMOaZ2PPPIIr732GsuWLQvEdgrR4Sml4PdX+I4cXn0OGhrgkqukHMRha/ETY1kW+fn5zJkzh9zcXNasWUNFRYXfMkVFRYSFhbF48WLGjBnD8uXLAaioqKC4uJgFCxaQnZ1Nfn4+lmW1uM5NmzaxZ8+eAG+qEB2fUgo1djzq7LGYN1ZiHl+MsRrsjiUcpsViKC8vJzY2lpiYGNxuN6mpqZSWlvots3btWtLS0gBISUlhw4YNGGMoLS0lNTWVkJAQoqOjiY2Npby8/KDrtCyLJ554gksvvTTwWytEJ6CUQp13GercizBrVmEeWYhpkHLoeGy8jsHr9RIZGdn4OjIyko0bNza7jMvlIjQ0lJqaGrxeLwkJCY3LeTwevF5v43qaWucrr7xCUlISvXv3PmiuwsJCCgsLAZg3bx5RUVEtbUqT3G53q98bDJyc38nZwQH5J85gT8+e7F7+AF0w9Lz2FlRIF8AB2Vvg5PyByl4bHk4N4ImMxNXLc+TB9hNUk89er5e3336buXPntrhsRkYGGRkZja+3b9/eqp8ZFRXV6vcGAyfnd3J2cEj+tDGoBsP3f3+QqrlZ6ClzUF27OSP7QTg5f6CyW7t3A+D1VqPqrVato2/fvk1+vcWhJI/HQ3V1dePr6upqPB5Ps8s0NDRQW1tLRETEAe/1er14PJ5m1/nFF1+wdetWZsyYwdSpU9m3bx/Tp08/vC0VQvjR6eeixl0NH3+AlXsjpna33ZFEkGuxGOLj46msrKSqqor6+nqKi4tJTk72WyYpKYnVq1cDUFJSQmJiIkopkpOTKS4upq6ujqqqKiorKxkwYECz6xw6dCgPPfQQS5YsYcmSJXTp0oXFixe3yYYL0ZnoU9LRV14HX5Rjzf8z1s4ddkcSQazFoSSXy8WECRPIycnBsixGjRpFXFwcK1asID4+nuTkZEaPHk1eXh7Tp08nPDycrKwsAOLi4hgxYgQzZ85Ea83EiRPRP5w619Q6hRBtRyWdgu7SDeu+O/BmT8FcPRfVO7LlN4pORxnThrfoa0dbtmxp1fucPFYJzs7v5Ozg3Pzmsw2YvNswYRHombei+sTaHemwOXXfQwDnGFa/jFl+P3r+Y6geBz9ZpzmtnmMQQnQsauAget+yGPbWYt15A2bLV3ZHEkFGikGITihkwHHo624HDNbdszFfltsdSRyuNhzrkWIQopNS/Y5B/+kO6Nod655szMfr7Y4kgoQUgxCdmIrui77+ToiMxlp0M2btW3ZHEoct8HfRlWIQopNTvSPR190BPx+I9eDdWK+9bHckYTMpBiEEKiwcfc3NcMIwzJP3Yz2/nA5ywqJoBSkGIQQAqktX9FWzUaeegXlxBWbZErn5XicVVPdKEkLYS7lccPk06NEb8/JTmJpd6EmzGm++JzoHOWIQQvhRSqHPuxR10SRY/w7Wwpvk/kqdjBSDEKJJOv1c1BXXwqZPse6ajfF+Y3ck4aft5oCkGIQQzdInn46++ibwfoN1+3WYrzbZHUm0AykGIcRBqeNO9F3r4NK+I4f/rLU7ktifkusYhBA2UP2OQc++G2L6YS2+DWv1v+yOJNqQFIMQ4pCoXpG++ysNGopZfh/WP5ZirNY9OUwENykGIcQhU926o6dmo9LOxqwswDx4N2bf93bHEgEm1zEIIQ6Lcrng/yZDn1jM00sx31b7niXdo5fd0USAyBGDEOKwKaXQZ56Hnnw9fLUZ6/ZZmIrP7Y4lAkSKQQjRairpFN8N+BrqseZdj3m/xO5InUcb3stKikEIcUTUsQno7PnQ92ise2/HenGF3IDP4aQYhBBHTPWKRM/KQQ0fiXl+OeahezDfy6R0+wj8dQwy+SyECAjVpStMnAn9jsEULMNUVfrOYOodaXc0cZjkiEEIETBKKfTZY9FTs2Hrf7FyZmLKP7Y7ljhMUgxCiIBTJ56Mnn0XdOmKdc8crKIXZd7BQaQYhBBtQvU7Bv3nBZA4FPO3BzH5CzDff2d3LHEIpBiEEG1GhYb75hl+ewnm3Tew7rgOU7XF7liiBVIMQog2pbRGn/t79Iyb4Fsv1m3XYta/a3cscRBSDEKIdqEGDfUNLUUfhZV3G1bBMnmm9JFowykbKQYhRLtRUTHo6+ehTjsT8/LTWPfMkSfDHanAX8YgxSCEaF8qpAv68mm+x4Z+/QXWzVdjyuRWGsFEikEIYQs9fCT6L7kQFYO15Hasvz2IqauzO5ZAikEIYSMV0xd9w12o9F9jil7EmncdZpuctWQ3KQYhhK1USAj6oj/6rpau/gbr1iysN1bKBXE2kmIQQgQFNWQ4+sa/Qv9fYJYtwVp8K2bnDrtjdUpSDEKIoKE8Ueism1G/vwI++QBr7jTMe8V2x+p0pBiEEEFFaY3O+I1vYjoyBuv+eViP5GJq99gdLci03VDbId12u6ysjKVLl2JZFunp6WRmZvp9v66ujry8PDZv3kxERARZWVlER0cDUFBQQFFREVprxo8fz5AhQw66zkWLFrFp0ybcbjfx8fFMmjQJt1vuDi5EZ6OOikPfcBfmpRWYl5/GfPof9KVTUIOT7Y7W4bV4xGBZFvn5+cyZM4fc3FzWrFlDRUWF3zJFRUWEhYWxePFixowZw/LlywGoqKiguLiYBQsWkJ2dTX5+PpZlHXSdp556KgsXLuSee+5h3759FBUVtcFmCyGcQLnd6N9egr7+TujaHWvRLVgPzceSuYf/UYG/wgiKZFAAAA0PSURBVK3FYigvLyc2NpaYmBjcbjepqamUlpb6LbN27VrS0tIASElJYcOGDRhjKC0tJTU1lZCQEKKjo4mNjaW8vPyg6xw6dChKKZRSDBgwgOrq6oBvtBDCWVT/X6D/shD164sw761h+4xLsEpWy5lLbaTFMRqv10tk5P+ewBQZGcnGjRubXcblchEaGkpNTQ1er5eEhITG5TweD16vt3E9B1tnfX09b775JuPGjWsyV2FhIYWFhQDMmzePqKioljalSW63u9XvDQZOzu/k7ODs/I7NPmEG9elj2HXfndTlLyCk7G16XHkdrj6xdic7ZIHa97VhYdQAkZ5IdESPIw+2n6AdvH/44Yc57rjjOO6445r8fkZGBhkZGY2vt2/f3qqfExUV1er3BgMn53dydnB2fidnJ6wnkTn38c3Tj7GvYBnbp12MOucC1JnnoUJC7E7XokDte2uPbzK+2luN+n5fq9bRt2/fJr/e4lCSx+PxG86prq7G4/E0u0xDQwO1tbVEREQc8F6v14vH42lxnU8//TS7du3i8ssvP8TNE0J0Jsrl8p25dMsSGDQU89wTWHOnYza8Z3e0DqHFYoiPj6eyspKqqirq6+spLi4mOdn/rICkpCRWr14NQElJCYmJiSilSE5Opri4mLq6OqqqqqisrGTAgAEHXeeqVatYv349WVlZaC1n0wohmqcio3FdNRt99VwArL/eTMO9t2Oqq+wN5nAtDiW5XC4mTJhATk4OlmUxatQo4uLiWLFiBfHx8SQnJzN69Gjy8vKYPn064eHhZGVlARAXF8eIESOYOXMmWmsmTpzY+Jd9U+sEeOihh+jTpw/Z2dkADB8+nLFjx7bV9gshOgA1aCh67mLMv5/DvPQU1o1TUBmZqF+dj+oeane8ttGGE+/KdJBp/S1bWnfjLUePteLs/E7ODs7O7+TscPD8pvobzLOPYd59AyJ6on59Eeq0s1BBcj1UwOYYCl/ArHgYvXA5KiyiVeto9RyDEEI4iYrsg/7jLHT2fDgqDvPkA1g3TcOsK+6gp7facB2DEEI4kfp5AnpWDnraX8DlwrpvHtYd12E2rOugBRE4wXFsJYQQbUApBScOQw8aillT6Jt/+OtciP8l+tcXw/FDfMsIP1IMQogOT7lcqNPPwqSOxqxZhXn5KayFN/kK4tyLIPEkKYj9SDEIIToN5Q5BjfwVJjXddwTx8tO+I4if/dx3gdyw04JmktpOMscghOh0VEgIOu1sdM4DqHEzoKEB80gu1pxJWCsLOv0tvqUahRCdlgoJQZ2SgUlNhw3rsFY+i/nHUsw//45KGYk6/Veoo/vbHbMZNj+PQQghOjKlFAxOwjU4CfNlOWbVi5jiIszrr8CxA1Ejz0Yln4rq2tXuqO1ChpKEEGI/6pgB6AlZ6Lsf9T1idG8t5tG/Yl03DuuxxZhPPsBYlt0x/6cN5szliEEIIZqgwsJRGb/BpP8aNn6IeevfmNK3MG/9G3pFok4+HTX8dIjr3+HOaJJiEEKIg1BKwcBBqIGDMJd8j/ngXUzJasyqFzCvFkBkNOrEk1FDhkNCYoc4q8n5WyCEEO1Ede2KGnYaDDsNU7MLU1aCWf8u5s1XMUUvQmgYKnEo/PIE1C9PgD6xjjyakGIQQohWUBE9UKedCaedifn+O/ioDFP2DubD96H0Td85Q5HRqF8O9h1J/HwgHNXP5tSHRopBCCGOkOraDU5KQZ2U4rsP09b/Yj5Z75uofv8dWLPKVxRdu+NNOA6r3zHQ7+eoo34GsT9Ddetu9yb4kWIQQogAUkrBUT/z/aU/aozvDKaqLZjPN8Lnn2EqPses+ifU1//vSgRPH4jth/L0gd5R4IlCeaKgRy8IDff907Vbuw1LSTEIIUQbUlr7jgpifwYjRhEZFcU3W7fCN1uh8itMZQVs+RpTtQXz3y9h5w6gicvXXC7oHgruEHC54fu9bZZZikEIIdqZcrvhqJ/5jix+8j1TXwc7qmHHdqjZhandDbW7Yc9u2FsLDfVQXwcNDdArErqHBTyfFIMQQgQR5Q6BPrG+f2iT69daJFc+CyGE8CPFIIQQwo8UgxBCCD9SDEIIIfxIMQghhPAjxSCEEMKPFIMQQgg/UgxCCCH8KGNM2z04VAghhON0+iOGG264we4IR8TJ+Z2cHZyd38nZwdn5nZC90xeDEEIIf1IMQggh/Ljmzp071+4Qduvfv7/dEY6Ik/M7OTs4O7+Ts4Oz8wd7dpl8FkII4UeGkoQQQviRYhBCCOGnUz+op6ysjKVLl2JZFunp6WRmZtod6QBTp06lW7duaK1xuVzMmzeP3bt3k5ubyzfffEOfPn245pprCA8PxxjD0qVLef/99+natStTpkxp97HMe++9l3Xr1tGzZ0/mz58P0Kq8q1ev5tlnnwXg/PPPJy0tzZbsTz31FKtWraJHjx4AXHzxxQwdOhSAgoICioqK0Fozfvx4hgwZAtj3udq+fTtLlizh22+/RSlFRkYG55xzjiP2f3PZnbL/9+3bx0033UR9fT0NDQ2kpKRw4YUXUlVVxcKFC6mpqaF///5Mnz4dt9tNXV0deXl5bN68mYiICLKysoiOjj7odrUr00k1NDSYadOmma1bt5q6ujoza9Ys8/XXX9sd6wBTpkwxO3fu9PvasmXLTEFBgTHGmIKCArNs2TJjjDHvvfeeycnJMZZlmU8//dTMnj273fN++OGHZtOmTWbmzJmtzltTU2OmTp1qampq/P7bjuwrVqwwzz///AHLfv3112bWrFlm3759Ztu2bWbatGmmoaHB1s+V1+s1mzZtMsYYU1tba2bMmGG+/vprR+z/5rI7Zf9blmX27t1rjDGmrq7OzJ4923z66adm/vz55q233jLGGPPAAw+YlStXGmOMeeWVV8wDDzxgjDHmrbfeMgsWLDjodrW3TjuUVF5eTmxsLDExMbjdblJTUyktLbU71iEpLS1l5MiRAIwcObIx99q1azn99NNRSjFw4ED27NnDjh072jXb8ccfT3h4+BHlLSsr44QTTiA8PJzw8HBOOOEEysrKbMnenNLSUlJTUwkJCSE6OprY2FjKy8tt/Vz17t278Tf+7t27069fP7xeryP2f3PZmxNs+18pRbdu3QBoaGigoaEBpRQffvghKSkpAKSlpfnt+x+PwlJSUtiwYQPGmGa3q7112qEkr9dLZGRk4+vIyEg2btxoY6Lm5eTkAHDGGWeQkZHBzp076d27NwC9evVi586dgG+boqKiGt8XGRmJ1+ttXNYuh5v3p/9vPB7PQf+SaGsrV67kjTfeoH///lx++eWEh4fj9XpJSEhoMmMwfK6qqqr4/PPPGTBggOP2//7ZP/nkE8fsf8uyuP7669m6dStnnXUWMTExhIaG4nK5Dsi4/z52uVyEhoZSU1Nz0O1qT522GJzi1ltvxePxsHPnTm677Tb69u3r932lFErZ8bjw1nFa3jPPPJOxY8cCsGLFCh5//HGmTJlic6qD++6775g/fz7jxo0jNDTU73vBvv9/mt1J+19rzd13382ePXu455572LJli92RWq3TDiV5PB6qq6sbX1dXV+PxeGxM1LQfM/Xs2ZNhw4ZRXl5Oz549G4eIduzY0Tgx5/F42L59e+N7g2WbDjfvT//feL1e27ajV69eaK3RWpOens6mTZuAAz8/P2a0+3NVX1/P/PnzOe200xg+fDjgnP3fVHan7X+AsLAwEhMT+eyzz6itraWhocEv40/zNzQ0UFtbS0RERNB89jttMcTHx1NZWUlVVRX19fUUFxeTnJxsdyw/3333HXv37m387w8++ICjjz6a5ORkXn/9dQBef/11hg0bBkBycjJvvPEGxhg+++wzQkNDbR9G+jHX4eQdMmQI69evZ/fu3ezevZv169fbc2YG+M3RvPvuu8TFxTVmLy4upq6ujqqqKiorKxkwYICtnytjDPfffz/9+vXj3HPPbfy6E/Z/c9mdsv937drFnj17AN8ZSh988AH9+vUjMTGRkpISwHem149ZkpKSWL16NQAlJSUkJiailGp2u9pbp77yed26dTz22GNYlsWoUaM4//zz7Y7kZ9u2bdxzzz2A77eKU089lfPPP5+amhpyc3PZvn37Aacf5ufns379erp06cKUKVOIj49v18wLFy7ko48+oqamhp49e3LhhRcybNiww85bVFREQUEB4DtdctSoUbZk//DDD/niiy9QStGnTx8mTZrUWLbPPvssr732Glprxo0bx0knnQTY97n65JNPuPHGGzn66KMbh4suvvhiEhISgn7/N5d9zZo1jtj/X375JUuWLMGyLIwxjBgxgrFjx7Jt2zYWLlzI7t27OfbYY5k+fTohISHs27ePvLw8Pv/8c8LDw8nKyiImJuag29WeOnUxCCGEOFCnHUoSQgjRNCkGIYQQfqQYhBBC+JFiEEII4UeKQQghhB8pBiGEEH6kGIQQQvj5f/eGqvJ09q1oAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"Yy8lkzpGwwdV"},"source":["As expected, the learning rate is updated in accordance with cosine annealing schedule."]},{"cell_type":"markdown","metadata":{"id":"t_4cZUXgwwdV"},"source":["The training process was terminated after _16 epochs_. Now we're going to restore the best weights saved during training, and apply the model to the validation subset of the data to see the final model's performance:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"b9WGkwZ7wwdV","executionInfo":{"status":"ok","timestamp":1633622853655,"user_tz":-330,"elapsed":116,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"90510a76-d643-4682-a7b3-aaf17de15494"},"source":["net.load_state_dict(best_weights)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{},"execution_count":41}]},{"cell_type":"code","metadata":{"id":"lI_DqmEIwwda"},"source":["# groud_truth, predictions = [], []\n","\n","# with torch.no_grad():\n","# for batch in batch_generator(*datasets['val'], shuffle=False, bs=bs):\n","# x_batch, y_batch = [b.to(device) for b in batch]\n","# outputs = net(x_batch[:, 1], x_batch[:, 0], minmax)\n","# groud_truth.extend(y_batch.tolist())\n","# predictions.extend(outputs.tolist())\n","\n","# groud_truth = np.asarray(groud_truth).ravel()\n","# predictions = np.asarray(predictions).ravel()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"YdXslUMBwwda"},"source":["# final_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n","# print(f'Final RMSE: {final_loss:.4f}')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"hB9t-ARGwwdb"},"source":["with open('best.weights', 'wb') as file:\n"," pickle.dump(best_weights, file)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"8Qcj-GoNwwdb"},"source":["### Embeddings Visualization\n","\n","Finally, we can create a couple of visualizations to show how various movies are encoded in embeddings space. Again, we're repeting the approach shown in the original post and apply the [Principal Components Analysis](http://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html) to reduce the dimentionality of embeddings and show some of them with bar plots."]},{"cell_type":"markdown","metadata":{"id":"OqwMsqWHwwdc"},"source":["Loading previously saved weights:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JKlP4S1zwwdc","executionInfo":{"status":"ok","timestamp":1633622853666,"user_tz":-330,"elapsed":93,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"ed020588-98cd-4d38-9ba8-4fee5dcd31b1"},"source":["with open('best.weights', 'rb') as file:\n"," best_weights = pickle.load(file)\n","net.load_state_dict(best_weights)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{},"execution_count":45}]},{"cell_type":"code","metadata":{"id":"omN9TxzFwwdd"},"source":["def to_numpy(tensor):\n"," return tensor.cpu().numpy()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"_8x4DFcbwwdd"},"source":["Creating the mappings between original users's and movies's IDs, and new contiguous values:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6TOmrBbbg9WA","executionInfo":{"status":"ok","timestamp":1633622965587,"user_tz":-330,"elapsed":682,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"6b34ad31-b1a9-4dbe-b12f-040dcb3a2c2b"},"source":["maps.keys()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["dict_keys(['ITEMID_TO_IDX', 'IDX_TO_ITEMID'])"]},"metadata":{},"execution_count":49}]},{"cell_type":"code","metadata":{"id":"qdHe7Nekwwdd","scrolled":true},"source":["user_id_map = umap['USERID_TO_IDX']\n","movie_id_map = imap['ITEMID_TO_IDX']\n","embed_to_original = imap['IDX_TO_ITEMID']\n","\n","popular_movies = ratings_df.groupby('ITEMID').ITEMID.count().sort_values(ascending=False).values[:1000]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"L7cJFbDXwwde"},"source":["Reducing the dimensionality of movie embeddings vectors:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"A0My7Epuwwde","executionInfo":{"status":"ok","timestamp":1633623038269,"user_tz":-330,"elapsed":20,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"44f167aa-656f-4fbc-ee20-898672fd5ee4"},"source":["embed = to_numpy(net.m.weight.data)\n","pca = PCA(n_components=5)\n","components = pca.fit(embed[popular_movies].T).components_\n","components.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(5, 1000)"]},"metadata":{},"execution_count":52}]},{"cell_type":"markdown","metadata":{"id":"MeHZJsE3wwdf"},"source":["Finally, creating a joined data frame with projected embeddings and movies they represent:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"g4_xKa2p7bIO","executionInfo":{"status":"ok","timestamp":1633623040449,"user_tz":-330,"elapsed":29,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"1f7539cf-8346-4f83-9a33-9d9abcd739e2"},"source":["movies = movies_df[['ITEMID','TITLE']].dropna()\n","movies.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(1682, 2)"]},"metadata":{},"execution_count":53}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"YkGwvIUqwwdf","executionInfo":{"status":"ok","timestamp":1633623040451,"user_tz":-330,"elapsed":27,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"c99192b1-7bd9-41da-efff-dd2e5a379e5a"},"source":["components_df = pd.DataFrame(components.T, columns=[f'fc{i}' for i in range(pca.n_components_)])\n","components_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
fc0fc1fc2fc3fc4
0-0.013240-0.0391030.061148-0.050387-0.020236
1-0.020302-0.040640-0.0052810.0136270.022263
2-0.046928-0.006252-0.027646-0.0124380.033915
30.068046-0.001613-0.0422350.006097-0.029498
4-0.056585-0.006064-0.015407-0.017673-0.018524
\n","
"],"text/plain":[" fc0 fc1 fc2 fc3 fc4\n","0 -0.013240 -0.039103 0.061148 -0.050387 -0.020236\n","1 -0.020302 -0.040640 -0.005281 0.013627 0.022263\n","2 -0.046928 -0.006252 -0.027646 -0.012438 0.033915\n","3 0.068046 -0.001613 -0.042235 0.006097 -0.029498\n","4 -0.056585 -0.006064 -0.015407 -0.017673 -0.018524"]},"metadata":{},"execution_count":54}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"chcYgtkw8ij6","executionInfo":{"status":"ok","timestamp":1633623042620,"user_tz":-330,"elapsed":26,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"4e996939-45d5-4031-ca55-5cef20b87a6a"},"source":["components_df.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(1000, 5)"]},"metadata":{},"execution_count":55}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":173},"id":"lTu7ZbLL8NaO","executionInfo":{"status":"ok","timestamp":1633623138456,"user_tz":-330,"elapsed":428,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"820a8b21-2a52-4c36-983b-cc0c9123efff"},"source":["movie_ids = [embed_to_original[idx] for idx in components_df.index]\n","meta = movies.set_index('ITEMID')\n","components_df['ITEMID'] = movie_ids\n","components_df['TITLE'] = meta.reindex(movie_ids).TITLE.values\n","components_df.sample(4)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
fc0fc1fc2fc3fc4ITEMIDTITLE
667-0.0117020.0078130.044108-0.036211-0.002407667Audrey Rose (1977)
703-0.0073030.0268040.0735950.0660340.041713703Widows' Peak (1994)
879-0.0255450.039012-0.001066-0.0167790.006389879Peacemaker, The (1997)
197-0.0386260.002753-0.045264-0.0338070.004753197Graduate, The (1967)
\n","
"],"text/plain":[" fc0 fc1 fc2 ... fc4 ITEMID TITLE\n","667 -0.011702 0.007813 0.044108 ... -0.002407 667 Audrey Rose (1977)\n","703 -0.007303 0.026804 0.073595 ... 0.041713 703 Widows' Peak (1994)\n","879 -0.025545 0.039012 -0.001066 ... 0.006389 879 Peacemaker, The (1997)\n","197 -0.038626 0.002753 -0.045264 ... 0.004753 197 Graduate, The (1967)\n","\n","[4 rows x 7 columns]"]},"metadata":{},"execution_count":60}]},{"cell_type":"code","metadata":{"id":"rW-l-0Izwwdg"},"source":["def plot_components(components, component, ascending=False):\n"," fig, ax = plt.subplots(figsize=(18, 12))\n"," \n"," subset = components.sort_values(by=component, ascending=ascending).iloc[:12]\n"," columns = components_df.columns\n"," features = columns[columns.str.startswith('fc')].tolist()\n"," \n"," fc = subset[features]\n"," labels = ['\\n'.join(wrap(t, width=10)) for t in subset.TITLE]\n"," \n"," fc.plot(ax=ax, kind='bar')\n"," y_ticks = [f'{t:2.2f}' for t in ax.get_yticks()]\n"," ax.set_xticklabels(labels, rotation=0, fontsize=14)\n"," ax.set_yticklabels(y_ticks, fontsize=14)\n"," ax.legend(loc='best', fontsize=14)\n"," \n"," plot_title = f\"Movies with {['highest', 'lowest'][ascending]} '{component}' component values\" \n"," ax.set_title(plot_title, fontsize=20)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":704},"id":"WpJkrTX9-Asp","executionInfo":{"status":"ok","timestamp":1633623144414,"user_tz":-330,"elapsed":1659,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"6724cabe-23d6-4ffe-8a21-1c3ba6cb3066"},"source":["plot_components(components_df, 'fc0', ascending=False)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABCUAAAMFCAYAAABDA0wqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde1yUZf7/8fcwI6ggMEqjKQieT5snQAwtPKBUnovS1S0NO9A3Navtt9JhZctSt93Nylr9WnksTVrLY2lSVmqaluX5UEqGmngAz2LA9fujL7NODIjD6Ki9no9Hj+K6r/u+P3PNfU/Mm/u+bosxxggAAAAAAOAy8/N1AQAAAAAA4PeJUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAwDVqyJAhslgsysrK8nUpFebJa1mxYoUsFovS09MrvP+srCxZLBYNGTKk3OtMmzZNFotF06ZNu+z7vtrMnj1bbdq0UbVq1WSxWDRy5EhflwR4ncViUadOnXxdBgBccQglAMADFotFFotFfn5++uGHH0rt17lzZ2ffin45havfw5f1K5GnYVdxSLNixQqX9i+//FKDBg3SiRMn9NBDD2n06NG65ZZbPKrtzJkzGj16tJo0aaLKlSvL4XDorrvu0rZt29z25/i59qSnp7s9zgAAVy5CCQDwkM1mkzFGb775ptvlu3bt0ooVK2Sz2S5zZb8aO3astm3bpjp16vhk/950Nb6Wfv36adu2berXr5+vS7miLV68WMYYzZgxQ3//+9+Vnp7uUSiRn5+vbt266dlnn1VwcLAeeeQRJSYm6v3331dMTIzWrl17CaoHAAAVRSgBAB6qWbOmYmJiNHXqVBUUFJRY/sYbb0iSevXqdblLkyRdf/31atq0qSpVquST/XvT1fhaQkJC1LRpU4WEhPi6lCva/v37JUm1a9eu0Hb+9a9/adWqVUpOTtbatWs1fvx4vfPOO3rvvfd0+vRppaSkqKioyBslAwAALyKUAIAKuP/++/Xzzz9r0aJFLu2//PKLpk2bpvj4eDVv3rzU9Xft2qV77rlHderUkb+/v2rXrq177rlHu3btcumXmpoqi8Wi+fPnu93O2rVrZbFYlJyc7Gwr6zL7tWvXKjk5WbVq1ZK/v78iIiL04IMPOr8gnm/37t164IEH1LBhQ1WpUkXVq1fXDTfcoNTUVB05cqSs4ZH065dNd1c4REZGymKx6LnnnnNp//DDD2WxWPTXv/611NeSnp6uevXqSZKmT5/uvEWmtNtkvv32W/Xo0UOhoaGqWrWqEhIStHr16gvW7k5WVpYGDBigsLAwVa5cWTExMSXef6nsOSWWLl2qDh06KDAwUNWrV1ffvn21ffv2C94aUd59F5s9e7Y6d+6s0NBQVa5cWc2aNdOYMWOUn59fou8XX3yhXr16KTw8XAEBAapVq5bat2+vv/3tb84+FotF06dPlyTVq1fPOeZRUVFlD5obxeMzderUEts7//VnZ2drxIgRatSokfP4a9eunctxY4zRpEmTJEl///vf5ef3319v+vTpo5tuuklbt27VZ599dtF1/tayZcvUq1cvORwOBQQEKCIiQn369NHy5ctd+hUVFWnSpEmKjY1VUFCQAgMDFRsbq3//+99uw5Hi+QYOHjyolJQU1axZU4GBgYqPj9cXX3whSTp16pSeeOIJRUZGKiAgQC1atFBGRkaJbZ1/7C1evFjx8fEKDAyU3W5XcnJyic+XYgcOHNDDDz+sqKgo+fv767rrrtPtt9+ur7/+usx9fPrpp+rUqZOqVaum4OBg9ejRo9RbZk6fPq2xY8eqdevWCgwMVFBQkG688UbNnj27RN/z54UpzzkcFRXlPF7Pv3XOYrG4raXYnDlzZLFY9Oijj7pdnp+fL7vdruuvv94ZQB87dkwvvviiunTpovDwcOd49e7dW19++WWZ+ztfWed8WfPiHD16VGlpaWrWrJmqVKmikJAQde3aVcuWLSvR99y5c3rllVfUtm1b2e12Va1aVVFRUW6PWwDwBUIJAKiAP/7xjwoMDHReFVFswYIFysnJ0f3331/quuvWrVNMTIxmzZql2NhY/fnPf1b79u01a9YsxcTEaN26dc6+gwcPliTNmDHD7baKvyiW5/74t956Sx06dNCHH36ozp07a+TIkYqJidEbb7yhmJgY7d2719n3wIEDio2N1dSpU9WiRQuNGDFCd999t+rVq6eZM2fqwIEDF9xfly5dtH//fm3fvt3Z9v333zv3k5mZ6dK/+OeuXbuWus1OnTrpkUcekSS1atVKo0ePdv7TunVrl77r169XfHy8zp49q/vuu089e/bUypUr1bVrV+3YseOC9Z/vxx9/VLt27ZSVlaW7775b/fv31+bNm9WnTx99+umn5drGnDlzdOutt2rDhg2688479eCDDyo3N1c33nhjmfM0XOy+U1JSNHDgQH3//fe644479PDDD6t69ep65plndMstt7hc3fPRRx+pU6dOznF5/PHH1bdvXwUEBOj111939hs9erRatWolSXrkkUecY+7JxJStW7cudXuhoaGSfn3vWrVqpVdffVW1a9fWiBEjNGjQIFWrVs3li9oPP/ygvXv3qnHjxs6w6ny33nqrJOmTTz656DrPN3r0aCUlJWnFihVKSkrS448/rq5du2rbtm2aNWuWS9+7775bDz30kA4ePKj77rtPDzzwgA4dOqT/+Z//0d133+12+3l5eerQoYM2bNigP/7xj7rjjju0fv16JSUl6bvvvlPXrl01f/589ezZU4MHD9bevXvVv39/rVmzxu325s2bp759+yo8PFyPPPKIbrzxRv3nP/9R+/btSxz7e/bsUUxMjF5//XU1aNBAjz/+uJKSkpyhRmnh16JFi9S9e3cFBwcrNTVVN910k5YsWaKEhAQdPny4xOvr2LGjnnzySVmtVqWkpGjw4ME6dOiQBg4cqKefftrtPsp7Do8cOVIJCQmSfv3MPP9zoSx9+/ZVSEiI3nnnHbdXvc2fP195eXkaNGiQ83a8bdu26amnnpKfn5969Oihxx57TN26ddMnn3yim2++WR999FGZ+6yIH3/8UdHR0Ro3bpyuu+46paamqn///tq2bZtuueUWTZkyxaX/kCFD9Mgjj+iXX37RPffcoxEjRujmm2/Wpk2bLmmdAFBuBgBw0SSZOnXqGGOMGTp0qLFareann35yLk9KSjLBwcHm1KlT5qmnnjKSzNSpU53Li4qKTNOmTY0kM2vWLJdtz5kzx0gyTZo0MYWFhc72xo0bG39/f3PkyBGX/mfPnjV2u904HA7zyy+/ONsHDx5sJJk9e/Y423bs2GEqVapkGjRoYLKzs122s3z5cuPn52f69u3rbHvllVeMJDNhwoQSY3Dy5Elz+vTpC47Vm2++aSSZiRMnOtsmTZpkJJlu3boZf39/c+rUKeey1q1bmypVqpj8/PwyX8uePXuMJDN48GC3+/3000+NpBJjf/7+H3rooQvWf/6+JJn09HSXZR999JGRZG699VaX9qlTp5bY9/Hjx01oaKjx9/c33377rUv/v/zlL859uHudnuy7X79+Jd6j0aNHl3hPb7/9diOpRE3GGHPo0CGXn929FxVR2vby8/NNVFSUkWTefvvtEuudf74tWrTISDI9e/Z0u4+MjAwjydx1110e17l06VIjydSrV6/EufPbet555x0jybRp08acOHHC2X7y5EkTHR3t9jUVv8cPPvigy3k/Y8YMI8nY7XbTs2dPc+bMGeeyzz//3EhyOWeN+e/7L8ksXLjQZdmECROMJNOlSxeX9u7duxtJZsyYMS7tq1atMlar1VSvXt3ltRTvw2q1muXLl7usM2rUKCPJjB8/3qW9+L3+bfuZM2dMUlKSsVgsZsOGDc52T87h4uP7008/NRfjgQcecDtexhhz2223GUlm48aNzra8vLwS54Yxvx4H119/vWnatGmJZZJMQkKCS1tZ51Px6x89erRLe0JCgrFYLGb27Nku7bm5uaZVq1amcuXK5ueff3bWabFYTHR0tCkoKCixj8OHD5doA4DLjVACADxwfiixZs0aI8n87W9/M8YYk5WVZfz8/Jy/LLsLJVauXGkkmRtvvNHt9jt27Ggkmc8++8zZ9vzzz5f4cm/Mf79wPfrooy7t7n7ZHTlypJFkFi1a5Ha/ffv2NVar1Rw/ftwY899QYvLkyeUYFfeysrKcX5CL3XnnnaZmzZpm4cKFRpJZunSpMebXX5AtFovp1q3bBV9LeUOJDh06lFh27tw5Y7PZTHR0dLleQ/G+IiMj3f5iX7duXVOjRg2XNnehxMyZM40kc++995bYxokTJ0xoaGipr/Ni9t26dWtjs9lMbm5uif4FBQWmRo0aJjY21tlWHErs2LGj1DEodrlCiffee89IMr17977gNt5++20jyQwaNMjt8mXLlhlJpnv37h7X2bNnTyPJzJs374J9ExMTXY7r8y1fvtxIMp07d3Zpl2SqVq3qPPeKFRQUGJvNZiSZH374ocT2oqKiTFRUlEtb8bH32+CheHsNGjQwkkxWVpYx5tcv0pJM3bp1zblz50qs86c//clIMtOnTy+xD3djvnv3biPJ3HHHHc62w4cPG6vVamJiYkr0N8aYb7/91kgyTzzxhLPNk3PY01Bi1apVRpJJTk52aT9w4ICxWq2mTZs25d7W8OHDjSTz448/urR7I5QoHqff1lnsgw8+MJLMa6+9Zowx5tixY0aSiY+PN0VFReV+DQBwOflmSngAuIbExcXphhtu0FtvvaWnn35ab7zxhoqKisq8deObb76R9OutDe506dJFK1eu1IYNG3TzzTdLku655x4988wzmj59uh5++GFn34u5daP4XufPPvvM5faQYjk5OSosLNTOnTsVHR2t3r1768knn9TDDz+spUuXKikpSR06dFDz5s0veJ92scjISNWvX18rVqxQUVGR83F9iYmJSkhIkM1mU2Zmprp3765PP/1UxphSx8UTMTExJdoqVaqkmjVrKjc396K21bp1a1mt1hLtERER5bqPfMOGDZKkjh07llgWFBSk1q1bl/oow/Lu+/Tp0/ruu+8UFhamCRMmuN1WQECAyz3/gwYN0rx58xQXF6f+/furc+fO6tChg8LDwy/4mi6V4lsSim+98LU1a9bIYrGU68kg33zzjfz8/NSpU6cSyxISEmS1Wp3HwvkaN26satWqubRZrVbVrFlTp06dUv369UusU6dOnVKfLFJ8K8Nvt9exY0f98MMP2rBhgyIjI5213HTTTW4nk+3SpYtmzZqlDRs26J577nFZ5u78ioiIkCSX82vdunUqLCwsdY6EX375RZLczkXhzXO4NPHx8WrcuLEWLlyo3Nxc2e12SdLbb7+twsJCt5+vq1at0ssvv6wvv/xSOTk5OnfunMvyffv2qW7dul6pr1jxuX7s2DG343jo0CFJ/x3H4OBg9erVSwsXLlTr1q11xx136KabblJcXJyqVq3q1doAwFOEEgDgBffff79GjBihDz/8UFOnTlV0dLTatGlTav9jx45J+vWpEu4Ut+fl5TnbwsPD1bVrV3388cfatm2bmjVrppycHH300Udq3bq1WrZsecE6iyemfPHFF8vsd/LkSUm/BgpfffWV0tPT9dFHH2nevHmSfv3S8ec//1kjRoy44D6lX+eHmDJlir755htVqlRJhw4dUteuXVWtWjXFxsY655Eoz3wSF6t4foLfstlsKiws9Nq2yvNkh+L3vWbNmm6Xl9Z+MfvOzc2VMUaHDh1ymaSyLLfffrsWLVqkf/7zn3rrrbc0efJkSVJ0dLTGjh2rbt26lWs73lR87JfnMbDFTzgpHt/fKm4vbQzLW4/dbleVKlUu2PfYsWOqXr26/P39Syyz2WwKCwtTTk5OiWWlPanFZrOVuczdPAhS6cdTrVq1nHWe/++L+Twq5m5Mi+ddOP/8Kv7sWbdundtAtFjxZ8+F9lG8n4s9h8syePBgPfXUU5ozZ44eeughSb+GvpUqVdLAgQNd+r7//vtKTk5W5cqV1a1bNzVo0ECBgYHy8/PTihUr9Nlnn7mdULaiisfx448/1scff1xqv/PH8d1333U+jaZ4fo3KlSsrOTlZ//jHP8r83AGAy4GJLgHAC+6++25VqVJFqamp2rdvnx544IEy+xd/wfj555/dLi+eQPK3X0SKJ7wsvjri7bffVkFBgbP9Qs7/8mZ+vYXP7T/n/4W1WbNmevfdd3XkyBGtX79e48aNU1FRkR555BG9+eab5dpv8ZUPy5cvLxE8dOnSRRs2bNDRo0eVmZmpkJAQtW3btlzbvdoEBwdLkg4ePOh2eWntF6P4PW7Tpk2Z77ExxmW9Hj166JNPPlFubq4yMzP16KOPasuWLerZs6e2bt1a4bouVvEX0X379l2wb5MmTSRJO3fudLu8+GkTjRs3rlA9ubm5OnPmzAX7hoSE6OjRo86//p+voKBAhw8fdh4Ll1Jpx1Px507xseLp59HFKF730UcfLfOYLO+EsZfC3XffLT8/P+fn64YNG7Rp0ybddtttCgsLc+n7zDPPyN/fX+vXr9cHH3ygf/7zn3r22WeVnp7uPB7Lo/hJMe6CJXchUPE4vvzyy2WOY/FTbSSpSpUqSk9P186dO7V3717NmjVLHTt21KxZs1ye2AQAvkIoAQBeEBoaquTkZGVnZyswMFB//OMfy+xffBVFaZfqF/9i/tsv57fffruCg4M1a9YsFRUVafr06bLZbCX+ilea9u3bS5LzEYMXw2azKTo6Wn/5y1+cj+/74IMPyrVuly5dZLFYlJmZqU8++UT169d3Pkaya9euKioq0owZM7Rr1y516tTJ7W0Kv1Xcx5t/Kb3Uit/3lStXllh28uRJffvttxXeR1BQkFq0aKEtW7bo6NGjF71+YGCgunTpon/961968sknde7cOX344YfO5Zdr3IuP1fP3XZoGDRqobt262rlzp/bs2VNiefE2KnJbUPv27WWMKdfTCtq0aaOioiJ9/vnnJZZ9/vnnKiwsvCzBm7tHoBYWFjqPv+Lj8fzj0t2X49I+jy5Gu3bt5Ofn59Fnz8WoyPEZERGhLl26aO3atdqxY4cznHAX+n7//fdq3ry5mjVr5tJeVFTk9vwuTfFtIj/99FOJZevXry/RVpHPcOnX1zho0CAtXbpUDRs21MqVK8v1aGcAuJQIJQDAS8aMGaP3339fS5cuLXFf+G916NBBTZo00cqVK/Xee++5LHvvvff0xRdfqHHjxiXmHqhSpYruuusu7du3Ty+99JK+++473XbbbXI4HOWqcdiwYapUqZIeffRRt39VPnfunMsvu19//bXbS+KL/wJb3nuSHQ6HWrRooVWrVunzzz93uT0jPj5elStX1tixYyWV/4uj3W6XxWJxeYTpla5Pnz4KCQnR22+/re+++85l2ZgxY9z+ZdQTjz32mM6dO6eUlBS328zNzXXOayL9+kXZ3ZdRd+9zjRo1JOmSj3uvXr0UFRWlBQsWOEOw82VnZzv/22KxKDU1VZL0//7f/3O5nWX+/Pn64osv1Lx5c7dzLJTX8OHDJUmPP/6426s3zm9LSUmRJKWlpen06dPO9tOnT2vUqFGSpKFDh3pcS3l98sknJR7lOXHiRP3www/q3LmzIiMjJf16a1i3bt2UlZVVYh6StWvX6p133pHdble/fv08rsXhcGjQoEFav369nnvuObehwQ8//OA2VLoYFT0+i+eOePPNNzV79myFhYWpZ8+eJfpFRUVp165d2r9/v7PNGKP09PSLurKoXbt2klTiMZ6bNm3Syy+/XKJ/TEyMbrrpJs2bN09vvfWW221u2rTJeXvQoUOHtGnTphJ9Tp06pZMnT8pms7m9zQgALifmlAAAL6lbt265JzWzWCyaPn26unXrpv79+6tPnz5q2rSpduzYoQ8++EDVqlXTjBkznJf2nm/w4MF64403lJaW5vy5vJo2baq33npLKSkpatGihW655RY1btxYv/zyi/bu3asvvvhC1113nbZv3y5JmjlzpiZPnqyOHTuqQYMGstvt+uGHH7Rw4UIFBARo5MiR5d53165dtXnzZud/FwsICFCHDh0uej6JoKAgxcXF6YsvvtCgQYPUuHFjWa1W9e7du1zza/hCcHCwXnvtNd19992Kj4/XXXfdpeuvv16rV6/Wd999p4SEBH322Wdu3/eLkZKSoq+//lqvv/66GjRooKSkJNWtW1dHjx7Vnj179Pnnn+vee+/VpEmTJEkjRozQvn371KFDB0VFRcnf319ff/21PvnkE0VGRmrAgAHObXft2lUvvvii7r//ft1xxx2qVq2aQkNDNWzYsArV/Fv+/v7KyMhQ9+7dNXDgQE2ePFnt27fX2bNntW3bNmVmZroEKY899pgWLVqk9957T3Fxceratav27t2rjIwMVa1aVW+99VaFxrV79+56+umnNWbMGDVr1kx9+/ZVRESEDh48qJUrV6p9+/aaNm2aJGngwIGaP3++5s6dqxYtWqhv376yWCz64IMPtGfPHvXv31+DBg2q6BBdUK9evdSvXz/169dPDRs21LfffqsPP/xQ1atX1+uvv+7Sd9KkSerQoYOeeOIJLVu2TDExMfrpp5+UkZEhPz8/TZ069YJh64VMnDhRu3bt0l//+lfNnDlTHTt2VM2aNbV//35t27ZN69at0+zZs1WvXj2P99G5c2f5+fkpLS1Nmzdvdl6J8PTTT5dr/X79+ik4OFgTJkzQL7/8ouHDh7ud/PPRRx9Vamqq2rRpozvuuEOVKlXSqlWrtHXrVufEkuXRp08fNWrUSLNnz1Z2drbi4uK0d+9ezZ8/X3369NHcuXNLrPPOO++oS5cuGjp0qF555RXFxcUpNDRU2dnZ2rhxozZv3qwvv/xSDodD+/btU5s2bXTDDTeoZcuWioiI0PHjx7Vo0SL9/PPPGjFiRIXfVwCosEv+fA8AuAbpvEeCXoi7R4IW2759u/nTn/5katWqZWw2m6lVq5YZNGiQ2b59e5nbbNiwoZFkqlevbvLz8932KetRcxs3bjSDBw82devWNf7+/sZut5sWLVqYBx54wGRmZjr7rVmzxqSmppqWLVsau91uKleubBo0aGCGDBliNm3aVK7XX2zBggVGkrFYLObgwYMuy1544QUjydSsWfOiXsuuXbtMz549TfXq1Y3FYnEZZ3eP0ztfZGSkiYyMLFftF3r8aEJCgvnt/1LdPRK02JIlS8yNN95oqlSpYkJDQ03v3r3Ntm3bTI8ePYwkl0d5erLvYgsXLjQ9evQw1113nalUqZKpWbOmiY2NNU899ZTZtm2bs9+7775rBgwYYBo2bGgCAwNNtWrVTIsWLcyTTz5pcnJySmz3n//8p2natKnx9/d3Pq7UUxd6xOiPP/5oHnroIRMVFWUqVapkqlevbtq1a2eef/75En1PnTplnnnmGdOwYUPj7+9vwsLCTHJystmyZYvH9f3W4sWLTVJSkrHb7cbf39+Eh4ebvn37upw3xhhTWFhoXnvtNRMdHW2qVKliqlSpYtq2bWsmTpxoCgsLS2xXbh4XWaysY/VCx97ChQtN+/btTdWqVU1ISIi5/fbbS330a3Z2tklNTTV169Y1lSpVMjVq1DB9+vQxX331VYm+ZR3fZb2e/Px88+qrr5obb7zRBAcHG39/fxMREWG6dOliXnrpJXP48GFnX0/P4ZkzZ5pWrVqZypUrG0mlnh+lGTp0qHO99evXl9pv6tSpplWrVqZq1aqmRo0apm/fvmbjxo2lPpa0tDHZu3evueuuu5yfsTExMeY///lPma//+PHj5vnnnzdt27Y1gYGBpnLlyiYqKsrcdtttZvLkyebkyZPGGGNyc3PN3/72N9O5c2dTu3Zt4+/vb2rVqmUSEhLMO++8w2NCAVwRLMb8ZqYrAABw2RUWFqp+/fo6d+6cc2JBwBPTpk3Tvffeq6lTp5brUcEAAPgSc0oAAHAZ5eXlucwzIP16L/qYMWO0d+/eCt23DwAAcLVhTgkAAC6jNWvWqH///urevbuioqJ08uRJrVmzRt9++60iIiKUnp7u6xIBAAAuG0IJAAAuoyZNmqhnz55atWqVlixZooKCAoWHh2vEiBF68skny/0kFQAAgGsBc0oAAAAAAACfYE4JAAAAAADgE4QSAAAAAADAJ66pOSX279/v6xIuKCwsTIcPH/Z1GdcMxtN7GEvvYjy9i/H0HsbSuxhP72I8vYex9C7G07sYT++6Gsazdu3apS7jSgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+cU09fQMAAAAAgEvll19+0blz5yRJFovFx9X86uDBg8rPz/fZ/o0x8vPzU+XKlT0aE0IJAAAAAAAu4OzZs5KkqlWrXjGBhCTZbDZZrVaf1lBQUKCzZ8+qSpUqF70ut28AAAAAAHABhYWFHl8NcK2z2WwqKiryaF1CCQAAAAAALoAwomyejg+hBAAAAAAA8IkKzSmxdOlSLViwQHl5eQoPD9eQIUPUrFmzUvtv3bpV06dPV3Z2tux2u3r37q3u3bs7lxcVFWnu3Ln64osvlJeXp9DQUN1000268847fX6PDAAAAAAA8C6PQ4nVq1dr2rRpGjp0qJo2baply5bphRde0EsvvaSwsLAS/XNycjR27Fh17txZw4cP1/bt2/Xmm28qODhY7du3lyR98MEHWrp0qR5++GHVrVtXe/fu1WuvvSabzabk5GTPXyUAAAAAAJdA4f29L+v+rFMWXPQ6RUVFGjVqlBYvXqy8vDxlZGQoPj7+ElR38Ty+fWPRokVKSEhQYmKiwsPDlZKSIrvdrmXLlrntv2zZMtntdqWkpCg8PFyJiYlKSEjQwoULnX127typ6OhoxcTEyOFwKCYmRtHR0fr+++89LRMAAAAAgN+1zMxMzZ07V9OmTdOGDRsUExNTZv+8vDwNHz5cTZs2VdOmTTV8+HAdO3bsktTmUShRUFCg3bt3q1WrVi7tLVu21I4dO9yus2vXLrVs2dKlrVWrVtq9e7cKCgokSU2bNtWWLWUpeXoAACAASURBVFu0b98+SVJ2dra2bNmiNm3aeFImAAAAAAC/e1lZWXI4HIqNjZXD4ZC/v3+Z/YcNG6bNmzdr1qxZmjVrljZv3qwRI0Zckto8un3j+PHjKioqUkhIiEt7aGioNm3a5HadvLw83XDDDS5tISEhKiws1IkTJ2S329WnTx+dOXNGjz32mPz8/FRYWKjbb79dSUlJbre5fPlyLV++XJI0btw4t7eNXGlsNttVUefVgvH0HsbSuxhP72I8vYex9C7G07sYT+9hLL2L8fSuq3U8Dx48KJut5Ffowstch7sa3LUVGzFihN59911JUp06dRQREaF169Zp0qRJmj59uvbt26caNWooOTlZTz/9tHbu3KlPP/1UCxcuVLt27SRJ//jHP9S7d29lZWWpYcOGbvcTEBDg0ftaoYkuvW316tX6/PPPNWLECEVERCgrK0tTp06Vw+FQly5dSvRPTExUYmKi8+fDhw9fznI9EhYWdlXUebVgPL2HsfQuxtO7GE/vYSy9i/H0LsbTexhL72I8vetqHc/8/Pwr4gEMxXcaFLPZbCXazpeenq7atWtrzpw5WrJkiaxWq8aMGaMZM2Zo9OjRiouL05EjR7R582YVFBToq6++UmBgoNq0aePcbtu2bVW1alWtXbtWUVFRbveTn59f6vtau3btUuvzKJQIDg6Wn59fiXtKip+Y4U5oaKjy8vJc2o4dOyar1apq1apJkmbNmqVevXqpQ4cOkqS6devq0KFDev/9992GEgAAAAAAoHTBwcEKCgqS1WqVw+HQqVOnNGXKFKWnp2vAgAGSpHr16jnnmcjJyVGNGjVksVic27BYLAoLC1NOTo7X6/NoTgmbzab69etr48aNLu2bNm1SkyZN3K7TqFGjErd2bNy4UfXr13deapKfny8/P9eS/Pz8ZIzxpEwAAAAAAHCenTt3Kj8/Xx07dvR1KZIq8PSNnj17asWKFcrMzFR2dramTp2qo0ePqlu3bpKkiRMnauLEic7+3bt319GjRzVt2jRlZ2crMzNTK1asUK9evZx9oqOj9cEHH+ibb75RTk6OvvrqKy1atMh5HwsAAAAAALh0HA6Hjhw54nJxgDFGhw8flsPh8Pr+PJ5TIj4+XidOnNC8efOUm5uriIgIpaWl6brrrpNUcn4Hh8OhtLQ0TZ8+3fl40HvvvVft27d39klJSdG7776rN954Q8eOHZPdblfXrl2VnJzsaZkAAAAAAOD/NGrUSAEBAVq5cqXq169fYnl0dLROnTql9evXKzY2VpK0fv16nT59WtHR0V6vp0ITXSYlJZX6ZIz09PQSbc2bN9f48eNL3V6VKlU0ZMgQDRkypCJlAQAAAAAAN4KCgjR06FCNGzdOAQEBiouLU25urjZu3KjBgwerUaNG6ty5s0aNGuX8/j5q1CglJiaW+uSNiriinr4BAAAAAMDVxDplga9LuGhpaWkKCQnRhAkTdODAAYWFhbncoTBx4kQ988wzGjRokKRfp2MYM2bMJanFYq6hWST379/v6xIu6Gp9/M2VivH0HsbSuxhP72I8vYex9C7G07sYT+9hLL2L8fSuq3U8T58+rapVq/q6jBIu9EjQy6Ws8SnrkaAeT3QJAAAAAABQEYQSAAAAAADAJwglAAAAAACATxBKAAAAAAAAn+DpG2UovL93mcuvxllWfYnx9J4LjaXEeAIAAAC48nGlBAAAAAAA8AmulADwu8dVPN7FeHoPV0V5F8emdzGe3sO57l2MJ3B14UoJAAAAAADgE4QSAAAAAADAJ7h9AwAAAAAAD/V5e/tl3d/8QU0vep2ioiKNGjVKixcvVl5enjIyMhQfH38Jqrt4hBIAAAAAAFzDMjMzNXfuXGVkZCgyMlKhoaFl9n/55Zf1ySefaMuWLTpz5oz27dt3yWrj9g0AAAAAAK5hWVlZcjgcio2NlcPhkL+/f5n9z507p1tvvVX33XffJa+NKyUAAAAAALhGjRw5UhkZGZKkOnXqKDw8XGvWrNHkyZM1c+ZM7d+/X9WrV1dycrLS0tIkSU888YQkadGiRZe8PkIJAAAAAACuUc8++6zCw8M1Z84cLVmyRFarVePGjdOMGTM0evRoxcXF6ciRI9q8ebNP6iOUAAAAAADgGhUcHKygoCBZrVY5HA6dOnVKU6ZMUXp6ugYMGCBJqlevnmJiYnxSH3NKAAAAAADwO7Fz507l5+erY8eOvi5FEqEEAAAAAADwEUIJAAAAAAB+Jxo1aqSAgACtXLnS16VIYk4JAAAAAAB+N4KCgjR06FCNGzdOAQEBiouLU25urjZu3KjBgwdLkvbt26fc3FxlZ2dLknMSzHr16ikwMNCr9RBKAAAAAADcKry/d5nLrVMWXKZKrlzzBzX1dQkXLS0tTSEhIZowYYIOHDigsLAwJScnO5e/+OKLzseISlJSUpIkKSMjQ/Hx8V6thVACAAAAAIBrWGpqqlJTU50/+/n5adiwYRo2bJjb/hMmTNCECRMuS23MKQEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8wubrAgAAAAAAuFotfDfvsu6vV//Qi16nqKhIo0aN0uLFi5WXl6eMjAzFx8dfguouHqEEAAAAAADXsMzMTM2dO1cZGRmKjIxUaGjpwcZPP/2kCRMmaPXq1crJyZHD4VDv3r01cuRIValSxeu1EUoAAAAAAHANy8rKksPhUGxs7AX7fv/99yosLNTYsWNVr1497dq1S3/5y1+Um5urv//9716vjTklAAAAAAC4Ro0cOVLp6enat2+f6tSpo7i4OBljNGnSJHXo0EH16tVTdHS0xo4dK0nq3LmzJkyYoE6dOikyMlKJiYkaPny4Fi9efEnq40oJAAAAAACuUc8++6zCw8M1Z84cLVmyRFarVePGjdOMGTM0evRoxcXF6ciRI9q8eXOp2zh58mSZt3xUBKEEAAAAAADXqODgYAUFBclqtcrhcOjUqVOaMmWK0tPTNWDAAElSvXr1FBMT43b97OxsTZo0ScOHD78k9XH7BgAAAAAAvxM7d+5Ufn6+OnbseMG+hw4d0qBBg3TzzTfrgQceuCT1EEoAAAAAAAAXOTk5uvPOO9WkSRO98sorslgsl2Q/hBIAAAAAAPxONGrUSAEBAVq5cmWpfQ4ePKjk5GQ1atRIr7/+umy2SzfzA3NKAAAAAADwOxEUFKShQ4dq3LhxCggIUFxcnHJzc7Vx40YNHjxYP//8s5KTk1WrVi2lp6fr6NGjznVr1Kghq9Xq1XoIJQAAAAAA8FCv/uV/KoXJ2lXmcktUowpWUz5paWkKCQnRhAkTdODAAYWFhSk5OVmS9Nlnn2nPnj3as2eP2rVr57LemjVrFBER4dVaCCUAAAAAALiGpaamKjU11fmzn5+fhg0bpmHDhpXo279/f/Xv3/+y1cacEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnbL4uAAAAAACAq9Urr7zixa19eMEeI0aMuOitFhUVadSoUVq8eLHy8vKUkZGh+Ph4Twr0OkIJAAAAAACuYZmZmZo7d64yMjIUGRmp0NDQUvsWFRUpJSVFW7Zs0ZEjRxQSEqKOHTvqySef1PXXX+/12rh9AwAAAACAa1hWVpYcDodiY2PlcDjk7+9fZv8OHTpo0qRJ+vzzz/W///u/+vHHH3Xfffddktq4UgIAAAAAgGvUyJEjlZGRIUmqU6eOwsPDtWbNGk2ePFkzZ87U/v37Vb16dSUnJystLU1+fn66//77neuHh4dr2LBhuvfee3X27FlVrlzZq/URSgAAAAAAcI169tlnFR4erjlz5mjJkiWyWq0aN26cZsyYodGjRysuLk5HjhzR5s2b3a6fm5urefPmqU2bNl4PJCRCCQAAAAAArlnBwcEKCgqS1WqVw+HQqVOnNGXKFKWnp2vAgAGSpHr16ikmJsZlveeff15Tp07VmTNn1LZtW82YMeOS1MecEgAAAAAA/E7s3LlT+fn56tixY5n9HnroIS1dulSzZ8+W1WrV8OHDZYzxej1cKQEAAAAAAFxUr15d1atXV4MGDdSwYUPFxsbqq6++UlxcnFf3w5USAAAAAAD8TjRq1EgBAQFauXJludcpvkIiPz/f6/VwpQQAAAAAAL8TQUFBGjp0qMaNG6eAgADFxcUpNzdXGzdu1ODBg7V+/Xpt3rxZsbGxCgkJUVZWll588UVFRESoXbt2Xq+HUAIAAAAAAA+NGDGi3H1N1q4yl1uiGlWwmvJJS0tTSEiIJkyYoAMHDigsLEzJycmSpMqVK2vRokV68cUXdebMGTkcDnXq1En//ve/efoGAAAAAAC4OKmpqUpNTXX+7Ofnp2HDhmnYsGEl+v7hD3/Qe++9d9lqq1AosXTpUi1YsEB5eXkKDw/XkCFD1KxZs1L7b926VdOnT1d2drbsdrt69+6t7t27u/TJzc3V22+/rQ0bNujs2bNyOBy6//771bx584qUCgAAAAAArjAehxKrV6/WtGnTNHToUDVt2lTLli3TCy+8oJdeeklhYWEl+ufk5Gjs2LHq3Lmzhg8fru3bt+vNN99UcHCw2rdvL0k6deqUnnnmGTVt2lRpaWkKDg7WwYMHFRwc7PkrBAAAAAAAVySPQ4lFixYpISFBiYmJkqSUlBR9++23WrZsmQYOHFii/7Jly2S325WSkiJJCg8P1/fff6+FCxc6Q4n58+fLbre7XELicDg8LREAAAAAAFzBPAolCgoKtHv3bvXq1culvWXLltqxY4fbdXbt2qWWLVu6tLVq1UqfffaZCgoKZLPZtG7dOrVu3VovvfSStmzZIrvdrq5duyopKUkWi8WTUgEAAAAAwBXKo1Di+PHjKioqUkhIiEt7aGioNm3a5HadvLw83XDDDS5tISEhKiws1IkTJ2S325WTk6Nly5apR48e6tu3r7KysvTWW29Jkm655ZYS21y+fLmWL18uSRo3bpzb20Yq4uAFlnuyP5vN5vU6rxaMp/dcaCylix/P3+tYShyb3sZ4eg/nundxbHoX4+k9nOvexXh6F+f6fx08eFA2W8WeFfHLBZZ7uv2K1uUNAQEBnh0Pl6AWjxUVFalBgwbO2z/q1aunAwcOaOnSpW5DicTEROftI5J0+PDhy1arp/sLCwu77HVeLRhP77rYcWEsS8ex6V2Mp3dxrnsPx6Z3MZ7exbnuXYyn9/yezvX8/HxZrdZLuo+CgoKLXsdms3m0nrfl5+eX+r7Wrl271PX8PNlZcHCw/Pz8dOzYMZf2vLw8hYaGul0nNDRUeXl5Lm3Hjh2T1WpVtWrVJEl2u13h4eEufcLDw6/KAxYAAAAAAJTNo1DCZrOpfv362rhxo0v7pk2b1KRJE7frNGrUqMStHRs3blT9+vWdl5o0adJE+/fvd+mzf//+q/LSHgAAAAAAUDaPQglJ6tmzp1asWKHMzExlZ2dr6tSpOnr0qLp16yZJmjhxoiZOnOjs3717dx09elTTpk1Tdna2MjMztWLFCpfJMnv06KFdu3Zp3rx5+vnnn/Xll1/qww8/VFJSUgVeIgAAAAAAuBJ5PKdEfHy8Tpw4oXnz5ik3N1cRERFKS0vTddddJ6nkvUUOh0NpaWmaPn268/Gg9957r/NxoJLUsGFDPfHEE5o9e7b+85//KCwsTP379yeUAAAAAABckRzfp3lvY99fuEtOw7EXvdmioiKNGjVKixcvVl5enjIyMhQfH+9Bgd5XoYkuk5KSSg0M0tPTS7Q1b95c48ePL3Obbdu2Vdu2bStSFgAAAAAA+D+ZmZmaO3euMjIyFBkZWepckL919uxZ9ezZU9u2bdOSJUvUqlUrr9fm8e0bAAAAAADgypeVlSWHw6HY2Fg5HA75+/uXa73nnntO119//SWtjVACAAAAAIBr1MiRI5Wenq59+/apTp06iouLkzFGkyZNUocOHVSvXj1FR0dr7FjX20KWLl2q1atX669//eslra9Ct28AAAAAAIAr17PPPqvw8HDNmTNHS5YskdVq1bhx4zRjxgyNHj1acXFxOnLkiDZv3uxcZ//+/UpLS9PMmTNVuXLlS1ofoQQAAAAAANeo4OBgBQUFyWq1yuFw6NSpU5oyZYrS09M1YMAASVK9evUUExMjSSosLNTw4cP1wAMPqEWLFvrpp58uaX3cvgEAAAAAwO/Ezp07lZ+fr44dO7pd/sorr6hSpUp68MEHL0s9XCkBAAAAAAAkSatWrdLatWsVGRnp0t6rVy/17t1bEydO9Or+CCUAAAAAAPidaNSokQICArRy5UrVr1+/xPJ//etfOn36tPPngwcPauDAgXr11VcVGxvr9XoIJQAAAAAA+J0ICgrS0KFDNW7cOAUEBCguLk65ubnauHGjBg8erLp167r0DwwMlCRFRUWpdu3aXq+HUAIAAAAAAA/lNBx74U7/x2TtKnO5JapRBaspn7S0NIWEhGjChAk6cOCAwsLClJycfFn2/VuEEgAAAAAAXMNSU1OVmprq/NnPz0/Dhg3TsGHDLrhuRESE9u3bd8lq4+kbAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAADABRhjfF3CFc3T8SGUAAAAAACgHAgm3KvIuBBKAAAAAABwAZUrV9apU6cIJtzIz8+Xv7+/R+vavFwLAAAAAADXHKvVqipVquj06dOSJIvFctHbKPphZ5nL/Rx1LnqbAQEBys/Pv+j1vMUYI6vVqkqVKnm0PqEEAAAAAADlYLVaFRgY6PH6hXMml739rj0uepthYWE6fPiwpyX5HLdvAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCUIJAAAAAADgE4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCVtFVl66dKkWLFigvLw8hYeHa8iQIWrWrFmp/bdu3arp06crOztbdrtdvXv3Vvfu3d32ff/99zV79mwlJSVp6NChFSkTAAAAAABcgTy+UmL16tWaNm2a+vXrp/Hjx6tJkyZ64YUXdPjwYbf9c3JyNHbsWDVp0kTjx49X3759NXXqVK1Zs6ZE3507d2r58uWKjIz0tDwAAAAAAHCF8ziUWLRokRISEpSYmKjw8HClpKTIbrdr2bJlbvsvW7ZMdrtdKSkpCg8PV2JiohISErRw4UKXfqdPn9arr76qhx56SIGBgZ6WBwAAAAAArnAehRIFBQXavXu3WrVq5dLesmVL7dixw+06u3btUsuWLV3aWrVqpd27d6ugoMDZNnnyZMXFxekPf/iDJ6UBAAAAAICrhEdzShw/flxFRUUKCQlxaQ8NDdWmTZvcrpOXl6cbbrjBpS0kJESFhYU6ceKE7Ha7li9frp9//lnDhw8vVx3Lly/X8uXLJUnjxo1TWFiYB6+mdAcvsNyT/dlsNq/XebVgPL3nQmMpXfx4/l7HUuLY9DbG03s4172LY9O7GE/v4Vz3LsbTuzjXvYvxLKlCE1160/79+zV79mw999xzstnKV1ZiYqISExOdP5c2n8Wl4sn+wsLCLnudVwvG07sudlwYy9JxbHoX4+ldnOvew7HpXYynd3Guexfj6T2c6951rY5n7dq1S13mUSgRHBwsPz8/HTt2zKU9Ly9PoaGhbtcJDQ1VXl6eS9uxY8dktVpVrVo1fffddzpx4oQee+wx5/KioiJt27ZNH3/8sWbOnKlKlSp5Ui4AAAAAALgCeRRK2Gw21a9fXxs3btSNN97obN+0aZPi4uLcrtOoUSOtW7fOpW3jxo2qX7++bDabYmNj9Y9//MNl+b///W/VqlVL/fr1K/fVEwAAAAAA4Org8Tf9nj176tVXX1XDhg3VpEkTffzxxzp69Ki6desmSZo4caIkadiwYZKk7t27a+nSpZo2bZoSExO1Y8cOrVixQo888ogkKTAwsMTTNgICAhQUFKS6det6WiYAAAAAALhCeRxKxMfH68SJE5o3b55yc3MVERGhtLQ0XXfddZJK3gvjcDiUlpam6dOnOx8Peu+996p9+/YVewUAAAAAAOCqVKF7IpKSkpSUlOR2WXp6eom25s2ba/z48eXevrttAAAAAACAa4OfrwsAAAAAAAC/T4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCUIJAAAAAADgE4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCUIJAAAAAADgE4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJW0VWXrp0qRYsWKC8vDyFh4dryJAhatasWan9t27dqunTpys7O1t2u129e/dW9+7dncvff/99ffXVV9q/f79sNpsaNWqkgQMHqm7duhUpEwAAAAAAXIE8vlJi9erVmjZtmvr166fx48erSZMmeuGFF3T48GG3/XNycjR27Fg1adJE48ePV9++fTV16lStWbPG2Wfr1q3q3r27nnvuOY0ePVpWq1XPPfecTp486WmZAAAAAADgCuVxKLFo0SIlJCQoMTFR4eHhSklJkd1u17Jly9z2X7Zsmex2u1JSUhQeHq7ExEQlJCRo4cKFzj5PPfWUOnfurLp166pu3boaPny4jh8/ru3bt3taJgAAAAAAuEJ5FEoUFBRo9+7datWqlUt7y5YttWPHDrfr7Nq1Sy1btnRpa9WqlXbv3q2CggK365w5c0bGGAUFBXlSJgAAAAAAuIJ5NKfE8ePHVVRUpJCQEJf20NBQbdq0ye06eXl5/5+9O4+3qq73x/9ikIAUDsrsAf0igwJCGfo9maAZDg/MIW4dHMAEynKMvDbw8GsaV0O8XTUTLczQmxIOOaRYGSQ30Yt+r6UIKmg5fOEGhHqOoiAy/P7w576eGA7nsHGBPp+PBw9da6+19me99xpf+7PXyf77719nXNu2bbNu3bq88cYbadeu3UbzTJ06NXvvvXd69+69yWXOnDkzM2fOTJJcdtllad++fWNWZ7OW1fN6Y96vefPmZW/nzkI9y6e+WiYNr+dHtZaJbbPc1LN87OvlZdssL/UsH/t6ealnednXy0s9N7ZND7rcnm666aYsXLgwEyZMSNOmm+7QMXTo0AwdOrQ0vLnnWWwvjXm/9u3bf+Dt3FmoZ3k1tC5quXm2zfJSz/Kyr5ePbbO81LO87OvlpZ7lY18vrw9rPbt27brZ1xr18402bdqkadOmqa2trTO+pqYmFRUVm5ynoqIiNTU1dcbV1tamWbNm2W233eqMv/HGG/Pwww/ne9/7Xjp16tSYJgIAAAA7uEaFEs2bN0+PHj0yb968OuOfeuqp9OnTZ5Pz9OrVa6OfdsybNy89evRI8+b/02Fj6tSppUBizz33bEzzAAAAgJ1Ao//6xuc///nMnj07s2bNyuLFizN16tS8+uqrOeKII5Ik11xzTa655prS9EceeWReffXV3HjjjVm8eHFmzZqV2bNn59hjjy1N87Of/SyzZ8/ON77xjey6666pqalJTU1NVq9evQ2rCAAAAOyIGv1MiYMPPjhvvPFG7rzzzrz22mvp1q1bxo8fnw4dOiTZ+LcwHTt2zPjx43PTTTeV/jzo6NGjU1VVVZrmvT8nOmHChDrzfvGLX0x1dXVjmwoAAADsgLbpQZdHHXVUjjrqqE2+dvHFF280rm/fvpk0adJml3fbbbdtS3MAAACAnUijf74BAAAAsC2EEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAqqE3eQAAIABJREFUAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIZoX3QAAAACgPK6++uotvn7uued+QC3ZOnpKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIXYpgdd/u53v8uvf/3r1NTUpLKyMqeddlr222+/zU7/9NNP56abbsrixYvTrl27HHfccTnyyCO3aZkAAADAzqnRPSUeeeSR3HjjjfnCF76QSZMmpU+fPvnBD36QFStWbHL65cuXZ+LEienTp08mTZqUE044IVOnTs3cuXMbvUwAAABg59XoUOK+++7LoYcemqFDh6aysjJjxoxJu3bt8sADD2xy+gceeCDt2rXLmDFjUllZmaFDh+bQQw/Nvffe2+hlAgAAADuvRoUSa9euzV//+tcMHDiwzvgBAwZk4cKFm5znueeey4ABA+qMGzhwYP76179m7dq1jVomAAAAsPNq1DMlXn/99axfvz5t27atM76ioiJPPfXUJuepqanJ/vvvX2dc27Zts27durzxxhvZsGFDg5c5c+bMzJw5M0ly2WWXpX379o1Znc2765EtvvyZH82pdxFfad75H8bU1Bl6Ydm/b3H+S4b9rd73WF91fb3T7BC2sZ4b1zIpdz0/LLVMGlPPhtUy+ejU077eQDvBvp7sJPW0r5fXDrCvJ+r5fo6d/78PYF9PXCe9n2NnA3wA+/ros3pucf6mc79a73t8VOq5NddJEyZM2OIy6qvnB13LbXrQZdGGDh2aoUOHloY/qs+e+Kiu9/agluWlnuWlnuWlnuWjluWlnuWlnuWjluWlnv+jvlp0LMMyPkq2tZ7bo5Zdu3bd7GuNCiXatGmTpk2bpra2ts74mpqaVFRUbHKeioqK1NTUTXBqa2vTrFmz7LbbbknS4GUCAAAAO69GPVOiefPm6dGjR+bNm1dn/FNPPZU+ffpscp5evXpt9DOMefPmpUePHmnevHmjlgkAAADsvBr91zc+//nPZ/bs2Zk1a1YWL16cqVOn5tVXX80RRxyRJLnmmmtyzTXXlKY/8sgj8+qrr+bGG2/M4sWLM2vWrMyePTvHHnvsVi8TAAAA+PBo9DMlDj744Lzxxhu5884789prr6Vbt24ZP358OnTokGTj36F07Ngx48ePz0033VT686CjR49OVVXVVi8TAAAA+PDYpgddHnXUUTnqqKM2+drFF1+80bi+fftm0qRJjV4mAAAA8OHR6J9vAAAAAGwLoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQiOZFNwAAAABI7jll3y2+fu+tNR9QSz44ekoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFaF50AwA+Cs4999wtT/D8+A+mIR8S6lk+9dYyUc8GUM/ysq+Xl3qWj30dykdPCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQHnQJAAAAHxHLe04sugl16CkBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFKJ50Q0AAACAzVnec2LRTWA70lMCAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAAChE88bMtGHDhtx+++2ZNWtWVq5cmV69emXs2LHp1q3bFuebO3dubr311ixbtiydOnXKSSedlIMOOihJsnbt2kyfPj1PPPFEli1bllatWqVfv3455ZRT0r59+8Y0EwAAANiBNaqnxD333JP77rsvo0ePzsSJE9OmTZtccsklWbVq1WbnWbRoUa666qoMHjw4l19+eQYPHpwrrrgizz33XJJkzZo1eeGFFzJ8+PBMmjQp3/72t/PKK6/k0ksvzbp16xq3dgAAAMAOq8GhxIYNG3L//ffnhBNOSFVVVbp3756zzz47q1atypw5czY734wZM9KvX78MHz48lZWVGT58ePr165cZM2YkSVq3bp0LL7wwBx98cLp27ZqePXvm9NNPz5IlS7JkyZLGryEAAACwQ2pwKLF8+fLU1NRkwIABpXEtWrTIfvvtl4ULF252vkWLFmXgwIF1xg0cODCLFi3a7DxvvfVWkuTjH/94Q5sJAAAA7OAa/EyJmpqaJElFRUWd8W3bts1rr722xfnatm270TzvLe8frV27Nr/4xS/yqU99Knvssccmp5k5c2ZmzpyZJLnssst2ymdPTJgwYcsTzP1qvcvYGdd7e9nWeqrl/6i3lol6NkC9tXi+DMv4CFHP8tmqOtRTT7X8H+pZXvb18trWeqrl/7Cvl5daNMSm75/fb2erZ72hxEMPPZQpU6aUhsePH79dG5Qk69aty9VXX50333wz3/72tzc73dChQzN06NDS8IoVK7Z728qtvjZ3LMMyPkq2tZ5q+T+2phbqufXs6+WlnuVjXy8v9Swv+3p5uU4qH/t6ealFee2I9ezatetmX6s3lBg0aFB69epVGn7nnXeSvNvz4f0JTG1t7UY9Id6voqIitbW1dcbV1tZu1ONi3bp1+dGPfpSXX345F198cXbbbbf6mggAAADshOp9pkSrVq3SuXPn0r/KyspUVFRk3rx5pWnWrFmTZ599Nn369Nnscnr37l1nniSZN29eevfuXRpeu3Ztrrzyyrz00ku56KKLNgosAAAAgA+PBj9TokmTJhk2bFjuuuuu7LnnnunSpUvuvPPOtGzZMoccckhpugkTJqRnz545+eSTkyTDhg3LRRddlLvvvjsHHnhgHnvssSxYsKD0u/V169bliiuuyF/+8pd85zvfSZMmTUrPm2jdunVatGhRjvUF2C6OHVE3RG3fvv0O2XVuZ/CPtUzUc1vYNstLPctLPctHLctLPeGD0+BQIkmOP/74rFmzJjfccEPefPPN9OzZMxdccEFatWpVmmbZsmV1HlDZp0+fjBs3LtOnT8+tt96azp07Z9y4caWfhrzyyiv5r//6ryTJd7/73Trvd+aZZ+awww5rTFMBAACAHVSjQokmTZqkuro61dXVm51m8uTJG42rqqpKVVXVJqfv2LFjbrvttsY0BwAAANgJ1ftMCQAAAIDtQSgBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFKJRf32DD87ynhOLbsKHinqWl3oCAADbQk8JAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQzYtuAMCO7p5T9i26CR8q6lle6lk+alle6lle6lle6gk7Dj0lAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQjQvugE7s3tO2bfeae69teYDaMmHQ331VMuGUU8AAGBHp6cEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQiOaNmWnDhg25/fbbM2vWrKxcuTK9evXK2LFj061bty3ON3fu3Nx6661ZtmxZOnXqlJNOOikHHXTQJqedMmVKZs6cmZEjR+a4445rTDMBAACAHVijekrcc889ue+++zJ69OhMnDgxbdq0ySWXXJJVq1Ztdp5FixblqquuyuDBg3P55Zdn8ODBueKKK/Lcc89tNO3cuXPz/PPPp127do1pHgAAALATaHAosWHDhtx///054YQTUlVVle7du+fss8/OqlWrMmfOnM3ON2PGjPTr1y/Dhw9PZWVlhg8fnn79+mXGjBl1pvv73/+eqVOn5txzz03z5o3qyAEAAADsBBocSixfvjw1NTUZMGBAaVyLFi2y3377ZeHChZudb9GiRRk4cGCdcQMHDsyiRYtKw+vWrcuPfvSj/NM//VMqKysb2jQAAABgJ9Lgrgg1NTVJkoqKijrj27Ztm9dee22L87Vt23ajed5bXpLcdttt2W233XLkkUduVVtmzpyZmTNnJkkuu+yytG/ffqvm+2DVbPHVHbPNO6ot1zJRz4axbW5PzZs3b1gNn69/ko/yZ6Ke5dPgWib11vOjWstEPcvNvl4+ts3yUs/tSy0a4sN3T1RvKPHQQw9lypQppeHx48dvl4YsWLAgs2fPzr/+679u9TxDhw7N0KFDS8MrVqzYHk3brnbGNu/I1LN81HLbtG/fvkE17LgV03yUPxP1LJ+G1jKpv54f1Vom6llu9vXysW2Wl3puX2pRXjtiPbt27brZ1+oNJQYNGpRevXqVht95550k7/Z8eH8CU1tbu1FPiPerqKhIbW1tnXG1tbWlHhcLFixITU1NTj/99NLr69evzy233JL7778/P/nJT+prKgAAALATqTeUaNWqVVq1alUa3rBhQyoqKjJv3rz07NkzSbJmzZo8++yzGTly5GaX07t378ybN6/On/ecN29eevfunSQ56qijUlVVVWeeSy+9NJ/5zGfq9IYAAAAAPhwa/KDLJk2aZNiwYbnnnnvy6KOP5uWXX861116bli1b5pBDDilNN2HChEybNq00PGzYsMyfPz933313lixZkrvuuisLFizIMccck+Td50t07969zr/mzZunoqJii109AAAAgJ1To/7m5vHHH581a9bkhhtuyJtvvpmePXvmggsuqNOjYtmyZdljjz1Kw3369Mm4ceMyffr03HrrrencuXPGjRtX56chAAAAwEdHo0KJJk2apLq6OtXV1ZudZvLkyRuNq6qq2ugnGluyqWUAAAAAHw4N/vkGAAAAQDkIJQAAAIBCCCUAAACAQjTqmRIAAABwzyn71jvNvbfWfAAtYWelpwQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQiOaNmWnDhg25/fbbM2vWrKxcuTK9evXK2LFj061bty3ON3fu3Nx6661ZtmxZOnXqlJNOOikHHXRQnWn++7//O9OmTcv8+fOzdu3a7LnnnjnnnHNSWVnZmKYCAAAAO6hG9ZS45557ct9992X06NGZOHFi2rRpk0suuSSrVq3a7DyLFi3KVVddlcGDB+fyyy/P4MGDc8UVV+S5554rTbN8+fJceOGF6dixY773ve/l3/7t3zJixIi0bNmyMc0EAAAAdmANDiU2bNiQ+++/PyeccEKqqqrSvXv3nH322Vm1alXmzJmz2flmzJiRfv36Zfjw4amsrMzw4cPTr1+/zJgxozTNL3/5ywwcODCnnnpqevTokU6dOuWAAw5I+/btG7d2AAAAwA6rwaHE8uXLU1NTkwEDBpTGtWjRIvvtt18WLly42fkWLVqUgQMH1hk3cODALFq0KEmyfv36PP7446msrMyll16asWPHZvz48XnkkUca2kQAAABgJ9DgZ0rU1NQkSSoqKuqMb9u2bV577bUtzte2bduN5nlvea+//npWr16du+66KyNGjMgpp5yS+fPn5+qrr07Lli1zwAEHbLTMmTNnZubMmUmSyy67bAftUVGzxVd3zDbvqLZcy0Q9G8a2uT01b968YTV8vv5JPsqfiXqWT4NrmdRbz49qLRP1LDf7evnYNstLPbeV687y+fDdE9UbSjz00EOZMmVKaXj8+PHbpSHr169PkgwaNCif//znkyR77713/vKXv+S3v/3tJkOJoUOHZujQoaXhFStWbJe2bU87Y5t3ZOpZPmq5bdq3b9+gGnbcimk+yp+JepZPQ2uZ1F/Pj2otE/UsN/t6+dg2y0s9ty+1KK8dsZ5du3bd7Gv1hhKDBg1Kr169SsPvvPNOknd7Prw/gamtrd2oJ8T7VVRUpLa2ts642traUo+LNm3apFmzZhv9lY0999zTTzgAAADgQ6jeZ0q0atUqnTt3Lv2rrKxMRUVF5s2bV5pmzZo1efbZZ9OnT5/NLqd379515kmSefPmpXfv3kne7RK1zz775L//+7/rTPO3v/0tHTp0aNBKAQAAADu+Bj9TokmTJhk2bFjuuuuu7LnnnunSpUvuvPPOtGzZMoccckhpugkTJqRnz545+eSTkyTDhg3LRRddlLvvvjsHHnhgHnvssSxYsCATJkwozXPcccflyiuvzH777Zf+/ftn/vz5eeSRR/Ktb32rDKsKAADAB+3YEXWfR9iYn8Pw4dXgUCJJjj/++KxZsyY33HBD3nzzzfTs2TMXXHBBWrVqVZpm2bJl2WOPPUrDffr0ybhx4zJ9+vTceuut6dy5c8aNG1fnpyEHHXRQvva1r+Wuu+7K1KlT06VLl5x11lmbfJ4EAAAAsHNrVCjRpEmTVFdXp7q6erPTTJ48eaNxVVVVqaqq2uKyDzvssBx22GGNaRYAAACwE6n3mRIAAAAA24NQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAAChE86Ib8GF37IiKOsPt27fPihUrCmrNzu0fa5mo57awbQIAAEXTUwIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAoRPOiGwAAAADU79gRFRuNa9++fVasWFFAa8pDTwkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBDNGzPThg0bcvvtt2fWrFlZuXJlevXqlbFjx6Zbt25bnG/u3Lm59dZbs2zZsnTq1CknnXRSDjrooNLrq1evzrRp0/LYY4/ljTfeSPv27XPEEUfk85//fGOaCQAAAOzAGtVT4p577sl9992X0aNHZ+LEiWnTpk0uueSSrFq1arPzLFq0KFdddVUGDx6cyy+/PIMHD84VV1yR5557rjTNTTfdlD/96U85++yzc+WVV2b48OGZNm1a/vjHPzammQAAAMAOrMGhxIYNG3L//ffnhBNOSFVVVbp3756zzz47q1atypw5czY734wZM9KvX78MHz48lZWVGT58ePr165cZM2aUplm0aFGGDBmS/v37p2PHjjn00EPTq1evOsEFAAAA8OHQ4FBi+fLlqampyYABA0rjWrRokf322y8LFy7c7HyLFi3KwIED64wbOHBgFi1aVBru06dPHn/88axYsSJJsnDhwrz44ov5xCc+0dBmAgAAADu4Bj9ToqamJklSUVFRZ3zbtm3z2muvbXG+tm3bbjTPe8tLkjFjxmTKlCk588wz06xZsyTJ6NGj86lPfWqTy5w5c2ZmzpyZJLnsssvSvn37hq7OB6558+Y7RTt3FupZPmpZXg2u5/P1T/JR/nzUs3wata/XU8+Pai0T9Sw3+3r52DbLSz3Ly3Vnee3s9aw3lHjooYcyZcqU0vD48eO3W2N+85vfZOHChfn2t7+dDh065JlnnskvfvGLdOzYcZO9JYYOHZqhQ4eWht/rYbEja9++/U7Rzp2FepaPWpZXQ+vZcSum+Sh/PupZPo3Z1+ur50e1lol6lpt9vXxsm+WlnuXlurO8doZ6du3adbOv1RtKDBo0KL169SoNv/POO0ne7fnw/jSmtrZ2o54Q71dRUZHa2to642pra0s9LtasWZNp06blvPPOy6BBg5Ike+21V1588cXce++9fsIBAAAAHzL1hhKtWrVKq1atSsMbNmxIRUVF5s2bl549eyZ5N1B49tlnM3LkyM0up3fv3pk3b16OO+640rh58+ald+/eSZK1a9dm3bp1adq07mMumjZtmvXr1zdsrQB2Mst7Tiy6CR8q6lle6lle6lk+alle6lle6glbp8EPumzSpEmGDRuWe+65J48++mhefvnlXHvttWnZsmUOOeSQ0nQTJkzItGnTSsPDhg3L/Pnzc/fdd2fJkiW56667smDBghxzzDFJktatW6dv376ZNm1aFixYkOXLl2f27Nn5j//4jxx00EFlWFUAAABgR9LgB10myfHHH581a9bkhhtuyJtvvpmePXvmggsuqNOjYtmyZdljjz1Kw3369Mm4ceMyffr03HrrrencuXPGjRtX56ch48aNy7Rp03L11Vdn5cqV6dChQ0aMGJGjjz56G1YRAAAA2BE1KpRo0qRJqqurU11dvdlpJk+evNG4qqqqVFVVbXaeioqKnHnmmY1pEgAAALCTafDPNwAAAADKQSgBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFKLJhg0bNhTdCAAAAOCjR0+JD9h3v/vdopvwoaKe5aOW5aWe5aWe5aOW5aWe5aWe5aOW5aWe5aWe5bWz11MoAQAAABRCKAEAAAAUotnFF198cdGN+Kjp0aNH0U34UFHP8lHL8lLP8lLP8lHL8lLP8lLP8lHL8lLP8lLP8tqZ6+lBlwAAAEAh/HwDAAAAKIRQAgAAACiEUKIAZ511Vn79618X3Qw+BNavX58pU6ZkzJgxqa6uzoIFC7br+1188cW54YYbtut7fFip3cYmT56cyy67rOhmsA2WL1+e6urq/OUvfym6KdvNggULUl1dnddff32Tw1tr9uzZGTVq1PZo4nZVXV2duXPnbtW0jnPsDG677bb88z//c9HN+FD5KJwLdlRbc2559tlnc/755+ekk07Kjvo4SaHE+0yePDnV1dWlf2PHjs1ll12WJUuWFNquxl4A7UjeX9uTTjopZ599dv793/89q1evrnfeza3/znzxs7m2N/Si9c9//nMefPDBfOc738mUKVPSp0+fcjZzI+eff35OPvnk7foeRSrHTfKHYX/dGpMmTcqECRM2+drixYtTXV2dJ5988gNu1Y6tpqYmU6dOzTnnnJOTTz45X/va1/KDH/wgf/rTn4pu2lbZGUOkTbX58ccfz8iRIzN9+vTt9r474pcPv//97zNq1KisXbu2NG7t2rUZOXLkRjdoS5fZCLt7AAAgAElEQVQuTXV1dZ566qlMmTIln/rUpz7o5pbVjn5c3pp96x/DofXr1+faa6/NmDFj8txzz23vJn5g3n+9eOKJJ+aMM87I9ddfn5UrVxbdtJKt3Z7eu1EfMWJEVqxYUee1lStX5pRTTtkhb+Qbc65q3759pkyZkr333vuDa+gOZEe9h3zPjTfemL322is//vGPc/7552/z8rbHMbV52Zb0IbH//vvnnHPOSZK8+uqrufnmm/PDH/4wV155ZcEt2/m9V9u1a9fm2WefzU9+8pO8/fbb+epXv1p003ZaS5cuTbt27bY5jFi7dm2aN6//cLDrrrtu0/vw4XH44Yfnhz/8YZYvX56OHTvWee0Pf/hDOnTokP3337+g1u14li9fngsvvDCtWrXKSSedlL333jvr16/P/Pnzc/311+e6664ruokfCX/84x/zk5/8JCNHjsywYcOKbs4Hql+/fnn77bfz/PPPZ999902SPPfcc2ndunX+9re/5fXXX0+bNm2SJPPnz88uu+ySPn36pEWLFkU2e4eztefL7WnNmjW56qqr8sILL2TChAmprKwstD3l9t714rp167J48eJcd911efPNNzNu3Liim9You+++ex588MF86UtfKo2bM2dO2rZtu1FYUbTGnKve2ycqKioKaPGOo6H3kB/ksWTp0qU56qij0r59+w/k/RpDKPEPdtlll9JOVVFRkWOOOSaTJk3KmjVrSifmW265JY899lhWrFiRioqKfPrTn051dXWdE/ef/vSn3HHHHXnppZfysY99LL1798555523yZP7H//4x9xwww0555xzMmjQoDqvLV++PN///veTJF/5yleSJIceemjOOuusvPPOO7nlllvy8MMP56233sree++dUaNGlS42djTvr+0hhxyS+fPn5//+3/+bPn365De/+U2WLFmSFi1apG/fvjnttNOy++67b3b9k+Tpp5/O008/nd/97ndJkmuuuSYdO3bM008/nZtvvjkvvfRSWrdunc985jMZOXJkace/+OKLU1lZmdatW2fWrFlp0qRJhgwZkpEjR6Zp0x2n89DkyZPzxhtvZMCAAbnnnnuyZs2aHHjggRk7dmw+9rGPZfLkyfmP//iPJO9+g9KhQ4dMnjy53u1iwYIF+f73v5/vfve7uf322/Piiy/m/PPPT79+/fKzn/0sjz76aFq2bJlhw4Zl4cKF2W233XLWWWclebd23bp1y9ixYzNt2rQ8+eSTmTRpUp12/5//83/So0ePjBkzJkny4IMP5te//nWWL1+e9u3b54gjjsiwYcNKta6urs7pp5+eefPm5c9//nPatm2b6urqDBky5IMq9WbV9xn8oy3tr0myYcOGTJs2bbPb3dq1azN9+vTMmTMnK1euTLdu3TJixIh84hOf+IDWeOsdcMABadu2bWbPnp3q6urS+LVr1+ahhx7KUUcdlWeffXaL++I/2rBhQ+677778/ve/z4oVK9KmTZsMGTKk1DunvmPvbbfdlkcffTTDhw/P9OnTU1tbm/79++frX/966WZr/fr1ufPOOzNr1qzU1tamS5cuOfHEE3PggQdu13q91zPqsssuS8uWLUvjKysrM3jw4NLwihUrMnXq1Dz11FNJkgEDBmT06NHZY489tnodn3/++UyfPj0vvPBC1q5dm+7du2fUqFHp3bt36X0aut/ddtttdY43SXLRRRelQ4cOSZK///3vmTZtWhYuXJgOHTpk9OjRGTBgQGn+xYsX5xe/+EWeeeaZtGjRIv37989pp532gV7EzpgxI7fccku+/vWvl9Zz9uzZueGGG/KLX/yiNN17x8if/exnpZo2xMUXX5y///3vufnmm3PzzTcnebd+73nqqady4403Zvny5enZs2fOOOOMOsHef/3Xf+X222/P4sWLU1FRkUMOOSRf+tKXtvnitWvXrmnXrl3mz59f53zQv3///P3vf8+CBQvy6U9/ujS+d+/eadGiRaqrq3PeeeelqqoqSXLHHXfkD3/4Q2pqavLxj388AwcOzNlnn116n/qOc2eddVY++9nPZtmyZXn00Ufz8Y9/PKNGjcrAgQNz/fXX5/HHH0+7du0yduzYDBw4MMm7++1Pf/rTzJ8/PzU1Ndljjz3yuc99Lscee2xpuS+//HJuvPHG/OUvf8n69evTuXPnfPnLX07Hjh03e1zesGFDfv3rX2fmzJl59dVX07lz5xx//PGl7WP58uU5++yzc+6552bWrFlZtGhRRo0alaOPPnqbPott8dZbb+Xyyy9PTU1N/uVf/qXOTcbW7Ncvv/xybrrppjz77LNp0aJFBg0alNGjR6d169ZZsmRJvvnNb2bKlCmpqKjI22+/ndGjR6dfv3654IILkiSzZs3K3XffnR//+MfbbR3ff724xx575OCDD87s2bNLr2/Ncfy9m8Innngia9asSZcuXfLlL385/fv33+j9VqxYkUsuuaS0P27YsGGz5+L6zvObcthhh2X27Nn54he/mCZNmiR5N7w/7LDDcscdd9SZthznuW2xNeeq6urqjBkzJvPnz8+TTz6ZI444IkcffXTOPvvsTJw4Mfvss0/pODp+/PhMnz49ixcvzj777JNvfOMbWbZsWaZOnZqlS5emX79+Oeuss7Lbbrsl2fx+/N7ntiNf42/pHrKmpmazx5L6rpPvu+++zJ49O8uWLUvr1q3zyU9+MqNGjcrHP/7xTbZj5cqVmTRpUlq2bJlTTz211BPuuuuuy3XXXZczzzwzQ4YM2S7H1G0hlNiCVatW5ZFHHkn37t3rhAkf+9jHcsYZZ2T33XfP4sWLc/3116d58+Y58cQTkyRPPPFELr/88pxwwgk588wzs27dujz55JPZ1F9fvf/++3P77bfnO9/5Tvr27bvR6+3bt88///M/59/+7d9yxRVXZNdddy215eabb85//ud/li5o7rvvvlx66aW5+uqr065du+1UlfJp0aJF1q1bl7Vr1+ZLX/pS9txzz7zxxhu55ZZb8qMf/Sjf//73t7j+f/vb39K1a9fSDUubNm3y6quvZuLEiRk8eHDOPPPMLFu2LD/5yU/StGnTnHrqqaX3fuihhzJs2LD8y7/8S1588cVcffXV6dGjRw455JBCarE5zzzzTCoqKnLhhRfmlVdeyZVXXpkuXbrkC1/4QkaPHp0OHTrkwQcfzMSJE0sHka3dLm655Zaceuqp6dy5c1q1apV///d/z9NPP51vfetbadeuXX71q1/lmWeeyUEHHbTJtg0ZMiR33313lixZkj333DNJsmzZsixatCinnXZakmTmzJm57bbbMmbMmPTo0SMvv/xyfvrTn6Z58+Z1LuruuOOOnHzyyTn55JPzhz/8Idddd1369u27QyS6W/oM/tGWttek/u3u2muvzbJly3Luuedmjz32yJ///OdMmjQpEydO3OG6RDZr1iyHHnpo6ULrve3v8ccfz+uvv55DDz005513Xr374vv98pe/zAMPPJBTTz01ffv2zeuvv54XXnih9Hp9x97k3ZuIRx55JOeff37efvvtXHXVVZk+fXpOP/30JO8ec++999589atfTY8ePfLQQw/lhz/8YSZNmrTdarxy5co88cQTGTFiRJ2LvPe8d2Gxfv36XH755WnRokUuuuiiJMnPf/7z/Ou//msmTpxYupitbx1Xr16dIUOG5LTTTkuTJk3y29/+NhMnTszVV19duvBLGrbfHXfccVmyZElWrlxZ+iZo1113zauvvpokmT59ekaOHJmvfOUr+dWvfpWrrroq1157bVq2bJnXXnstF110UT772c9m1KhRWbduXX75y1/m8ssvzyWXXPKBhMHTp0/Pfffdl/PPPz8HHHDAdn2v888/P9/61rfy2c9+NkceeWSd19auXZu77747Z5xxRnbZZZdMnjw5119/femG74knnsiPf/zjnHbaadlvv/2yYsWKXH/99XnnnXc2u980RL9+/bJgwYJ88YtfTPJu+HDIIYdk+fLldUKJp59+OkccccRG88+dOzf33ntvvvGNb6R79+6pra3d6KcDW3N+nTFjRk488cQMHz48v//97zN58uT0798/Bx98cE488cTcdddd+fGPf5xrr702LVq0yPr167P77rvnm9/8Ztq0aZPnn38+U6ZMyW677ZbDDz88SfKjH/0oe+21V37wgx+kWbNmefnll9OiRYstHpenT5+euXPnZuzYsenatWsWLVqUn/70p9l1113rbCe//OUvM2rUqJxxxhlp1qzZNn8OjVVbW5vvf//7adKkSSZMmLDJm9At7derV6/OpZdemn322ScTJ07MypUr89Of/jTXXnttzj///Oy5556pqKjIggUL8pnPfCYLFy5Mq1atsnDhwqxbty7NmjXLggUL0q9fvw9snZctW5YnnniiTt3rO46vXr06F198cdq2bVu6pnnppZc2ufzFixfn0ksvTVVVVU499dQ0adIkV1999WbPxd27d9/ieX5TPvnJT2bWrFmZP39+9t9//7zwwgtZtmxZPv3pT28USpTjPNdYW3uuSt7dzk466aSMGjWqdG7alNtuuy2nnXZaWrdunauvvjpXXXVVdtlll5x++ulp2rRprrjiitx+++2lL7I2tx8n2amu8Td3D/mPx5KtuU5u0qRJTjvttHTs2DErVqzIz3/+8/z85z8vnYvf79VXX82ll16aysrKnHPOOWnatGmmTJmSc845JyeddFIOPvjgtG7dersdU7fFjvO18A7iiSeeyKhRozJq1Kh8+ctfztNPP51zzz23zjRf/OIXs++++6Zjx4454IAD8oUvfCEPP/xw6fVf/epXqaqqyoknnpjKysrstddeOe644zb6ZnX69Om566678r3vfW+TgUSSNG3atNRlvk2bNqmoqEjr1q2zevXqPPDAAznllFNywAEHpLKyMqeffnoqKipKPQd2ZM8//3wefvjh9O/fP4cffngOOOCAdOrUKT179sxXvvKVPPPMM3nllVc2u/6tW7dO8+bN87GPfSwVFRWpqKhI06ZN87vf/S7t2rXLV77ylVRWVuZTn/pUTjnllPz2t7/N22+/XXr/ysrKjBgxIl27ds3BBx+cfv36Zf78+UWVY7Nat26d008/PZWVlRk4cGCqqqpK7WzdunVatmyZpk2bpqKiIm3atGnQdvGlL30pAwcOTKdOndKiRYs8+OCDOeWUUzJgwIB069YtX//617d4s1BZWZn/9b/+Vx566KHSuDlz5qRLly7p2bNnknf3hZEjR6aqqiodO3bMoEGDcsIJJ2zUliFDhmTIkCHp3LlzRowYkWbNmuXpp58uVxm3yZY+g3+0ue31PVva7pYuXZqHH3443/zmN9O3b9906tQpRx99dD75yU9m5syZ239FG+Hwww/PihUrSt/qJ+9++zNw4MDMnDlzq/bF96xevTozZszIySefnMMPPzydO3dO7969c9RRR5Wmqe/Ym7x7Y3/WWWdlr732Su/evTN06NA67bv33ntz7LHH5pBDDknXrl0zYsSI7Lffftv19/9Lly7Nhg0b6u1iPX/+/Lz00ks599xzs88++2SfffbJueeemxdeeKHOOtS3jv3798+QIUNSWVmZPffcM2PGjMkuu+ySP//5z3XeryH7XcuWLdOiRYvSN0EVFRV1vrk/5phjMmjQoHTp0iUnn3xyVq5cmRdffDFJ8sADD2SvvfbKyJEjS+fEs88+O88//3z++te/NrScDTZv3rzceeedOe+887Z7IJG8G9Y0bdo0LVu2LNXqPevWrcvYsWPTs2fP7LXXXjn22GOzYMGC0pcWd911V4499th89rOfTefOndO/f/+ccsop+f3vf7/JLzYaqn///lm0aFHeeeedrFmzJosWLUq/fv3St2/f0oOSlyxZktdee22z3yhXVFRkwIABad++ffbZZ5+Neg1szfl14MCBOeqoo9KlS5dUV1fnnXfeSadOnXLooYemc+fO+ad/+qe8/vrr+X//7/8lSZo3b54RI0akZ8+e6dixYw4++OAcccQRdfb/FStWZMCAAdlzzz3TuXPnHHTQQendu/cWr6Puu+++fP3rX88nPvGJdOzYMYccckg+97nPbXSOOvroo0vnsfd6LRXhpptuyltvvZWLLrpos9+Kb2m/njNnTlavXp1zzjkn3bt3T9++fXP66afnsccey9KlS5Okzrbw9NNPp6qqKrvttlvpuQfPPPPMZq9Zy+W9a/FTTjkl55xzThYvXpzjjz++9Hp9x/E5c+akpqYm3/rWt7Lffvulc+fO+d//+39vtE0/99xzueiii3LEEUfky1/+cpo0aVLvubi+8/ymNGvWLEOGDMmDDz6Y5N3z5Kc//elN9rgsx3musbb2XJUkBx98cD73uc+lU6dOG/2E8/3e+2z22muvHHHEEVm4cGFGjhyZXr16ZZ999smhhx5a5yHtm9uPk+zw1/hbcw/5j8eSrblOPuaYY9K/f/907Ngxffv2zciRI/Of//mfWb9+fZ1lL126NBdeeGH69OmTb3zjG2nevHnpHiF593q2oqIiLVq02C7H1G2lp8Q/2G+//fK1r30tybuJ4QMPPJBLL700l156aenbo7lz52bGjBlZunRpVq9enfXr19fZMF544YXSTww25ze/+U1WrVqViRMnpkuXLg1u57Jly7Ju3bo6zxJo2rRpevXqlcWLFzd4eR+E93bW9evXZ+3atTnwwAMzZsyY/PWvf80dd9yRF198MStXrixdeK1YsaLBJ/8lS5akV69edW6m991336xduzZLly7NXnvtlSSl/76nXbt2qa2t3cY1LL/Kyso667L77rvn+eef3+z0Ddku9tlnn9L/L126NOvWrSuFCcm7NyHdunXbYvsGDx6c3/3ud6UEf86cOaXufa+//npeeeWVTJkyJddff31pnvXr1290cd29e/fS/zdr1ixt2rTZYR5I1tDPYEu2tN298MIL2bBhQ775zW/WmWbt2rWbvDnYEXTp0iV9+/bNgw8+mIEDB+bVV1/Nk08+mXHjxmXOnDlbtS++Z/HixXnnnXe2+ByK+o69ybu9Vd5/cmzXrl1pW3rrrbfy2muvbfQMln333XejG/Zy2tqbycWL/z/2zjysqmp9/B+mwyTDQWYQkAAnFERucv1apJSaXrWMtFQcL1pidUtv2i0TyuFqZVbiFcccSM2BB8RSLjij1jVnQAWVZBBQ4QAOeBjO7w+es38cDigoCOj6PA/PA2uvvfbai7XWu/a73vW+2VhZWWks8Ozs7JDL5WRnZ0vHIR70jlC9m7p161ZSUlJQKBRUVVWhVCq1zi435bir+f9UW2Sp+/aVK1dIS0ur04lvXl6exrzTHHTo0IG7d++ybds2OnXqVK/J65PAwMAAR0dH6W+5XE5FRQV37tyhXbt2XLlyhYyMDGJjY6U8KpVKMv99XCtIb29vysvLuXTpEiqVCnNzc+zt7bG0tCQvLw+FQkFKSgqGhoZ1/l8CAgL45ZdfmD59Oj4+Pvj6+uLv74+BgYGUpyHytWYeIyMjDA0NNfqjehFd876EhAT27dvHjRs3UCqVVFZWSseHoHrhHhUVxcGDB+nevTu9e/eWrPjqQj3nLFiwQCO9drmgKS9bEj8/P06cOMGvv/7KiBEj6szzoHGdk5ODq6srxsbGUp5OnTqho6NDdnY29vb2dO3ald27dwPVljSvvvoqSqWSlJQUzM3NuXXrVrNbSqjX4kqlksTERPLz8yUfMA2ZxzMzM3F1dX3gcYbCwkK+/PJLgoODGTZsmJTeXLK4f//+fPzxxygUCo4cOcInn3xSZ77HlXOPQ2MUn+7u7g3KV3OsW1hYAGiN9Zrj/EHjuLWv8R/0Damm5lzS0HXy+fPniYmJIScnh7t370rfUQqFAisrK6C6f86ZM4fevXtLRyoeRlPPqY+LUErUwtDQEHt7e+lvd3d3xo8fT2JiIm+99RaXLl1i6dKlBAcHM378eExNTTlx4oTGedSG0KlTJ86cOUNycrJkRvm0ox6senp6yOVy9PX1JVPC7t27M336dCwsLCgtLeXzzz/X8BDeFNQ0L6ttfqmjo9Mku1ANxdjYmLt372ql37lzR0PQ1GUm2lT1rEtD31j+7//+j02bNnHp0iX09fXJycmRlBJqIRoaGvpQR5y1z0rr6OhoCeGWoin/Bw/qdyqVCh0dHRYuXKjVHq3Z0Vz//v2Jiori9u3bHDhwgHbt2uHv78+RI0fqvedBpp710dC5tzX2JQcHB2nRX99xqIdRs80e9o6RkZEUFxczfvx4bGxsMDAw4IsvvtCaU5uyrWr2bXVda/btnj171nn8QL1IbU7kcjmzZs0iIiKCL7/8ks8++0za5amrL1ZWVjZbXWpbn6mfr273qqoqgoODpWMUNWmK8+K2trbY2NhIO5NdunQBqhUD7u7upKSkkJKSQufOnev0YWFtbc3SpUs5f/48Z8+eZcOGDWzfvp358+dL5t4Nka91zat1PU9939GjR1m/fr3kG8XExIQ9e/bwv//9T8o7cuRIXnjhBU6dOsWZM2fYtm0boaGhkilyfWXPmjVL68hS7fo1hbxsCvr27UtAQADLli2T+kptHndcq/1L5eXlceXKFbp164ZSqeTIkSOYm5tjZ2fX7NYiNdfikyZNIiIigu3bt2v4L3pczMzMsLGxITk5mf79+0tzQnPJYkdHRzp27Mh3332HpaUlXl5eFBQUaORpaTnXGFlV1/GOuqhLNtSuf835obHjuHbZtZ+pvvYk1vgP+oZU17/mXNKQdfKNGzdYuHAhQUFBjBo1inbt2nH16lW+++47DZmur6+Pj48Pp06d4saNG1qK1do0x5z6uIjjGw1AV1cXpVIJwMWLF7GysiI4OBgPDw8cHBy4ceOGRv6OHTs+1EzI3d2dTz/9lPj4eK3zZLVRD96aE46dnR36+vpcvHhRSquqqiI9Pb3VemFWD1YbGxvpnXJzcyktLWX06NF07doVJycnLW1mXe+vTq+d5uTkRHp6ukb6hQsX0NfXx87Orjle65FwdHSUtPE1uXr1qsYuWmN51H5hb2+Pnp6eRliq+/fvS6az9SGXy/H29ubw4cMcOXIELy8vqZ0tLS2Ry+Xk5+djb2+v9fO0Ul9/fRhubm6oVCoUCoVWW6k14a2RgIAADAwMOHToEPv37+fFF19EX1+/0WPRyckJAwODek1QGzL3PgwTExPkcrnG+FDXqznnzXbt2uHj48PevXvrDIN8584doNoqp7CwUGOhmp+fT1FRUaPqd+HCBQYNGoSfnx8dOnSQ/Do8LnXNuQ2hY8eOZGdnY21trdW3a+7YNidWVlaEh4dz//59vvzyS0pLS4HqD/379+9rKInVx04eh0dtK3d3d3JycuqcM5vKl4Har0Rt3wBqE+fU1NQH7oTLZDL8/PyYMGECCxcuJCsrS2tMNTUXLlzAw8ODQYMG4e7ujr29Pfn5+Vr5HBwcGDx4MJ988gn9+/dn3759QN3zsrOzMwYGBty4cUOrrR+2oG9J+vbtywcffMCOHTs0HKg2BCcnJ65du8a9e/ektIsXL2qY7Kv9SuzcuRM7OzssLCzo2rUrFy9e5OzZs0/Un4Sa4OBgYmNjKSwsbNA87ubmxp9//vlA6wEDAwNmzZpFu3btmDdvnjQPN0QWP6qc79+/PykpKfTr16/O600h5x6Hhsqq5qa+cdxW1vg1qfkNWZuGrJMvX75MRUUFEyZMwMvLC0dHx3rl+bRp0+jcuTMREREPjerSHHPq4yKUErUoLy9HoVCgUCjIzs5m7dq1lJWVSTG6HRwcKCws5PDhw+Tn55OQkKB11uv111/n2LFjkrfZrKws4uPjtc5Re3h48NlnnxEfH8+OHTvqrZONjQ06OjqcPHmSkpISysrKMDIyYsCAAURHR3Py5EnJGY5CodA4g93asba2xsDAgD179pCfn8/JkyfZunWrRp663l+dnpGRQUFBASUlJVRVVTFw4ECKiopYvXo12dnZnDx5kujoaAYNGtRqdjoABgwYQH5+PmvXriUzM5Pc3Fzi4+NJTk7WMCNsLI/aL4yMjOjXrx/R0dGcO3eO7OxsVqxYQVVV1UN3tV944QWOHj1KcnKyRhQBqNayxsbGEh8fT25uLteuXePgwYPExMQ88ju2durrrw/D0dGRvn37snz5co4fP05+fj6XL18mLi6O3377rZlr/ejIZDL69u3Ltm3byM/PlzTojR2LxsbGvPrqq2zevJn9+/eTl5dHRkYGCQkJQMPm3oYwbNgwdu3axZEjR8jNzWXr1q2kpaUxdOjQx2uIhzB58mRUKhWzZ8/m2LFj5ObmkpOTQ0JCghQzvHv37lIc8cuXL3P58mW+//57Onbs2CizYQcHBw4fPkx2djYZGRl89913TRJ2zMbGhqysLHJzcykpKWmwNdvAgQO5e/cuS5cuJT09nfz8fM6ePUtUVJTGx1FzI5fLmTt3LhUVFXzxxReUlJTg6emJoaEhP/30E3l5eRw/frxJ/DLZ2Nhw4cIFCgsLG2VW/cYbb5CcnMzWrVu5du0aOTk5HD9+XIri0RR069aN9PR00tPTNT4wu3btytGjRyVv/nVx4MABkpKSuHbtGgUFBRw4cAA9Pb1HOobaGBwcHLh69SqnTp3i+vXrbN++XcP/iVKpZPXq1aSkpFBQUEB6errGR2pd87KxsTFDhw5l48aN7Nu3j7y8PDIzM0lISGi1fnzU/PWvf+XDDz8kJiaGzZs3N/i+F154AUNDQ5YtW8a1a9dITU1l5cqVPP/88xqbBV27duXw4cNS/7C1tcXc3Jzff/+92f1J1EW3bt1wdnZm586dwMPn8b59+2JhYcFXX31FWloa+fn5nDhxQmvDUCaTMWvWLExMTCTFRENk8aPK+cDAQFavXs2QIUPqvN5Ucu5xaIisai4eNo5b+xr/Yd+QdfGwdbKDgwMqlb9ooLEAACAASURBVIrdu3dTUFDAkSNHpONVtdHV1SUsLAwvLy/Cw8MfqJhojjn1cRHHN2px7tw5yXutsbExjo6OfPjhh9LE7O/vz7Bhw/jxxx9RKpX4+PgwatQoVq9eLZXh5+fHP//5T7Zt20ZcXBzGxsZ4eXlpeeGG/6+YmDdvHlC9IKmNlZUVb775Jlu2bCEqKooXX3yRsLAwxowZAyDFb+7YsSOffvppm4i8ocbc3JywsDA2b97M3r17cXFxYdy4cRpnPOt7/6FDhxIZGclHH32EUqmUQoJ+8sknbNq0iY8//hhTU1P+7//+j7fffrsF31IbOzs7IiIi2Lp1K/Pnz0epVOLk5MSHH35Iz549H6vsR+0X48aNY9WqVSxevBgjIyOGDBlCcXGxxlnhuujduzerV6/m7t279OnTR+NaUFAQhoaG7Nq1i82bNyOTyXB2dm7RcGrNTX39tSFMmzaNnTt3smnTJm7dukW7du3w8PBotT4l1PTv35+EhAQ6deokCSwrK6tGj8XRo0fTrl07duzYwa1bt7C0tJRC2TVk7m0Ir776Kvfu3SM6OhqFQoGjoyMzZsxo9ugmdnZ2LFq0iJiYGKKjoyksLMTMzAxXV1fpDKqOjg4ff/wxa9eulcJtde/enUmTJjXqyMu7777LypUrmTVrltQfm+K88csvv0xqaiqzZ8+mrKxMIyTog7CysuLLL7/kp59+YsGCBSiVSqytrfHx8Xno/NLUWFpaMnfuXL788ksiIiL4/PPPef/999m0aRP79++na9eujBo1imXLlj3Wc0aOHMmqVat47733KC8vb/COtq+vL7Nnz2bHjh3s2rVL+uB/6aWXHqs+NenWrRsVFRW0b99e40O0c+fOKJVKjI2N6z0vbmJiQmxsLBs3bqSyshJnZ2dmzpz5QEd3TcErr7wiedFXqVT07t2boUOHSo4DdXV1uXPnDsuXL6eoqAgzMzP8/PwkPyb1zcujRo3CwsKCXbt2sXr1aoyNjXFzc9Nwqthaef7555kxYwZLliyhsrKSsWPHPvQeQ0NDPv30U3788Uc++eQTjZCgNVErqGorrQ4ePNgilhIAQ4cOZfny5QwfPvyh87iRkRHh4eFs2LCBRYsWUVFRgaOjI+PHj9cqVyaTMXv2bP79738zb948Pvvss4fK4keV87q6ug88htVUcu5xaIisai4aMo5b8xr/Qd+QtY/qqHnYOtnV1ZUJEyYQGxvLli1b6NSpEyEhISxdurTO8nR1dZk+fTrLli0jIiKCuXPn1hlRq7nm1MdBR/UkD9ILBII2Q3l5OdOmTWPYsGHNvossEAgEAoFAIBAInk2EpYRAIACq/Vnk5OTg4eHBvXv3iI2NpaysTMv6QSAQCAQCgUAgEAiaCqGUEAgEEuozbXp6eri5uREREdGiMdkFAoFAIBAIBALB0404viEQCAQCgUAgEAgEAoGgRRDRNwQCgUAgEAgEAoFAIBC0CM+sUiIlJYWRI0c2iUdyQcO4ffs2oaGh5OXltXRVtFiyZAm7du1q6Wo0GNGWT4awsDDi4uJauhoAJCYm8u677zJq1KhGx6Z/0rTm/rlx40bWrl3b0tVoFsLDw1mzZk2zlH3gwAHJ67ZAoEaM9eajNcmftojom62HhsiPuLi4x47eIGjbPBM+JcLDw+nQoQOTJ09u9mcVFBQwffp0rXR/f38+/vjjBpUxcuRIPvroIwICApq6ei1KTEwMPXv2lEKQrVu3josXL5KVlYWlpSWRkZFa9xw9epSYmBiuX7+Oubk5gwYNYtiwYdL1yMhIDh48qHWfoaEhGzdulP5OTU1l/fr1ZGdnI5fLGTZsmEaI1uDgYObOnUtQUBAmJiZN+drNQnO0JcCePXvYu3cvBQUFWFtbM2LECAIDA6XriYmJHDp0iKysLFQqFR07dmTUqFF07txZytOW2jIyMpLS0lJmz55d5/WFCxe2itjXt2/fZs2aNYwbN46AgACMjY1bukoPpKX6Z3h4uEacbTXOzs4sWbIEgOHDh/Pee+8xZMgQ7OzsmvK1G0TtOcvMzAxPT09CQkJwcnJ64vVpKH369HnscMVNjUKhICYmhpMnT3Lr1i0pbN2gQYPw8/Nr6eo9E4ix/mD++9//smHDBtatW4e+fvWSu6KiggkTJmBnZ8c333wj5c3Ly+P9999nzpw5dO/evUmef+DAAdasWaOxHqrN2bNnWbhwIREREXh5eUnpSqWSf/7zn3h7exMaGtok9XmSiL75ZBg5cuQDrwcGBhIaGtrq5MfTivo7dOHChTz33HMtXZ1G8UwoJVqCf/3rXxpx7590LHaAyspKdHV1GxXfvrm4f/8++/btY9asWVKaSqUiMDCQa9eucfbsWa17Tp06xffff8/EiRPx9fUlJyeHqKgoZDKZFL934sSJjBkzRuO+OXPm0KVLF+nvgoICFi5cSL9+/Xjvvfe4cOECa9aswdzcXFL8uLi4YGdnx6FDh6SyWyvN1ZYJCQlER0czdepUPD09ycjIICoqClNTU/z9/YFq5U6fPn3o1KkThoaGxMfHM3/+fBYvXoyDgwPQttryYTwonviT5ObNm1RWVtKrVy/kcnlLV+eBtGT/nDlzJhUVFVK55eXlzJw5k7/+9a9Smrm5OT169CAhIaHFdv67d+/Oe++9B0BhYSGbNm3i66+/5ttvv32k8tRzfXNRUVGBTCZDJpM12zMaS0FBAXPmzMHY2Ji3334bNzc3qqqqOH/+PKtWreI///lPS1fxsamoqJA+ZJ/EfY1FjPWH061bN+7fv09GRoakvE9PT8fExITr169TUlIiyZnz589jYGBAp06dnmgde/TowSuvvEJkZCSLFy+WFPE//fQTKpWqTVpIib755Fi5cqX0+x9//EFUVJRGmlp2tCb5IWidPPVKicjISFJTU0lNTWXv3r0ALFu2TLr+559/snnzZq5du4azszNTpkzB3d1dun7x4kV++uknLl++LE06Y8aMeegOsJmZGZaWllrpdVlBhIWFMXDgQIYNGyaZLqm1qTY2NkRGRvLzzz/z22+/aWjVa2vA1XmGDh3Kjh07KCgoYP369VRVVbFx40b+97//oVQq6dixI+PGjXuiGrRTp04BaAjbSZMmAdUmW3UJiEOHDtGrVy8GDhwIgJ2dHa+99hqxsbEMHDgQHR0dTExMNP4XFy5cID8/X8NaJSEhAblcLj3P2dmZjIwMdu3apfF/8Pf3Jzk5udV/SDdXWx46dIigoCD69u0r5bl8+TKxsbGSsH3//fc1yg0NDeV///sfp0+flpQS0Hba8mHUHJtQPX6nTJnC2bNnOXXqFBYWFowcOZIXX3xRuqewsJANGzZw5swZALy8vJgwYYJG+9Tm5s2brFu3jnPnzgHVi8SJEyfSvn17Dhw4wPLlywGkfr1s2TJsbW2b5Z0fl5bsn+3atdMo9/Dhw9y/f59+/fpppPv7+7N58+YWWwwaGBhI8sHS0pIhQ4awaNEilEolMpmM6Ohofv/9d27evImlpSV//etfGTlypLSoq2+uh2oFxbp16zh06BAA/fv3Z8yYMZLS4tChQ/z666/k5OQgk8no2rUrEyZMwMrKCqg+2hgREcHs2bPZtm0bmZmZzJw5k9LSUg15k5eXx4YNG0hPT6esrAxHR0dGjhxJr169pPcMCwujf//+3Lp1i+TkZIyNjRk8eLDWzuOjoD6m8u9//xsjIyMp3dnZmRdeeAF48Liq2Y4jRoxgy5YtFBcX4+3tzTvvvCN9KFZVVbFp0yb2798PVO/6lZeXk5OTQ3h4OFC3NWZtKyyVSkVcXByJiYkUFhZib2/P8OHDpblDvbv1/vvvk5SUxKVLlwgJCWHAgAHs3LmTpKQkiouLcXBw4K233uIvf/nLA+97EnOvGOsPx9HREblczvnz5yWlREpKCt7e3ty4cYOUlBTpYzUlJQUvLy+Nj7fy8nJWrlxZ7/iJj4/nwIED5OfnY2JiQs+ePQkJCcHU1JSUlBRJdqh3s4ODg+vc2R47dixnzpwhOjqaSZMmSWvmiIgI9PT0+PHHH0lOTubu3bu4ubkREhKi8T4RERGsXr1aGjctvVsr+uaTo+a3jqmpqVYa1G2xExsbS3x8PGVlZfTu3bvONc3+/fuJi4uTrFJeeeUVBg8e3KxK+LbA6dOn2blzJ1lZWQB4eHgwfvx4nJ2dpXXiJ598AkDXrl0lWdXaeer/qxMnTsTLy4uXXnqJlStXsnLlSqytraXrP/30E6NHj2bRokWYmZnxww8/oA5Icu3aNebNm4e/vz9fffUVM2fOJDMzs1l3YBYuXAjA1KlTWblypfR3QykoKODIkSN8+OGHfPXVV+jr67Nw4UIKCwuZPXs2ixcvpkuXLnzxxRcUFRU1xyvUSVpaGu7u7o2y2igvL9eyMJHJZNy6dYsbN27UeU9SUhIdOnTQEETp6en06NFDI5+Pjw9XrlzR0GZ7eHiQkZGBUqlscB1bguZqy/Lyci1NtkwmIyMjQ6OdalJRUUF5ebkkiNS0lbZ8FLZv3y7NCX369OE///kPN2/eBKp3ZyIiIjAwMCA8PJx58+Yhl8v58ssvuX//fp3lVVVVsXjxYoqLi5k7dy5z586lqKiIr776CpVKRZ8+ffjXv/4FwIIFC7TmsNZGa+qfSUlJ+Pr6arWXh4cHhYWFreKs8b179zh69CguLi7S+xkaGvLuu+/y7bffMnnyZJKTk9m5c6fGfbXnenX7HTlyBJVKxbx58wgNDSUxMZFffvlFuq+iooI333yTr776itmzZ1NaWsp3332nVa/o6Gjeeustli5diqenp9b1srIyfH19mTNnDl999RW9e/fm66+/JicnRyPf7t27cXFxYdGiRQwfPpxNmzZx6dKlx2qz27dvc/r0aQYOHKihkFBjamr60HGlpqCggKNHjzJz5kw+++wzMjMz2bJli3R9165dJCUlERoayrx586iqquLIkSONrvOWLVvYt28fkydP5ttvv+X1119n1apVnDx5UiPf5s2bGThwIN9++y1/+ctf+OWXX9i1axdjxozh66+/5vnnn+frr78mMzPzgfc9CcRYbxjdunUjJSVF+jslJYVu3brRtWtXjfTU1FS6deumce/Dxo+Ojg4TJkzgm2++4YMPPiAjI0PyVdCpUycmTJiAoaGhtP6tTyEok8l47733+O9//8uJEydYvnw5w4YNw8vLi02bNnH06FHeffddFi1aRIcOHZg/f/4TXUM2FtE3WzdHjx5ly5YtjBw5kkWLFuHo6Mju3bs18iQmJrJ582ZGjRrFt99+S0hICLGxsSQkJLRQrVsPZWVlDB48mAULFhAeHo6xsTGLFi2ioqKCBQsWANUW+ytXrmTmzJktXNuG89QrJUxMTNDX18fQ0BBLS0ssLS01NGyjRo3C29sbJycn3njjDXJycigsLASqtal9+vRh6NChODg44OnpSWhoKL/99hvFxcUPfO7cuXMJCQmRftLS0hpUX7WW2dTUFEtLy0abj1dUVDB9+nTc3d1xcXHhwoULZGZmMmPGDDw8PLC3t+ett97C1tZW2kl7Ety4caPRZue+vr6cOHGCM2fOUFVVRW5uLvHx8UD1WeLa3L17l2PHjhEUFKSRrlAotLS2FhYWVFZWUlpaKqXJ5XIqKyul/39rpbna0sfHh/3795ORkYFKpeLy5cskJSVptVNNtmzZgpGRkbRDoKattOWj8OKLL/Liiy9ib2/PqFGj0NPTk86PJicno1KpmDZtGq6urjg5OTFlyhTKysr4448/6izv/Pnz/Pnnn7z//vs899xzPPfcc7z//vtcvXqVc+fOIZPJMDMzA6rnh9pzWGujtfTP3NxcUlNTteYDQKpffcrN5ub06dOSbBg/fjypqakaVkjBwcF07twZW1tb/Pz8eP3110lOTtYoo/Zcr6enB1S/28SJE3FycqJPnz4MGzZMakuotpzw8/PDzs4ODw8P/v73v5OWlsatW7c0yn/zzTfx8fHBzs6uTjnk5ubGgAEDcHFxwd7enhEjRuDu7s7x48c18vXo0YNBgwZhb2/Pq6++ir29vWS58Kjk5eWhUqlwdnauN8/DxpWaqqoqwsLCcHV1xcvLi5dfflnj+i+//MLw4cPp06cPTk5OTJgwoU4ryAdRVlZGfHw877zzDr6+vtja2tK3b1+CgoIkC041gwYNIiAgAFtbW9q3b8+uXbsYOnQoffv2xdHRkVGjRtGlSxctB4i173sSiLHeMLy9vbl06RLl5eUolUouXbqkpZTIycmhqKgIb29vjXsfNn6GDBmCt7c3tra2dO3albFjx3Ls2DGqqqrQ19eXLEnV69+6lHhqPDw8eO211/jqq68wMTHhzTffpKysjISEBMaMGYOfn59kUWxpaanVd1sTom+2bn755RcCAwN55ZVXcHR0ZMSIEXh4eGjk2bFjB2PHjpXmNX9/f1577bVW3e+eFAEBAQQEBODg4ICrqyvTpk2joKCAjIwMSV6rLfZrW+60Zp764xsPw9XVVfpdbb5aXFxM+/btuXLlCnl5eRw9elTrvvz8fCwsLOot9/3338fFxUWr7ObGyspKY8F05coVlEqllpPP8vJy8vPzn0id1M9r7HmyoKAg8vLyWLx4MZWVlZLp4rZt2+rUfh86dAiVSqVhSt8Y1PVr7bv7zdWWwcHBKBQK5syZg0qlwsLCgsDAQOLi4ups719++YXExETmzJmjdZyprbTlo1BzXOvp6WFubi5F8bly5QoFBQWMGzdO4x6lUlnveMvOzsbKykrDdNHOzg65XE52draWlU9rp7X0z6SkJORyeZ0OD1u6f3bp0oWpU6cC1bv+CQkJzJ8/n/nz52Ntbc3x48fZvXs3eXl5lJWVUVVVRVVVlUYZted6NZ6enhrt4eXlxdatW7l79y4mJiZcuXKF7du3k5mZye3btyWrgZs3b2p8zD7M5LqsrIzt27fzxx9/oFAoJKupmuMDNGUsVC/EH6bUfxg1LR3qo6HjytraWmP+ksvl0ni+e/cuRUVFGs7/dHV18fDw0FLiPKwu5eXl0g6WmsrKSmxsbDTSara7+vm1fQx07txZMk+v674nhRjrDcPb25vy8nIuXbqESqXC3Nwce3t7LC0tycvLQ6FQkJKSgqGhodaH2cPGz/nz54mJiSEnJ4e7d+9SVVVFRUUFCoXikdadwcHB7Nixg+HDh6Ovr09OTg6VlZUafVBXVxdPT0+ys7MbXf6TQvTN1k1OTg79+/fXSPP09JSsRkpKSrh16xYrV65k1apVUp6qqqoGzf9PO3l5eWzdupWMjAxKSkqkdrl58+YT+95sDp55pYR6d6km6g6vUqno378/f/vb37TyPOyf3r59e8njb010dHS0BlR9Jl810dXVbdB9tbXgVVVVWFhY8MUXX2jlfZIe/M3MzLh9+3aj7tHR0WHs2LGMHj0ahUKBubm5tENQl7fipKQkevfuraUVtLS01LKsKC4uRk9PT9qBBqT6tRbnhvXRXG0pk8mYNm0aU6ZMobi4GLlcTmJiIsbGxlptsnv3brZu3cq//vUvrUUUtJ22fBRqO5DT0dGRPhhVKhVubm784x//0LrvUbTVrcFJbWNpDf2zoqKCgwcPEhQUVOcc39L909DQUEM+uLu7M378eBITE/Hz82Pp0qUEBwczfvx4TE1NOXHihJb3/AfteNZHWVkZ8+fPp3v37kyfPh0LCwtKS0v5/PPPteTJw6LObNy4UbL4cHBwwNDQkGXLlmmVU7v965KBjcXBwQEdHR2ys7N5/vnnG31/zXH1oPHcmPJqv1NlZaX0u/rarFmztEy4a7fPo0b7aYkoQWKsNwxbW1tsbGwkqwi1I24jIyPc3d1JSUkhJSWFzp07a/XHB42fGzdusHDhQoKCghg1ahTt2rXj6tWrfPfddw1aV9aF+nl1tWV91CWnavb/lkD0zbaNeg4ODQ194o5f2wKLFi3CysqK0NBQrKys0NPT46OPPnrkcd9aaL02wE2Ivr5+oxcZAB07diQ7Oxt7e3utn0f1Imtubq5xDk+hUGh9MOvp6WnV19zcnOLiYo2FT+0zpXXh7u5OcXExOjo6Wu/wIEuPpsbNzU3rrHFD0dXVxcrKCn19fZKTk/Hy8tKaxDMyMvjzzz/rNJHz9PTUMhc+e/Ys7u7uGguArKysencfWxPN3Zb6+vq0b98eXV1dkpOT8fPz0zguEB8fz9atW5k9e7ZGKNCatJW2bGo6duxIXl4eZmZmWuOtPqWEs7MzhYWFFBQUSGn5+fkUFRU90Dy9tdLS/RPg999/p7S0VGsnRk1WVhZ6enpau/otia6uLkqlkosXL2JlZUVwcDAeHh44ODg0yrw3PT1dQ06kp6cjl8sxMTEhNzeX0tJSRo8eTdeuXXFycnpkq4ULFy4QGBhIQEAArq6uWFlZPTHru3bt2uHj48PevXspKyvTun7nzp0mGVcmJibI5XKNM/wqlYqMjAyNfObm5lpy/M8//5R+d3Z2xsDAgBs3bmjNC7UtJep6/sWLFzXSL1y40CrmBjHWG47ar4Tan0TN9PPnz9fpT+JhXL58WQov6uXlhaOjo5afh0dd/6qxs7NDX19fow9WVVWRnp4u9UH1/63msxuyPm1ORN9s3Tg5OZGenq6RVvNvS0tL5HI5+fn5dX6DPcuUlpaSk5PD66+/To8ePXB2dubevXuSIlD9XfM4476leCaUEjY2NmRkZFBQUCCZuTSE4cOHk5GRwcqVK7l69Sp5eXn88ccfGqFuGku3bt3Yu3cvly9f5urVqyxfvlzLsY6trS3nzp1DoVBImtSuXbty+/ZtYmJiyMvLY9++ffz2228PfV737t3p1KkTixcv5tSpUxQUFHDp0iV+/vnnBvu5aAp8fX3Jzs7WOHOXl5dHZmYmRUVFVFRUkJmZSWZmpqTpKykpISEhgezsbDIzM1m3bh3Hjh1jwoQJWuUnJibi4OBQp1AfMGAAhYWF/Pjjj2RnZ5OUlMSBAwcYOnSoRr60tDR8fHya9sWbgeZqy9zcXA4dOsT169fJyMhg6dKlZGVl8fbbb0t54uLiiI6O5p133sHR0VFSqt29e1ejjm2lLaHa0aC6vdQ/NT9kGsMLL7yAhYUFixcvJjU1lYKCAlJTU9mwYQPXr1+v857u3bvj6urKDz/8wOXLl7l8+TLff/89HTt21Dpf3BZoyf6pJikpCW9v73rjv6elpdGlS5cW2V2GatNi9djJzs5m7dq1lJWV0atXLxwcHCgsLOTw4cPk5+eTkJCg5U/iQRQVFfHjjz+Sm5vL8ePHiYuLY8iQIUD1UQUDAwP27NlDfn4+J0+eZOvWrY/0Dg4ODvz+++9cuXKFa9eu8cMPPzxRM+TJkyejUqmYPXs2x44dIzc3l5ycHBISEpg5c2aTjatXX32VuLg4jh8/Tm5uLj/++KOWAsLb25tTp05x4sQJcnNzWb9+veT8FqqtEocOHcrGjRvZt2+fNB4SEhJITEx84POHDRvGrl27OHLkCLm5uWzdupW0tDQt+dUSiLHecLp160Z6ejrp6eka65SuXbty9OhRKfJLY3BwcEClUrF7927J8W1tZ4E2NjaUl5dz9uxZSkpK6nW4XB9GRkYMGDCA6OhoTp48SXZ2NqtWrUKhUEhRKuzt7Wnfvj3btm0jNzeXM2fOaDnmfdKIvtm6GTx4MAcPHiQxMZHr168TExOjpewdOXKkFKEjNzeXa9eucfDgQWJiYlqo1q0DU1NTzMzMSEpKIi8vj9TUVFatWiVZ41hYWCCTyThz5kyd6/PWzDNxfGPo0KFERkby0UcfoVQqNUKCPghXV1ciIiLYsmUL4eHhVFVVYWtr+0jmomrGjRvHihUrCA8Px9LSkjFjxmhpc0NCQtiwYQPvvvsuVlZWREZG4uzszN///ndiYmKIiYmhV69evP7662zevPmBz9PR0eGTTz5hy5YtREVFUVxcjKWlJZ06dXpk3wuPgouLCx4eHhphIlesWCE5CAT4+OOPAc1whwcPHpTMlr28vAgPD9c6LnDv3j2Sk5MJDg6u89m2trZ88sknrF+/XgoPOnHiRI1woEqlkt9//51PP/206V66mWiutqyqqpImfz09Pbp168a8efM0zmTv3buXyspKli5dqlGnwMBAKZxtW2pLqF4YqNtLTe/evZkxY0ajyzI0NCQiIoKffvqJJUuWcPfuXeRyOd26ddOKUKJGR0eHjz/+mLVr1xIREQFUKyomTZrUJo9vtGT/hOrd8PPnz/PBBx/UW8fk5OQ6w+I9Kc6dO8eUKVOA6g9WR0dHPvzwQ+ljZdiwYfz4448olUp8fHwYNWoUq1evblDZffv2paqqin/961/o6OhoHEE0NzcnLCyMzZs3s3fvXlxcXBg3bpyWr4OGMH78eFasWMHcuXMxNTVl8ODBlJeXN7qcR8XOzo5FixYRExNDdHQ0hYWFmJmZ4erqytSpU5tsXA0dOhSFQsGKFSuAake3ffv21ZDb/fr1488//5Qicw0cOJDnn39e44No1KhRWFhYsGvXLlavXo2xsTFubm4MHz78gc9/9dVXuXfvHtHR0SgUChwdHZkxYwZubm4NfofmQoz1htOtWzcqKiq0jvZ27twZpVKJsbGxRjj6huDq6sqECROIjY1ly5YtdOrUiZCQEA353KlTJ1555RW+++47SktL6w0J+iDGjBkDwH/+8x/u3LlDx44d+fTTTyVHjfr6+vzjH/9g9erV/POf/8TNzY23336bf//73416TlMi+mbrpk+fPuTn57Nlyxbu37+Pv78/Q4YM4eDBg1KeoKAgDA0N2bVrF5s3b0Ymk+Hs7NzmQ80/Lrq6unz44YesW7eOGTNmYG9vT0hICN988w1QbW0/ceJEtm/fzrZt2+jSpUubCQmqoxIeQwRPiNOnT7Nu3Tq+/fbbVhc9YM+ePZw4cYLPPvuspavSIERbx/7NHQAAIABJREFUClozrbl/njx5ko0bN/L111836ty0QKBmzZo1ZGVltZmFXnMixrqgtSL6pkDQttALF1JV8ISwt7dHpVIhl8vr3TVuKTIzMxkwYICG48vWjGhLQWumNffPq1ev8tJLL2k5HBQIGsqpU6coKSnhpZdeaumqtDhirAtaK6JvCgRtC2EpIRAIBAKBQNBAhKWEQCAQCARNi1BKCAQCgUAgEAgEAoFAIGgRWtchK4FAIBAIBAKBQCAQCATPDEIp8QBu375NaGgoeXl5LV0VLZYsWcKuXbtauhoCgQCIjIxsUU/jgqcXIYealtbcnhs3bmTt2rUtXQ1BKyI8PJw1a9Y8MM+MGTP4+eefn1CNBM8qrXnubIuy6MCBA4SEhGikJSYm8u677zJq1ChpTNeV9rTyTIQEfVRiYmLo2bOnFL5p3bp1XLx4kaysLCwtLYmMjNS65+jRo8TExHD9+nXMzc0ZNGgQw4YN08hz5MgRYmNjuX79OsbGxnTv3p1x48ZhaWkJVHfU5cuXa5W9adMmZDIZAMHBwcydO5egoCBMTEya+tUFAkEdREZGaoSsUhMeHo6rq2sL1EjwtNNScigrK4uff/6Zq1evUlBQUGcowbYoh5qrPffs2cPevXspKCjA2tqaESNGEBgYKF0PDw/XCEeoxtnZmSVLlgAwfPhw3nvvPYYMGYKdnV1TvragFVFTjujp6WFqakqHDh3o3bs3L7/8Mvr6/39pPnPmzMeOznDgwAHWrFkjhbpsasLDw+nQoQOTJ09ulvIFrYOWmjvb4jfRw0K9BgYGEhoaSs+ePaW027dvs2bNGsaNG0dAQADGxsZ1pj0pIiMjsbGxeaJha4VSoh7u37/Pvn37mDVrlpSmUqkIDAzk2rVrnD17VuueU6dO8f333zNx4kR8fX3JyckhKioKmUwmxdW9cOECP/zwAyEhITz//PMoFArWrFnD999/z+effy6VZWhoyA8//KBRvnrwQXUMZjs7Ow4dOvTMx+wVCJ4k3bt357333tNIMzMze+DCsaKiQmOhKRA0hJaUQ/fv38fGxobevXuzZcuWOuvX1uRQc7VnQkIC0dHRTJ06FU9PTzIyMoiKisLU1BR/f3+g+uOyoqJCKre8vJyZM2fy17/+VUozNzenR48eJCQkaO2gCZ4u1HKkqqqKkpISzp8/z7Zt2zh8+DBz5szByMgIgHbt2rVwTQWClp07oe19E61cuVL6/Y8//iAqKkojTSaTST9qbt68SWVlJb169UIulwPV0exqpz3NiFVyPZw6dQqATp06SWmTJk0CIC4urs4BeOjQIXr16sXAgQMBsLOz47XXXiM2NpaBAweio6PDpUuXaN++PX/7298AsLW1ZdCgQXWabKp3rOrD39+f5OTkVjEABYJnBQMDA62xGRkZSWlpKbNnzwaqd46cnJwwNDTk4MGD2NrasnDhQrKzs9m4cSNpaWnIZDK8vb2ZMGHCQ8e64NmkJeWQh4cHHh4eQPUOWX20JTnUXO156NAhgoKC6Nu3r5Tn8uXLxMbGSgvr2h+Xhw8f5v79+/Tr108j3d/fn82bNwulxFNOTTliZWWFm5sbPXr0YNasWcTFxUm7k7WtEIqLi4mKiuLMmTNYWFgQHBz82HU5ffo0O3fuJCsrC6ge++PHj8fZ2VnKs337dvbt24dCocDU1BQfHx+mT59OZGQkqamppKamsnfvXgCWLVuGra3tY9dL0HpoyblTTVv6JqpZV3U42tr1r2nBVNMaZPr06QBMmzZNK23ZsmVUVVWxYcMG0tPTKSsrw9HRkZEjR9KrVy+p7LCwMPr378+tW7dITk7G2NiYwYMHS1Yqy5cvp6SkRFqzAlRVVREWFsaQIUOktUFNfvvtN7Zt28b169eRyWS4uLjw4YcfNun6VSgl6iEtLQ13d3d0dHQafE95eTkGBgYaaTKZjFu3bnHjxg1sbW3p3Lkzmzdv5sSJE/Tq1YvS0lKOHj2qYcIDoFQqmTZtGlVVVbi5uTFq1Cg6duyokcfDw4MdO3agVCo1tG0CgaDlOXz4MC+//DJffPEFKpWKoqIi5s6dS79+/QgJCaGyspLNmzezePFi5s2bh66ucPEj0KSl5VBDaEtyqLnas7y8XOvdZTIZGRkZ9VpJJSUl4evri7W1tUa6h4cHhYWF5OXlSWbSgmcDFxcXfH19+e233+o1mV6+fDk3btxgzpw5GBoasn79egoKCh7ruWVlZQwePBhXV1eUSiU7duxg0aJFfPvtt+jr63P8+HF27drFBx98gIuLC8XFxaSnpwMwceJErl+/jqOjI6NHjwaqLX4ETxctPXc+7d9Effr0wdLSkgULFrBgwQKsra0xMjLSSjM3N+fatWv4+vry1ltvIZPJOHr0KF9//TVff/01Tk5OUpm7d+9m5MiRDBs2jFOnTrFu3To6d+6Ml5cXL7/8Mp9//jlFRUWSBcbZs2dRKBS8+OKLWvVTKBQsXbqU0aNH07t3b8rKyqQ5oCkRq+B6uHHjRqNNZXx9fTlx4gRnzpyhqqqK3Nxc4uPjgep/KICXlxf/+Mc/+OGHHxg9ejR///vfUalUkhYMwNHRkXfffZePP/6YDz74AAMDA+bMmcP169c1nieXy6msrKSwsPAx31YgEDSU06dPExISIv0sWLCgzny2traMGzcOJycnnJ2dSUhIwNXVlbFjx+Ls7IyrqyvTp08nIyODK1euPOG3ELQFWlIONZS2JIeaqz19fHzYv38/GRkZqFQqLl++TFJSEpWVlZSWlmqVmZubS2pqKkFBQVrX1PW7ceNGY19P8BTg7OxMfn5+nddyc3M5deoUU6ZMoXPnznTs2JGwsDCUSuVjPTMgIICAgAAcHBxwdXVl2rRpFBQUkJGRAVSblVtaWtKjRw+sra157rnnpN1oExMT9PX1MTQ0xNLSEktLS6FgfwppybnzWfgmkslkmJmZAdVKPUtLS4yMjLTSdHV1cXNzY8CAAbi4uGBvb8+IESNwd3fn+PHjGmX26NGDQYMGYW9vz6uvvoq9vT3nzp0DqtcATk5OGj7S9u/fj7+/v6RUDAsLk5SjhYWFVFZWEhAQgK2tLS4uLgQFBTW5la+wlKiHurR3DyMoKIi8vDwWL15MZWWlZC6zbds2SbuYnZ3N2rVreeONN/Dx8aGoqIhNmzaxcuVKaUHo5eWFl5eXVG6nTp345z//ya+//iqZS8H/P0/1uAJJIBA0nC5dujB16lTpb5lMxubNm7Xyubu7a/x95coV0tLS6jTLzsvLk0zlBQI1LSmHGkpbkkPN1Z7BwcEoFArmzJmDSqXCwsKCwMBA4uLi6txZTEpKQi6X4+fnp3WtLbWnoOlRqVT17kbn5OSgo6OjIStsbGywsrJ6rGfm5eWxdetWMjIyKCkpoaqqCpVKxc2bN4FqpcUvv/zC9OnT8fHxwdfXF39/f61dcMHTS0vOneKbSJOysjK2b9/OH3/8gUKhoKKigvLyclxcXDTy1Xa+LpfLKS4ulv4OCgpi7969vPbaa9y+fZsTJ04wc+bMOp/p5uZG9+7dmTFjBj169KBHjx4EBAQ0uVWUUErUg5mZGbdv327UPTo6OowdO5bRo0ejUCgwNzeXtFJqT9oxMTF4eHhI53pcXV0xMjLi888/5+2336Z9+/Za5erq6vLcc89pheFR10+YygkETw5DQ8MGmVUbGhpq/K1SqejZsyfjxo3TymthYdFk9RM8PbQmOVQfbUkONVd7ymQypk2bxpQpUyguLkYul5OYmIixsbFWu1RUVHDw4EGCgoLqdI7bltpT0PRkZ2c/1B9DY0zoG8KiRYuwsrIiNDQUKysr9PT0+OijjyTHrNbW1ixdupTz589z9uxZNmzYwPbt25k/f77kkFPwdNMa5k41z/o30caNGyWLXQcHBwwNDVm2bJmGI2VAS77o6OigUqmkv1988UWio6O5cOECV69exdzcHB8fnzqfqaury2effUZ6ejpnzpxh3759/PTTT4SHh+Pm5tZk7yZsrOrBzc2NnJycR7pXV1cXKysr9PX1SU5OxsvLSxok9+/f1zJtU/9ds7PURKVS8eeff2qZyWRlZWFlZSWc5AkEbYCOHTuSnZ2NtbU19vb2Gj9PMsyToO3QmuRQfbQlOdRc7alGX1+f9u3bo6urS3JyMn5+flrt/Pvvv1NaWkr//v3rfE5WVhZ6enpau16Cp59r165x5swZAgIC6rzu5OSESqWSjlVA9dGKxzFXLy0tJScnh9dff50ePXrg7OzMvXv3qKys1Mgnk8nw8/NjwoQJLFy4kKysLC5evAhU9/uqqqpHroOg9dMa5k41z/o30YULFwgMDCQgIABXV1esrKzqPfL1INq1a8fzzz/Pvn372L9/P4GBgQ88eqWjo4OXlxdvvvkmCxcuRC6Xc/To0cd5FS2EpUQ9+Pr6Eh0dTWlpqXSmJy8vj7KyMoqKiqioqCAzMxOoPgOor69PSUkJx48fp2vXrlRUVLB//36OHTtGRESEVK6/vz9RUVEkJCRIZrPr16+nY8eOksOrbdu24enpiYODA/fu3eOXX37h2rVrhIaGatQxLS2tXq2WQCBoXQwcOJCkpCSWLl3K8OHDMTc3Jz8/n2PHjjFu3DihmBBo0ZJyqKKiguzsbKDaHFahUJCZmYmRkZGGpVBbkkPN1Z65ublkZGTg6enJnTt3iI+PJysri7CwMK06JCUl4e3tLe0U1iYtLY0uXbpoWVoJni7Ky8tRKBQaIUFjYmJwd3dn6NChdd7j6OiIr68vK1euZOrUqchkMtavX98gs3qVSiX1bTW6uro4OztjZmZGUlIS1tbWFBYWsnHjRo1d1gMHDlBZWYmnpydGRkYcPXoUPT09HBwcgOojJBkZGRQUFGBkZES7du2EX4mnjJacO8U3kSYODg78/vvv+Pv7o6+vz7Zt2x75yEpQUBALFiygsrKSGTNm1Jvv0qVLnDt3Dh8fHywtLbl69Sq3bt3SiNDTFAilRD24uLjg4eGhEV5mxYoVpKamSnk+/vhjQDP80cGDB9m4cSNQfQ4qPDxc4/zfSy+9xL1799izZw8bNmzAxMQEb29vxowZI+W5c+cOK1euRKFQYGJiQseOHYmIiNAoR6lU8vvvv/Ppp582XyMIBIImw8rKii+//JKffvqJBQsWoFQqsba2xsfHR5zNFdRJS8qhwsJCqWyA/Px8EhMT6dq1K+Hh4UDbk0PN1Z5VVVXEx8eTm5uLnp4e3bp1Y968eVpm+Pn5+Zw/f54PPvig3jomJyfXG3lB8PRw7tw5pkyZgq6uLqampnTo0IE333yTl19+uc5oLWqmTZtGVFQUERERmJubExwcTElJyUOfp1QqNcYzVJvkr1mzhg8//JB169YxY8YM7O3tCQkJ4ZtvvpHymZiYEBsby8aNG6msrMTZ2ZmZM2dK/Xvo0KFERkby0UcfoVQqRUjQp5CWnDvFN5Em48ePZ8WKFcydOxdTU1MGDx5MeXn5I5XVrVs32rdvj7W1db2KcqieAy5evMiePXu4c+cO7du354033qgzUsfjoKNqrK3mM8Tp06dZt24d3377bavT+u7Zs4cTJ07w2WeftXRVBAKBQNBMCDnUtLTm9jx58iQbN27k66+/rtPfhEAgELQUrXnubIuyqDWgVCqZOnUqkyZN4oUXXmjp6qAXrt7yEGhhb2+PSqVCLpdjamra0tXRIDMzkwEDBkhmVAKBQCB4+hByqGlpze159epVXnrpJekIjUAgELQWWvPc2RZlUUuiPjYWFxdHVlYWU6dObRWKJmEpIRAIBAKBQCAQCAQCwVNOQUEB06dPp3379rzzzjutxheHUEoIBAKBQCAQCAQCgUAgaBFa3lZDIBAIBAKBQCAQCAQCwTOJUEoIBAKBQCAQCAQCgUAgaBGEUkIgEDzz3L59m9DQUPLy8lq6KlosWbKEXbt2tXQ1BAKBQNAElJSUsHr1asLCwhg9ejShoaF88cUXnD17tqWrJhAInhBi3alN/cGQBQKB4BkhJiaGnj17Ym9vD8C6deu4ePEiWVlZWFpaEhkZqXXP0aNHiYmJ4fr165ibmzNo0CCGDRumkWfPnj3s3buXgoICrK2tGTFiBIGBgdL1Y8eOERsbS15eHpWVldjb2zNkyBBeeuklKU9wcDBz584lKCgIExOT5mkAgUAgEDwRvvnmG+7fv88777yDvb09xcXFpKamUlpa2tJVazIqKirQ1xefGAJBfTTXurOiooIdO3Zw6NAhioqKsLCwYOjQoQwePBiArKwsfv75Z65evUpBQQHBwcGMHDlSo4yWWneKGUMgEDzT3L9/n3379jFr1iwpTaVSERgYyLVr1+rcvTp16hTff/89EydOxNfXl5ycHKKiopDJZAwaNAiAhIQEoqOjmTp1Kp6enmRkZBAVFYWpqSn+/v4AmJmZMWLECJycnNDT0+PkyZOsWLECc3Nz/Pz8AHBxccHOzo5Dhw5JZQsEAoGg7XHnzh3S0tL47LPP6N69OwA2NjZ4eHhIecLCwhg4cKDGx0Z4eDgdOnRg8uTJUp5+/fqRn5/Pb7/9hqmpKSEhIfj4+LBq1Sr++OMP5HI5kydPljzrp6SkEBERwSeffMKWLVvIzs7mueee44MPPiA/P59169aRl5dHt27dCAsL0wivuH//fuLi4iQF+yuvvMLgwYOlMIIjR45k0qRJnD9/njNnzvDKK68wbty4Zm9PgaAt0lzrToClS5dy69Ytpk6dKik9lUqlxrNtbGzo3bs3W7ZsqbN+LbXuFEoJgUDwTHPq1CkAOnXqJKVNmjQJgLi4uDqFw6FDh+jVqxcDBw4EwM7Ojtdee43Y2FgGDhyIjo4Ohw4dIigoiL59+0p5Ll++TGxsrKSU8Pb21ih38ODBHDx4kAsXLkhKCQB/f3+Sk5OFUkIgEAjaMEZGRhgZGXHixAk6d+6MTCZ75LJ2797NW2+9xYgRI/jvf/9LZGQk3t7e9OnTh7feeouYmBh++OEHli9frvGcn3/+mQkTJmBiYsL333/P0qVLMTAwYMqUKejq6rJkyRK2bdsmycHExER+/vlnJk2ahLu7O9euXSMqKgp9fX0NmbR9+3befvttQkJC0NHRefRGEgiecppr3XnmzBnOnTvHDz/8gLm5OQC2trYa5Xh4eEhK0JiYmHrr2BLrTuFTQiAQPNOkpaXh7u7eqEVUeXk5BgYGGmkymYxbt25x48YNKU/tBadMJiMjI4OKigqtMlUqFefOnSM3N5cuXbpoXPPw8CAjI0ND2y0QCASCtoWenh7Tpk3j8OHDTJw4kU8//ZQNGzaQnp7e6LJ8fHwYOHAgDg4OjBw5kvLycuzs7AgMDMTe3p433niDkpISsrKyNO4bNWoUXbp0wdXVlVdeeYWLFy8yduxYPD09ee655wgMDCQlJUXKv2PHDsaOHUtAQAC2trb4+/vz2muvsXfvXo1y+/TpQ1BQEHZ2dlofQgKB4P/TXOvO//3vf3h4eBAfH88777zD+++/z9q1aykrK2t0HVti3SksJQQCwTPNjRs3kMvljbrH19eXH3/8kTNnztC9e3fy8vKIj48HQKFQYGtri4+PD/v37+f555/nueee48qVKyQlJVFZWUlpaan0zLt37zJ16lQqKirQ1dVl8uTJ9OzZU+N5crmcyspKCgsLpfOHAoFAIGh7BAQE4Ofnx4ULF7h06RKnT58mPj5esnpoKK6urtLvRkZGGBoa4uLiIqVZWloCUFxcXO99FhYWAFr3qe8pKSnh1q1brFy5klWrVkl5qqqqUKlUGuW6u7s3uO4CwbNMc6078/PzuXDhAvr6+syYMYM7d+6wbt06ioqKmDFjRqOe1xLrTqGUEAgEzzR1WTQ8jKCgIPLy8li8eDGVlZUYGxszePBgtm3bJmm+g4ODUSgUzJkzB5VKhYWFBYGBgcTFxWlox42MjPjqq68oKyvj3LlzrF+/HhsbG+m8MSDVT1hKCAQCQdtHJpPRo0cPevToQXBwMCtWrGDbtm0MGzaszt3TyspKrTQ9PT2ttLqcS9ZWHtS8T/2s2vep76mqqgIgNDRUw9S8LoyMjB54XSAQVNNc6071uP3ggw8kB5WTJk1i/vz5KBQKSVHZEFpi3SmUEgKB4JnGzMyM27dvN+oeHR0dxo4dy+jRo1EoFJibm3Pu3Dmg+pwfVE/o06ZNY8qUKRQXFyOXy0lMTMTY2Fg66wegq6sraaHd3NzIyckhJiZGQymhrl/N+wQCgUDwdODs7ExVVRVKpRJzc3OKioqka0qlkpycHNzc3J54vSwtLZHL5eTn52tEjhIIBI9Oc607LS0tsbKy0oiY4eTkBMDNmzcbpZRoiXWnUEoIBIJnGjc3Nw4ePPhI9+rq6mJlZQVAcnIyXl5eWhO4vr4+7du3l/L4+flJHsvroqqqivLyco20rKwsrKysGiVQBAKBQNC6KC0tZcmSJfTr1w9XV1eMjY0lB8je3t6YmJjg7e3N/v378ff3x9zcnJ07d9ZpKfGkGDlyJGvXrsXExAQ/Pz8qKiq4evUqhYWFvP766y1WL4GgrdJc687OnTtz/PhxysrKJMul69evA9VRfhpDS6w7hVJCIBA80/j6+hIdHU1paakUAi0vL4+ysjKKioqoqKggMzMTqN7N0tfXp6SkhOPHj9O1a1cqKirYv38/x44dIyIiQio3NzeXjIwMPD09uXPnDvHx8WRlZREWFibl2blzJx4eHtjZ2VFeXs6pU6ckB2g1SUtLk8K6CQQCgaBtYmRkhKenJ7/++it5eXmUl5djZWVF3759eeONNwB47bXXKCgoYPHixRgZGTFixAgNy4knTVBQEIaGhuzatYvNmzcjk8lwdnYW0aAEgkekudadffv2ZceOHf+vvbsLiWrr4zj+05mmxJy0RHsRidCJNGoyg6JCagKngoqoiLKLBAus6EbqwgKDbopIEIu80UDspgsxvDBpEoUpCUl7QQ0mEF8mU6YmX0BzdJ6LcHhsPD3P6djZneP3A3Oz9lpr//e+GJf/+e+1dffuXR09elSjo6O6f/++tm7dGto/JhAIqLe3V9K3Kiy/36+uri4tWrRoxt4RRqw7I4LfP2wGAPNMYWGhdu7cGVpkFRUVqb29PaxfaWmpEhISNDQ0pBs3bqi7u1uSZLPZdPz4caWmpob69vb2qqSkRF6vVyaTSenp6crJydHKlStDfR48eKDm5mb5fD5ZLBatWrVKTqcz9BpR6dsfjby8PBUWFspms/2qWwAAAIC/wa9Yd0rffhArLy9XZ2enoqOjtWXLFp08eVJRUVGSpIGBAZ0/fz7sPGlpaSoqKpJk3LqTpASAea+trU0VFRUqLi7+4aMVRqirq1NLS4uuXLlidCgAAAD4i1h3hvu97gIAGMButys7O1s+n8/oUMKYzWbl5uYaHQYAAADmAOvOcFRKAAAAAAAAQ1ApAQAAAAAADEFSAgAAAAAAGIKkBAAAAGCgkZER5eXlqb+/3+hQwlRWVqq8vNzoMAD8i5mNDgAAAACYz6qrq7Vp0yYtX75cklRRUaF3796pp6dHsbGxunPnTtiYZ8+eqbq6Wh8+fJDVapXT6dSBAwdm9Kmrq9Pjx481MDCg+Ph4HT58WFlZWaHjf/QqwqSkJN2+fVuSdPDgQV24cEH79+9XYmLiXF42AEgiKQEAAAAYZnx8XE+fPtXly5dDbcFgUFlZWeru7tbr16/DxrS2tqqkpESnT5+W3W5XX1+fysrKZLFY5HQ6JUn19fWqqqrS2bNnlZqaKo/Ho7KyMkVHRyszM1OSVFBQoEAgEJp3YmJCBQUF2rZtW6jNarVqw4YNqq+v16lTp37VbQAwj/H4BgAAAGCQ1tZWSdLatWtDbbm5udq7d69WrFgx65impiZt3rxZ2dnZSkxMVEZGhg4dOqSamhpNv1ivqalJDodDO3bsUGJiorZv3649e/aopqYmNM/ixYsVGxsb+nR2dmp8fFy7du2acb7MzEy53e65vnQAkERSAgAAADBMR0eH1qxZo4iIiP97zMTEhBYsWDCjzWKxyOfzaXBwMNTHYrGE9fF4PDOqI/6by+WS3W5XfHz8jPaUlBR9+vTpt9zzAsA/H0kJAAAAwCCDg4OKi4v7U2PsdrtaWlr06tUrTU1Nyev1qra2VpLk9/slSRs3blRDQ4M8Ho+CwaDev38vl8ulyclJDQ8Ph83p9XrV3t4uh8MRdmw6vumEBwDMJfaUAAAAAAwyW0XD/+JwONTf36+bN29qcnJSUVFR2rdvnx4+fBiquDhy5Ij8fr+uXr2qYDCoJUuWKCsrS48ePZq1KsPlcikuLk4ZGRlhx6bj+/r1609cIQD8GEkJAAAAwCAxMTEaGRn5U2MiIiKUk5OjEydOyO/3y2q16s2bN5IUekOGxWJRfn6+zpw5oy9fviguLk5PnjxRVFSUrFbrjPkCgYAaGxvlcDhkMpnCzjcd3/fjAGAu8PgGAAAAYJDVq1err6/vp8ZGRkZq6dKlMpvNcrvdstlsYYkDs9msZcuWKTIyUm63WxkZGYqMnPkvwIsXLzQ8PKzdu3fPep6enh6ZTCYlJyf/VJwA8CNUSgAAAAAGsdvtqqqq0vDwsGJiYiRJ/f39Ghsb0+fPnxUIBNTV1SVJSkpKktls1tDQkJqbm5WWlqZAIKCGhgY9f/5c165dC83r9Xrl8XiUmpqq0dFR1dbWqqenR+fOnQuLweVyaf369aEqi+91dHRo3bp1Wrhw4dzfAADzHkkJAAAAwCDJyclKSUmR2+2W0+mUJN27d0/t7e2hPpcuXZIklZaWKiEhQZLU2NioyspKSZLNZlNRUZFSUlJCY6amplRbWyuv1yuTyaT09HRdv349NH7ax48f9fbtW128ePEPY3S73Tp27NjcXDAAfCciOP0yYwAAAAB/u7a2NlVUVKi4uDjs0QqjvXz5UpWVlbp169as+011BXLuAAAAa0lEQVQAwF/1e33rAQAAAPOM3W5Xdna2fD6f0aGEGRsbU35+PgkJAL8MlRIAAAAAAMAQVEoAAAAAAABDkJQAAAAAAACGICkBAAAAAAAMQVICAAAAAAAYgqQEAAAAAAAwBEkJAAAAAABgiP8Ay6vDS8y6EY0AAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":690},"id":"VCnP_uOT-ApN","executionInfo":{"status":"ok","timestamp":1633623148765,"user_tz":-330,"elapsed":1506,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"af46a636-923f-4a9b-81e1-418a70319878"},"source":["plot_components(components_df, 'fc0', ascending=True)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABCUAAAL1CAYAAADw5l6HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZQV5Z038G9DAyICzSIYZYuyqLxRCS4oGFQQJyqoqGFcIkiirzGOS0YzGscBjVGJM1l8jTETFUmicRsVMYkLKFFj1GhU3FAScQETiUATXJCt3j883WPbDULTWmg+n3NyYj/1VNVTv1t1D/d7q55bURRFEQAAAICPWbOyBwAAAAD8YxJKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClEEoA/IMYN25cKioq8tJLL5U9lA3WmGOZOXNmKioqMnHixA3e/0svvZSKioqMGzdug7f1j2TOnDk55JBDssUWW6SioiJVVVVlDwma3KfpvRbg4yCUAGgCFRUVqaioSLNmzfLnP/95jf323nvv2r5XX331xzfAfwCCgqa1ISFOQ6/DqlWrcvDBB+fXv/51DjzwwEyYMCFnnnlmo8d3++23Z6+99kr79u2z2WabZbfddsuUKVMa7FvzIZFPD9c7wKeHUAKgiVRWVqYoilx55ZUNLp8zZ05mzpyZysrKj3lk77nwwgvz3HPPZauttipl/03p03Qs/yjmzp2bZ599NmPHjs0VV1yRiRMnNjqUuPTSSzNy5Mg8/fTTOfroo3Pcccfltddey7hx43L66ac38cgBgI+SUAKgiXTt2jU777xzJk+enJUrV9ZbfsUVVyRJRo4c+XEPLUnymc98Jttuu21atGhRyv6b0qfpWP5RvPbaa0mSLbfccoO289JLL+X0009Px44d8+ijj+ZHP/pRvv/972fWrFnZZptt8l//9V/5/e9/3xRDBgA+BkIJgCZ03HHH5a9//Wtuv/32Ou0rVqzI1VdfnT322CPbb7/9GtefM2dOjjnmmGy11VZp2bJlttxyyxxzzDGZM2dOnX4nnHBCKioqMnXq1Aa38/DDD6eioiKHHXZYbdvannN++OGHc9hhh2WLLbZIy5Yt07179/zf//t/az9Ivt+LL76Y448/Pr17907r1q3TsWPHfO5zn8sJJ5yQhQsXrq08Sd77UNrQHQ49e/ZMRUVFvv3tb9dp/81vfpOKior8x3/8xxqPZeLEifnsZz+bJJkyZUrtIzJrekzmiSeeyAEHHJCqqqpsuummGTp0aB588MEPHfu6+Mtf/pKvf/3r6dWrV1q2bJnNN988o0ePzmOPPVan35133pmKioqcffbZddrvvffe2rG/+uqrdZaNGTMmFRUVefHFF+u0z549O+PGjUv37t3TsmXLdO3aNUceeWSef/75euN7/fXXc/rpp6dfv35p06ZNqqqq0q9fv4wbN652u+PGjcvee++dJDn33HPr1HPmzJnrXZOKiooMHTq03vbe/2jIqlWrcvnll2fw4MFp3759Wrdund69e+erX/1qnfP/qquuyrvvvpuTTjopvXr1qm3v0KFDvvWtbyVJLr/88vUe4wfNmzcvJ598cvr06VN7nu+66671zs8keeyxx3LooYemS5cuadWqVXr27JkTTzwxf/nLX+r1rTl3586dm0svvTTbb799Ntlkk/Tq1SsXXHBBiqJIktx4443Zdddd06ZNm3Tp0iUnnXRS3nnnnXrbq6ioyF577ZXXXnstX/7yl9OlS5e0bt06AwcOzLXXXtvgsa1evTqXX355dtlll2y22WZp06ZNdtlll/z4xz/O6tWr17iPN954I8cff3w+85nPpFWrVunfv38mT568xhreeeed2X///dO5c+e0atUq22yzTc4444xUV1fX69urV6/06tUrb731Vs4444z06NEjrVq1Su/evTNp0qTauiTrf73XWLZsWaqqqtKlS5cGg+Mk+drXvpaKioo67+G33nprjj766PTt2zdt2rRJmzZtMnDgwFxyySUN1qshH/Y4VM3xN+SXv/xl9t5771RVVWWTTTbJdtttl/PPPz/vvvtuvb73339/Ro4cmW7duqVVq1bZYostMmjQoJx77rnrNE6AMpRzDzHAp9QRRxyRb3zjG7niiity8MEH17bfdtttWbBgQSZNmpQ//elPDa77hz/8IcOHD8/SpUszatSobL/99pk9e3Z+8YtfZOrUqZk+fXp22WWXJMnYsWPzk5/8JD/72c9y0EEH1dtWzbP16/K89VVXXZXjjz8+rVq1yqhRo9K9e/fMmTMnV1xxRaZNm5aHHnooPXr0SPLeB+5ddtklf//737P//vvn0EMPzbJlyzJ37tz8/Oc/z0knnZROnTqtdX/77LNPrrnmmsyePTvbbrttkuRPf/pTXnnllSTJjBkzcs4559T2nzFjRpJk2LBha9zmXnvtlerq6vzwhz/MjjvuWKf2O+20U52+jz76aL773e9m9913z1e/+tW88sor+Z//+Z8MGzYsTzzxRPr16/ehNVuTuXPnZsiQIXnttdeyzz775Igjjsirr76aG2+8Mb/61a/yP//zPznwwAOTJHvuuWdatmyZGTNm5Dvf+U69463575rXsCiK3HvvvenVq1e23nrr2j533HFHRo8enRUrVmTkyJHp3bt35s2bl5tvvjm/+tWvcu+99+bzn/98kuTtt9/O4MGD8+c//zn77rtvRo4cmaIo8vLLL2fq1Kk57LDDsvXWW9fWb8qUKRk6dGj22muv2v2t6YPT2kyYMCEvvfRSve3V/P/y5ctz4IEH5u6770737t1z5JFHpl27dnnppZdyyy23ZMiQIenTp0+S5J577kmS/NM//VO9/Xzxi1+s06exHn300ey3335ZtGhRvvCFL2T06NF5++238+yzz2bixIl1zs/bb789hx56aIqiyGGHHZaePXvmsccey49//ONMnTo1DzzwQO0H6Pc7/fTTM3PmzIwcOTIjRozIbbfdlrPPPjvLly9Px44dc+aZZ+bggw/Onnvumbvvvjs/+tGPsmrVqvz4xz+ut63Fixdnjz32SFVVVY499thUV1fnhhtuyFFHHZX58+fnjDPOqNP/y1/+cq699tp07949X/3qV1NRUZFbbrklJ554Yh544IFcc8019fZRXV2dwYMHp2XLljnssMPy7rvv5sYbb8z48ePTrFmzjB07tk7/c889NxMnTkzHjh1z4IEHpkuXLpk1a1b+8z//M7/+9a/z+9//Pu3atauzzooVK7Lffvvltddeyxe/+MVUVlbm1ltvzZlnnplly5ZlwoQJSdbven+/TTbZJGPGjMl///d/5ze/+U29u9befffdXH/99enatWud8+vMM89Ms2bNsttuu2WrrbbKkiVLcs899+SUU07JH/7wh/z85z9f4z431Pjx4zN58uR069Ythx56aKqqqvLQQw/lnHPOyYwZM3L33XfXPhJ4xx135IADDki7du0yatSobLXVVlm0aFGee+65XHbZZbX1A9joFABssCTFVlttVRRFUXzlK18pmjdvXrz66qu1y/fbb7+iXbt2xVtvvVWcffbZRZJi8uTJtctXr15dbLvttkWS4he/+EWdbV933XVFkqJfv37FqlWratv79u1btGzZsli4cGGd/suWLSs6dOhQdOnSpVixYkVt+9ixY4skxdy5c2vbnn/++aJFixbFNttsU8ybN6/OdqZPn140a9asOPjgg2vbLrnkkiJJ8YMf/KBeDd58883i7bff/tBaXXnllUWS4tJLL61tu/zyy4skxb777lu0bNmyeOutt2qX7bTTTkXr1q2Ld999d63HMnfu3CJJMXbs2Ab3e++99xZJ6tX+/fv/2te+9qHjX9u+RowYUSQpzj///Drtv/vd74rmzZsXHTt2LJYuXVrbvueeexbNmzcvqqura9sGDRpUDBgwoOjUqVNx9NFH17Y/8cQTRZJi/PjxtW2LFi0qqqqqik6dOhXPPPNMnX0+9dRTRZs2bYoBAwbUtt12221FkuLUU0+td0zvvvtu8fe//73275p6TZgwYZ1q8mHWtr2zzjqrSFKMHDmyWLZsWZ1ly5YtKxYsWFD7d+fOnYskxRtvvNHgftq0aVMkqXMOrY9333236NWrV5GkuOaaa+otf/91vXTp0qJjx45Fs2bNivvuu69Ov4suuqj2nH6/mnO3Z8+eda65xYsXF506dSo23XTTonPnzsWzzz5bu2zZsmXFdtttV7Rs2bJ4/fXX62yv5pw+/PDD67w/vPjii0WHDh2KFi1aFH/+859r26+99toiSTFgwIA65+Kbb75ZDBw4sMHjrtnHV77ylWLlypW17c8880zRvHnzYrvttqvT/5577imSFLvvvnuxePHiOssmT57c4DnYs2fPIknxxS9+sc77yOuvv160b9++aN++fbF8+fLa9g+73tfkwQcfLJIUhx56aL1lN9xwQ5Gk+MY3vlGn/U9/+lO9vqtWrSqOOeaYIknx0EMP1VnW0PvTh11PPXv2LHr27FmnraZWhxxySL331gkTJtR7Lx49enSRpHjiiSfqbf9vf/tbg/sF2Bh4fAOgiR133HFZtWpVrrrqqiTJyy+/nLvvvjtHHXVUNt100wbXefDBBzN79uzsvvvuOeqoo+osGzNmTIYMGZLnn38+DzzwQG372LFjs3z58vzyl7+s03/atGlZvHhxjjrqqA+dVPPHP/5xVqxYkR/+8If1HqkYNmxYRo0alWnTpmXp0qV1lrVu3brettq0adNg+wfV3PHwwTsCunbtmpNPPjnLly+vPc6FCxfmySefzJAhQ9KyZcsP3fa6GDx4cL07SMaPH5/Kyso88sgjjd7uvHnzctddd6VHjx755je/WWfZHnvskSOOOCKLFi3KzTffXNs+bNiwrFq1Kr/97W+TJEuXLs2jjz6afffdN3vvvXedb/wbumPkZz/7Waqrq3PuuefWeyzo//yf/5Pjjjsujz/+eJ599tk6yxp6nVq2bJm2bds28ugbb9WqVbnsssvSunXrXH755WnVqlWd5a1atcrmm29e+/eSJUuSJO3bt29wezXtNf3W17Rp0/LSSy9l1KhROfLII+st79atW+1/T506NYsWLcqYMWOy55571un3r//6r+nVq1fuvvvu2ruA3u+cc86pc81VVVVl1KhRefvtt/O1r30t2223Xe2yVq1aZcyYMVm+fHmee+65ettq3rx5Jk2alGbN/vefdZ/97Gdz8sknZ8WKFXW+ya95X7rooouy2Wab1ba3adMmkyZNSvK/89+836abbprvfe97ad68eW3b9ttvn8GDB+e5557Lm2++Wdt+ySWXJEl++tOf1vvZ13HjxmWnnXZq8G6MmnXff3526dIlBx10UJYsWdLg40jra/fdd0/fvn0zbdq0LFq0qM6ymjvMPnjXxzbbbFNvO82aNcspp5yS5L3HVD4KP/zhD1NZWZmrrrqq3jV7zjnnpFOnTg3WsaHru3Pnzh/JGAGagsc3AJrYbrvtls997nO56qqr8u///u+54oorsnr16hx33HFrXOePf/xjkvcebWjIPvvskwceeCCPP/54vvCFLyRJjjnmmJxzzjmZMmVKvv71r9f2XZ9HN2omBPztb3+bP/zhD/WWL1iwIKtWrcoLL7yQgQMHZtSoUfnWt76Vr3/967nzzjuz3377ZfDgwdl+++3X+ScXe/bsma233jozZ87M6tWra+cpGD58eIYOHZrKysrMmDEjI0aMyL333puiKNZYl8bYeeed67W1aNEiXbt2zeLFixu93ccffzzJe49lNDQB5z777JNf/OIXefzxx3PMMcfUtk2cODEzZszIqFGj8tvf/jYrV67MsGHD0qtXr9x000157rnnst1229UGFO+vRc3r9+STTzb4rPoLL7yQJHnuueey/fbbZ+jQodlqq61y0UUX5Y9//GP233//DB48ODvttFOdD5sfp9mzZ2fJkiXZbbfdNngSzKbw0EMPJfnfR0HWZm3XbWVlZb7whS/kpZdeyuOPP177CFSNhs7DmuMfOHBgvWU1Aca8efPqLevRo0eDj4jstddeOffcc2vPzZoxN2vWrM4jOTWGDh2a5s2b1+lfo0+fPvUet0iS7t27J3nvEZKakOP3v/99WrRokRtvvDE33nhjvXWWL1+ev/3tb1m4cGGdx73at2+f3r17r3UfTWHs2LE5++yzc9111+XEE09M8t5cK3feeWcGDBiQHXbYoU7/hQsX5uKLL86vf/3rvPjii3nrrbfqLJ8/f36TjOv93n777Tz55JPp3LlzfvCDHzTYp1WrVnVCqqOOOio333xzdtttt4wZMyZ77713Bg8eXCdIA9gYCSUAPgLHHXdcTj755PzmN7/J5MmTM3DgwAwYMGCN/Wu+1f3MZz7T4PKa9vdPENetW7cMGzYsd999d+0H1wULFuSOO+7ITjvtVO8f1g2pmZjy4osvXmu/mm9Be/bsmUceeSQTJ07MHXfcUfutf/fu3XP66afn5JNP/tB9Ju992//Tn/40f/zjH9OiRYv87W9/y7Bhw9K2bdvssssutXcFrMt8Euvrg9/c1qisrMyqVasavd3GvIaDBg1KmzZt6hxvy5YtM2TIkNq5G2bMmJE+ffrkvvvuy/bbb58tttiidv2a1++nP/3pWsdW8/q1a9cuDz30UCZMmJDbbrut9hvezp0758QTT8y///u/f+y/aFJTj3X9edf27dvnjTfeyJIlSxqcv+TD7qRoyvE05jWv0dD4au5sWtuyFStW1FvWtWvXBvdfc668/66RJUuWpGPHjg3eeVRZWZnOnTtnwYIF9Zat7bpJUufaWbhwYVauXPmhkyu++eabdV7D9dnHhnh/oFsTSlxzzTVZuXJlvbskqqurs8suu2Tu3LnZddddc8wxx6Rjx46prKysndeioQknN9TixYtTFEX+9re/rfMklaNHj87tt9+e//qv/8pVV12Vn/zkJ0neC7kuvPDC7Lvvvk0+ToCm4PENgI/Al7/85bRu3TonnHBC5s+fn+OPP36t/Ws+hPz1r39tcHnNLP4f/LBS8w/omrsj1vQP6w/b75IlS1IUxRr/V/PLCUmy3Xbb5frrr8/ChQvz6KOP5qKLLsrq1atzyimn5Morr1yn/dZ8szx9+vR6wcM+++yTxx9/PIsWLcqMGTPSvn372okaN2aNeQ1btGiRIUOG5Jlnnslf//rXzJgxI7vvvns23XTT9O3bN926dcv06dPzyCOPZOnSpfW+ka/Z1pNPPrnW1+/950O3bt1y5ZVXZsGCBXn66adzySWXpFOnTjnvvPNy3nnnNWlN1kXNB9F1/ba5ZiLSmrtA3u8vf/lL3nrrrXTr1m2Nj0o15Xgae902tddff73B9ppxvX//7du3z6JFixoMN1auXJk33nijwTsi1kf79u3ToUOHtZ6TRVGkZ8+eG7SfxurWrVv22WefPPLII5k9e3aS995DW7RoUe+RnSuuuCJz587NhAkT8vDDD+eyyy7L+eefn4kTJ2bMmDHrvM+aR2vW9KsfHwyual6zAQMGfGgd3++AAw7IPffck8WLF2fGjBk57bTT8swzz+TAAw+s9xgXwMZCKAHwEaiqqsphhx2WefPmpU2bNjniiCPW2r/mLoo1/dzivffemyT1PpyPHj067dq1yy9+8YusXr06U6ZMSWVlZYPPwjdk0KBBSd77Gbn1VVlZmYEDB+bf/u3faue1uPXWW9dp3X322ScVFRWZMWNG7rnnnmy99da1dwYMGzYsq1evzs9+9rPMmTMne+211zo9WlDTp6m+TV1fNa/hAw880OAHjzW9hjVhzC9/+cs8/fTTde4K2WeffTJz5szcfffddfrW2JDXr6KiIv3798+//Mu/1G7//a/fx1XPbbfdNlVVVZk1a1aDP0H7QTXBzB133FFv2W9+85s6fRqjpqY121qbtV23K1eurH1dPupQ7ZVXXmnwp35rxvX+u7QGDBiQ1atX57777qvX/7777suqVas2eLyDBg3K4sWL88wzz2zQdtZmQ8/PmsfbpkyZkieeeCKzZs3KF7/4xTrzlySp/bWkQw89tN42auaCWRcdOnRIkno/81uzjw/OgbLZZpulf//+eeaZZ+rNfbEu2rRpk3322Sff+9738q1vfSvLly9fp3MaoAxCCYCPyPnnn59bbrkld95554dOIDh48OD069cvDzzwQG666aY6y2666abcf//96du3b4YMGVJnWevWrfOlL30p8+fPz/e///08+eST2X///dOlS5d1GuNJJ52UFi1a5LTTTmvwm+fly5fX+cD72GOPNTiBYM03tev67XSXLl3Sv3///O53v8t9991X58P2HnvskU022SQXXnhhknX/gNmhQ4dUVFQ0OKngx6Fbt27Zd99989JLL9V7Bvzhhx/Otddemw4dOuSQQw6ps6zm+C666KIURVEvlFiyZEkuu+yyBucBOPbYY1NVVZVzzz23wUk6V69eXecD8zPPPNPgt+oNvX41t9V/1PVs3rx5TjzxxLzzzjs54YQT6t0KXzP/QI1jjz02rVq1yqWXXlrng/jixYtzwQUXJElOOOGERo9n5MiR6dWrV2677bZ6k8gmded0OPjgg9OxY8f88pe/rJ2LosYPfvCDzJ07N8OHD683n0RTW7VqVf7t3/4tq1evrm2bO3duLrnkklRWVuboo4+ubR8/fnyS5Kyzzsrbb79d2/7222/nzDPPTJJ85Stf2aDxnHbaaUnee4ytoaDprbfeqlev9bWh1/v7A92rr746ScPz8NSEpR8Mnh5//PHa96h1se2226Zdu3aZOnVqncdj3nnnnTU+9vaNb3wjy5cvz/jx4xt8BGjx4sW185ok74VKDQWi6/v+DPBxM6cEwEekR48e6/xhpKKiIlOmTMm+++6bMWPG5KCDDsq2226b559/Prfeemvatm2bn/3sZ3Vm168xduzYXHHFFTnrrLNq/15X2267ba666qqMHz8+/fv3zz/90z+lb9++WbFiRV555ZXcf//92XzzzWtvcf75z3+en/zkJxkyZEi22WabdOjQIX/+858zbdq0tGrVKqeeeuo673vYsGF5+umna/+7RqtWrTJ48OD1nk9is802y2677Zb7778/Rx11VPr27ZvmzZtn1KhR6zS/RlO4/PLLM3jw4Jxxxhm56667svPOO+fVV1/NjTfemGbNmmXy5Mn1AqoBAwakQ4cOWbBgQdq2bZtdd921dlnNsS9YsCA777xzvWfuO3XqlJtuuimHHHJIBg0alGHDhqV///6pqKjIq6++mt///vdZuHBhli1bliS5++67c8YZZ9T+AkGXLl0yb968TJ06Nc2aNcsZZ5xRu+1+/fplq622ynXXXZcWLVqkZ8+eqaioyJe//OUmv+2+5tb4adOmpW/fvjnwwAPTtm3bvPrqq7nrrrty8cUX135g/OxnP5uLL744J598cnbeeeeMGTMmLVu2zE033ZR58+blX//1X7P77rs3eiwtW7bMjTfemBEjRuTII4/MT37ykwwaNCjLli3Lc889lxkzZtR+8Ntss81y1VVX5fDDD8/QoUNz+OGHp0ePHnnsscdy1113ZYsttqh9rv+jtMMOO+Thhx/OwIEDM2LEiFRXV+eGG25IdXV1vvvd79b59YgjjzwyU6dOzQ033JD+/fvn4IMPTkVFRW699dbMnTs3Y8aMqfcLQOtr2LBhueiii3LWWWelT58+2X///fPZz342b775Zl5++eX89re/zZAhQxq822Vdbej13rp16xx++OG58sorc9lll6VTp0454IAD6vU75phjcvHFF+fUU0/Nvffemz59+mTOnDm5/fbbM3r06Fx//fXrNN4WLVrklFNOybe//e0MGDAghxxySFauXJm77747W265ZYOTvI4fPz6PPfZYLrvssmyzzTbZb7/90qNHjyxatChz587Nfffdl2OPPTaXX355kuTkk0/O/PnzM3jw4PTq1SstW7bMY489lnvuuSc9e/bMP//zP6/TWAE+dh/5j44C/ANIUmy11Vbr1Pfss88ukhSTJ0+ut2z27NnF0UcfXWyxxRZFZWVlscUWWxRHHXVUMXv27LVus3fv3kWSomPHjsW7777bYJ+xY8cWSYq5c+fWWzZr1qxi7NixRY8ePYqWLVsWHTp0KPr3718cf/zxxYwZM2r7PfTQQ8UJJ5xQ7LDDDkWHDh2KTTbZpNhmm22KcePGFU899dQ6HX+N2267rUhSVFRUFK+//nqdZRdccEGRpOjatet6HcucOXOKAw88sOjYsWNRUVFRp8733ntvkaSYMGFCg9vs2bNn0bNnz3Ua+9y5c4skxdixY+stmzdvXnHCCScUPXr0KFq0aFF06tSpOOigg4pHHnlkjdsbPXp0kaTYf//96y3r27dvkaT45je/udbxfP3rXy969+5dtGrVqmjbtm3Rr1+/4uijjy5uueWW2n7PPvtscdpppxUDBw4sOnfuXLRs2bLo2bNnceihhxa/+93v6m33kUceKfbZZ5+iXbt2tfW89957116cNfiw+q9YsaL4f//v/xW77LJL0aZNm2LTTTctevfuXRx33HHFnDlz6vW/7bbbii984QvFZpttVmy66abFzjvvXFx99dWNGltDXn755eJrX/ta0atXr6JFixZFx44di1133bX4zne+U6/vI488Uhx88MFF586dixYtWhTdu3cvTjjhhGL+/Pn1+q7tOpwwYcIaazx58uQG3zeSFEOHDi3mz59fHHXUUcXmm29etGrVqhgwYEBxzTXXNHhsq1atKn70ox8VAwcOLFq3bl20bt26+PznP19ceumlxapVq+r1r9lHQ9Z2PPfff39x+OGHF5/5zGeKFi1aFJ07dy523HHH4rTTTiv+8Ic/1Om7tutvTXVZ2/W+Lu6///4iSZGkOOmkk9bY75lnnilGjhxZbL755sWmm25afP7zny9++tOfrvF9YE01Wb16dXHhhRcWW2+9de15csYZZxRvvfXWWo9/2rRpxQEHHFBsvvnmRYsWLYquXbsWu+yyS3H22WcXzz33XG2/66+/vvjnf/7nonfv3kWbNm2Ktm3bFv3798mFA2YAACAASURBVC++9a1vFQsWLFjnugB83CqK4gMz5AAA8IlQUVGRoUOHrnE+GgDY2JlTAgAAACiFUAIAAAAohVACAAAAKIVf3wAA+IQyNRgAn3TulAAAAABKIZQAAAAASvGpenzjtddeK3sIH6pz58554403yh7Gp4Z6Nh21bFrq2bTUs+moZdNSz6alnk1HLZuWejYt9Wxan4R6brnllmtc5k4JAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBSVZQ/gH90ll1yy1uUnn3zyxzSSTwf1bDofVstEPdeHc7NpqWfTca03LfVsWq71pqWeTce13rScm03rk1ZPd0oAAAAApRBKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClqNyQle+8887cdtttqa6uTrdu3TJu3Lhst912a+z/7LPPZsqUKZk3b146dOiQUaNGZcSIEbXLb7nlljzyyCN57bXXUllZmT59+uTII49Mjx49NmSYAAAAwEao0XdKPPjgg7n66qtzyCGHZNKkSenXr18uuOCCvPHGGw32X7BgQS688ML069cvkyZNysEHH5zJkyfnoYcequ3z7LPPZsSIEfn2t7+dCRMmpHnz5vn2t7+dN998s7HDBAAAADZSjQ4lbr/99gwdOjTDhw9Pt27dMn78+HTo0CF33XVXg/3vuuuudOjQIePHj0+3bt0yfPjwDB06NNOmTavtc/bZZ2fvvfdOjx490qNHj/zLv/xL/v73v2f27NmNHSYAAACwkWpUKLFy5cq8+OKL2XHHHeu077DDDnn++ecbXGfOnDnZYYcd6rTtuOOOefHFF7Ny5coG13nnnXdSFEU222yzxgwTAAAA2Ig1ak6Jv//971m9enXat29fp72qqipPPfVUg+tUV1fnc5/7XJ229u3bZ9WqVVm6dGk6dOhQb53JkyenV69e6du3b4PbnD59eqZPn54kueiii9K5c+fGHM7HqrKycr3G+Uk4pjKpZ9NZ31om6rk2zs2mpZ5Nx7XetNSzabnWm45zs2mpZ9NyrTetT3o9N2iiy4/SlClT8vzzz+e8885Ls2YN39AxfPjwDB8+vPbvNc1nsTHp3Lnzeo3zk3BMZVLPprO+tUzUc22cm01LPZuOa71pqWfTcq03Hedm01LPpuVab1qfhHpuueWWa1zWqMc32rVrl2bNmmXJkiV12qurq1NVVdXgOlVVVamurq7TtmTJkjRv3jxt27at03711Vfnd7/7Xf7jP/4jXbt2bcwQAQAAgI1co0KJysrKbL311pk1a1ad9qeeeir9+vVrcJ0+ffrUe7Rj1qxZ2XrrrVNZ+b83bEyePLk2kNhqq60aMzwAAADgE6DRv75x4IEHZubMmZkxY0bmzZuXyZMnZ9GiRdl3332TJJdeemkuvfTS2v4jRozIokWLcvXVV2fevHmZMWNGZs6cmZEjR9b2ueKKKzJz5syccsop2WyzzVJdXZ3q6uosW7ZsAw4RAAAA2Bg1ek6JPfbYI0uXLs3NN9+cxYsXp3v37jnrrLOy+eabJ6n/nEqXLl1y1llnZcqUKbU/D3rsscdm0KBBtX1qfk70vPPOq7PuYYcdli996UuNHSoAAACwEdqgiS7322+/7Lfffg0umzhxYr227bffPpMmTVrj9m644YYNGQ4AAADwCdLoxzcAAAAANoRQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAoRWXZA/i0m3Z99QdaPvg366p+LRP1bDznZtNSz6bjWm9azs2mpZ5NSz2bjlo2LfVsWurZdD6N/05ypwQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUIrKDVn5zjvvzG233Zbq6up069Yt48aNy3bbbbfG/s8++2ymTJmSefPmpUOHDhk1alRGjBixQdsEAAAAPpkafafEgw8+mKuvvjqHHHJIJk2alH79+uWCCy7IG2+80WD/BQsW5MILL0y/fv0yadKkHHzwwZk8eXIeeuihRm8TAAAA+ORqdChx++23Z+jQoRk+fHi6deuW8ePHp0OHDrnrrrsa7H/XXXelQ4cOGT9+fLp165bhw4dn6NChmTZtWqO3CQAAAHxyNSqUWLlyZV588cXsuOOOddp32GGHPP/88w2uM2fOnOywww512nbccce8+OKLWblyZaO2CQAAAHxyNWpOib///e9ZvXp12rdvX6e9qqoqTz31VIPrVFdX53Of+1ydtvbt22fVqlVZunRpiqJY721Onz4906dPT5JcdNFF6dy5c2MOZ41eP2SPtS4fvdd3P3QbvztlSJ2/Kysrs3Llyve1nLfW9Zs9dNyH7mP1oJ9+aJ+NwYbW84O1TJq+np+WWibrX8/1rWXyj1NP1/r6+SRc68kno56u9aa1MVzriXq+n/fO93w813ri30n/y3vnunOtN61Pwr+TPu5abtBEl2UbPnx4hg8fXvv3xjj3xAfH1Llz5/UaZ5dG7OPTqqHjbOp6/qPUMtnwczNRz/dzrTedj+NaX9N+Po1c601LPZuW986m49xsWurZtFzrTeeT+ployy23XOOyRoUS7dq1S7NmzbJkyZI67dXV1amqqmpwnaqqqlRXV9dpW7JkSZo3b562bdsmyXpvEwAAAPjkatScEpWVldl6660za9asOu1PPfVU+vXr1+A6ffr0qfcYxqxZs7L11lunsrKyUdsEAAAAPrka/esbBx54YGbOnJkZM2Zk3rx5mTx5chYtWpR99903SXLppZfm0ksvre0/YsSILFq0KFdffXXmzZuXGTNmZObMmRk5cuQ6bxMAAAD49Gj0nBJ77LFHli5dmptvvjmLFy9O9+7dc9ZZZ2XzzTdPUv85lC5duuSss87KlClTan8e9Nhjj82gQYPWeZsAAADAp8cGTXS53377Zb/99mtw2cSJE+u1bb/99pk0aVKjtwkAAAB8ejT68Q0AAACADSGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASlFZ9gA+yaYetW3ZQ/hUUc+mpZ5NRy2blno2LfVsOmrZtNSzaaln01LPpqOWTesfsZ7ulAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEpR2ZiViqLIjTfemBkzZuTNN99Mnz598pWvfCXdu3df63oPPfRQrr/++rz++uvp2rVrjjjiiOy6665JkpUrV+a6667LE088kddffz2tW7dO//79c9RRR6Vz586NGSYAAACwEWvUnRJTp07N7bffnmOPPTYXXnhh2rVrl/PPPz/vvPPOGtd54YUX8oMf/CB77rlnvvvd72bPPffM9773vcyZMydJsnz58sydOzejR4/OpEmT8s1vfjMLFy7Md77znaxatapxRwcAAABstNY7lCiKIr/+9a9z8MEHZ9CgQenRo0dOOumkvPPOO3nggQfWuN6vfvWr9O/fP6NHj063bt0yevTo9O/fP7/61a+SJJtuumnOOeec7LHHHtlyyy3Tu3fvHH/88Zk/f37mz5/f+CMEAAAANkrrHUosWLAg1dXV2WGHHWrbWrZsme222y7PP//8Gtd74YUXsuOOO9Zp23HHHfPCCy+scZ233347SdKmTZv1HSYAAACwkVvvOSWqq6uTJFVVVXXa27dvn8WLF691vfbt29dbp2Z7H7Ry5cr8/Oc/z8CBA9OpU6cG+0yfPj3Tp09Pklx00UVNPvfE6x+yvDH7q6ysXL/1/vThXT4pc258Eur5aallsv7Hst61TP5h6rkxnJuNHUcZ1LPpuNab1kZxbibquRau9TVzra879WxarvWm9Umo58ddyw8NJe6///7893//d+3fZ5111kc6oCRZtWpVLrnkkrz11lv55je/ucZ+w4cPz/Dhw2v/fuONNz7ysb1fY/bXuXPn9Vqvy0c0jo3RxlDPT0stk/U/lvWtZfKPU8+N4dxs7Dg2RurZtFzrTefjODcT9Vwb1/qaudablno2Hdd609oY6vlR1HLLLbdc47IPDSV23nnn9OnTp/bvFStWJHnvzof3JyhLliypdyfE+1VVVWXJkiV12pYsWVLvjotVq1blhz/8YV555ZVMnDgxbdu2/bAhAgAAAJ9AHzqnROvWrbPFFlvU/q9bt26pqqrKrFmzavssX748s2fPTr9+/da4nb59+9ZZJ0lmzZqVvn371v69cuXKfP/738/LL7+cCRMm1AssAAAAgE+P9Z7osqKiIvvvv3+mTp2ahx9+OK+88kouu+yybLLJJhkyZEhtv/POOy/XXntt7d/7779/nn766dx6662ZP39+brnlljzzzDM54IADkrx3h0TNT4SecsopqaioSHV1daqrq7N8+fImOFQAAABgY7LeE10myUEHHZTly5fnyiuvzFtvvZXevXvn7LPPTuvWrWv7vP7663UmqOzXr19OPfXUXHfddbn++uuzxRZb5NRTT619NGThwoV59NFHkyRnnnlmnf2deOKJ2WuvvRozVAAAAGAj1ahQoqKiIl/60pfypS99aY19fvSjH9VrGzRoUAYNGtRg/y5duuSGG25ozHAAAACAT6D1fnwDAAAAoCkIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCWA/8/enYdlUe//H3+xyCYCIqICrom5pYkLuJWeSIvSzAzNJTOtNLE81fFIy9Hjt0KrY5ZpLqnlLlhqKm2uRyItPZqiEmqauYEgmKKALL8/vJgfyI43DOrzcV1ded/3zNyf+82sr/nMDAAAAACYglACAAAAAACYglACAAAAAACYwrY8I+Xk5CgiIkKbN2/W5cuX5evrq5EjR6p+/frFjrdz506tWrVK8fHxqlOnjp566il16tSp0GHnzZunTZs2aejQoerbt295mgkAAAAAAKqwcvWUWLdunTZs2KARI0YoLCxMLi4uevvtt3X16tUix4mLi9OMGTPUvXt3vffee+revbumT5+uI0eOFBh2586dOnr0qGrWrFme5gEAAAAAgFtAmUOJnJwcRUZGql+/fgoICFCDBg0UEhKiq1evKioqqsjxNm7cqFatWql///7y8fFR//791apVK23cuDHfcOfPn9eiRYv00ksvyda2XB05AAAAAADALaDMoURCQoJSUlLUpk0b4z07Ozu1aNFCv/32W5HjxcXFqW3btvnea9u2reLi4ozXWVlZ+uijj/TEE0/Ix8enrE0DAAAAAAC3kDJ3RUhJSZEkubm55Xvf1dVVycnJxY7n6upaYJzc6UlSeHi4atSooV69epWqLZs2bdKmTZskSVOnTpWHh0epxiut+BI+L8/32dralm28oyUPYunfXVFuhXreLrWUyv5bylxL6Y6pZ1WYN8vbDjNQT8thWbesKjFvStSzGCzrRWNZLz3qaVks65Z1K9SzsmtZYiixY8cOzZs3z3gdGhpaIQ05ePCgtm3bpvfff7/U4wQGBiowMNB4nZiYWBFNK1J5vs/Dw6NM43lWUDuqoqpQz9ulllLZf0tZayndOfWsCvNmedtRFVFPy2JZt5zKmDcl6lkclvWisaxbFvW0HJZ1y6oK9ayIWnp5eRX5WYmhRIcOHeTr62u8vnbtmqTrPR/yJigXL14s0BMiLzc3N128eDHfexcvXjR6XBw8eFApKSl6/vnnjc+zs7O1bNkyRUZGas6cOSU1FQAAAAAA3EJKDCUcHR3l6OhovM7JyZGbm5v279+vpk2bSpIyMjIUGxuroUOHFjmdZs2aaf/+/fke77l//341a9ZMktS7d28FBATkG+edd95R165d8/WGAAAAAAAAt4cy3+jSyspKQUFBWrdunXbt2qWTJ09q9uzZcnBwULdu3YzhpkyZouXLlxuvg4KCFBMTo7Vr1+r06dNas2aNDh48qEceeUTS9ftLNGjQIN9/tra2cnNzK7arBwAAAAAAuDWV65mbjz32mDIyMrRgwQKlpqaqadOmeuONN/L1qIiPj1etWrWM13fffbfGjx+vlStXatWqVapbt67Gjx+f79IQAAAAAABw5yhXKGFlZaXg4GAFBwcXOcysWbMKvBcQEFDgEo3iFDYNAAAAAABweyjz5RsAAAAAAACWQCgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMYWt2A1C8hKZhZjfhtkI9LYt6Wg61tCzqaVnU07Kop+VQS8uinpZFPS2HWlpWVasnPSUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApbM1uQGXJyspSWlqaJMnKyqpU42QPeqHYz62vXClzO+Lj45Wenl7m8SwtJydHkuTg4CAbGxuTWwMAAAAAuBPdEaFEVlaWrl69qurVq5c6kJCknLuaFfu5lZNTmdtia2tbZUKAnJwcpaamytHRscq0CQAAAABw57gjLt9IS0srcyBxJ7CyslL16tWNHiQAAAAAAFSmOyKUkEp/ycadhroAAAAAAMxSrss3cnJyFBERoc2bN+vy5cvy9fXVyJEjVb9+/WLH27lzp1atWqX4+HjVqVNHTz31lDp16pRvmDNnzmj58uWKiYlRZmamvL29NW7cOPn4+JSnqZI48C4J9QEAAAAAmKFcPSXWTa/vggAAIABJREFUrVunDRs2aMSIEQoLC5OLi4vefvttXb16tchx4uLiNGPGDHXv3l3vvfeeunfvrunTp+vIkSPGMAkJCXrrrbfk6empf/3rX/rPf/6jgQMHysHBoTzNBAAAAAAAVViZQ4mcnBxFRkaqX79+CggIUIMGDRQSEqKrV68qKiqqyPE2btyoVq1aqX///vLx8VH//v3VqlUrbdy40RhmxYoVatu2rZ5++mk1adJEderUkZ+fnzw8PMr3625x2dnZmjBhglq1aiVvb29FR0eb3SQAAAAAACymzJdvJCQkKCUlRW3atDHes7OzU4sWLfTbb7/pwQcfLHS8uLg4Pfzww/nea9u2rb799ltJ1w/A9+zZo379+umdd97R77//Lk9PT/Xp00ddunQpazNLJeu5vhUy3aLYzP+6TMNv3rxZ4eHhioiIUMOGDeXm5lbs8CkpKXrrrbf0ww8/SJIefPBBvf3223J1dS13mwEAAAAAqChlDiVSUlIkqcABsqurq5KTk4sd78aDY1dXV2N6f/31l9LS0rRmzRoNHDhQQ4YMUUxMjD7++GM5ODjIz8+vwDQ3bdqkTZs2SZKmTp1aZI+K+Ph42doW/KlZxfzOipDbhsLaUpiTJ0+qTp066ty5c6mGHzdunE6fPq0VK1ZIkl599VW9/PLLWrp0abHj2dvbV0pvlPgSPi9PG2xtbe/InjQl1VIqez3v1FpKzJuWRj0th2Xdspg3LYt6Wg7LumVRT8tiWbcs6llQiUfHO3bs0Lx584zXoaGhFdKQ7OxsSVKHDh306KOPSpIaNWqkY8eO6dtvvy00lAgMDFRgYKDxOjExsdBpp6eny8bGpgJaXTaZmZmytbVVZmZmicOOHz9eERERkqQ6derIx8dHO3fu1Ny5c7VkyRKdOXNG7u7uGjBggEJDQ3XkyBFt2bJFa9euVbt27SRdD2oef/xxxcbGqmnTpkV+V3p6epG1q0zlaYOHh0eVaHtVVNa6UMuiMW9aFvW0LJZ1y2HetCzqaVks65ZFPS2HZd2ybtd6enl5FflZiaFEhw4d5Ovra7y+du2apOs9H/KmMRcvXiz2MgE3NzddvHgx33sXL140ely4uLjIxsamwFM27tR7KUyZMkU+Pj5auXKlIiMjZWNjo6lTp2rx4sWaNGmS/P39lZSUpJiYGEnSnj17VL16dXXo0MGYRseOHeXk5KQ9e/YUG0oAAAAAAGCGEkMJR0dHOTo6Gq9zcnLk5uam/fv3Gwe6GRkZio2N1dChQ4ucTrNmzbR//3717fv/7+Owf/9+NWvW7HpDbG1111136cyZM/nGO3v2rGrXrl22X3UbcHFxkbOzs2xsbOTp6anU1FTNnz9fkydP1qBBgyRJjRs3NkKIhIQE1apVK9/jPa2srOTh4aGEhARTfgMAAAAAAMUp89M3rKysFBQUpHXr1mnXrl06efKkZs+eLQcHB3Xr1s0YbsqUKVq+fLnxOigoSDExMVq7dq1Onz6tNWvW6ODBg3rkkUeMYfr27avo6Ght2rRJ586d06ZNmxQdHa3evXvf5M+89cXFxSk9PT1fjQEAAAAAuJWV+UaXkvTYY48pIyNDCxYsUGpqqpo2bao33ngjX4+K+Ph41apVy3h99913a/z48Vq5cqVWrVqlunXravz48fkuDenUqZNeeOEFrVmzRosWLVK9evU0duzYQu8ngfw8PT2VlJSknJwco7dETk6OEhMT5enpaXLrAAAAAAAoqFyhhJWVlYKDgxUcHFzkMLNmzSrwXkBAgAICAoqddo8ePdSjR4/yNOu25uvrK3t7e0VFRalJkyYFPm/fvr1SU1O1e/dudezYUZK0e/duXblyRe3bt6/s5gIAAAAAUKJyhRKofM7Ozho5cqSmTp0qe3t7+fv7Kzk5Wfv379fw4cPl6+urnj17auLEiZo2bZokaeLEiQoMDOQmlwAAAACAKolQ4hYSGhoqV1dXzZgxQ2fPnpWHh4cGDBhgfP7JJ5/orbfe0pAhQyRJvXr10ttvv21WcwEAAAAAKNYdHUrYzP+62M9zThwp9nOrRr7Ffn6zRo8erdGjRxuvra2tFRISopCQkEKHd3Nz08yZMyu0TQAAAAAAWEqZn74BAAAAAABgCYQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSVVh2drYmTJigVq1aydvbW9HR0WY3CQAAAAAAi7E1uwFmemxZ7M1N4Meyjb9uSPMyDb9582aFh4crIiJCDRs2lJubW7HDf/TRR9qyZYsOHjyoq1ev6vTp02X6PgAAAAAAKhM9JaqwEydOyNPTUx07dpSnp6fs7OyKHT4jI0MPP/ywRo0aVUktBAAAAACg/O7onhJV2fjx4xURESFJ8vb2lo+Pj3bu3Km5c+dqyZIlOnPmjNzd3TVgwACFhoZKkv7xj39IkjZs2GBauwEAAAAAKC1CiSpqypQp8vHx0cqVKxUZGSkbGxtNnTpVixcv1qRJk+Tv76+kpCTFxMSY3VQAAAAAAMqFUKKKcnFxkbOzs2xsbOTp6anU1FTNnz9fkydP1qBBgyRJjRs3VocOHUxuKQAAAAAA5cM9JW4RcXFxSk9PV7du3cxuCgAAAAAAFkEoAQAAAAAATEEocYvw9fWVvb29oqKizG4KAAAAAAAWwT0lbhHOzs4aOXKkpk6dKnt7e/n7+ys5OVn79+/X8OHDJUmnT59WcnKyTp06JUnGTTAbN26s6tWrm9Z2AAAAAAAKQyhxCwkNDZWrq6tmzJihs2fPysPDQwMGDDA+f//9943HiEpS7969JUkRERHq0qVLpbcXAAAAAIDi3NGhxLohzYv9POfEkWI/t2rka8HWFDR69GiNHj3aeG1tba2QkBCFhIQUOvyMGTM0Y8aMCm0TAAAAAACWwj0lAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglqrDs7GxNmDBBrVq1kre3t6Kjo81uEgAAAAAAFmNrdgPMtH5VSglD1C7+410ljZ9fn4FuZRp+8+bNCg8PV0REhBo2bCg3t6LH//PPPzVjxgxFR0crISFBnp6e6tu3r8aPHy9HR8cyfS8AAAAAAJXhjg4lqroTJ07I09NTHTt2LHHYo0ePKisrS2FhYWrcuLGOHDmif/7zn0pOTtZ7771XCa0FAAAAAKBsuHyjiho/frwmT56s06dPy9vbW/7+/srJydGcOXPUtWtXNW7cWO3bt1dYWJgkqWfPnpoxY4Z69Oihhg0bKjAwUOPGjdPGjRtN/iUAAAAAABSOnhJV1JQpU+Tj46OVK1cqMjJSNjY2mjp1qhYvXqxJkybJ399fSUlJiomJKXIaly9fLvaSDwAAAAAAzEQoUUW5uLjI2dlZNjY28vT0VGpqqubPn6/Jkydr0KBBkqTGjRurQ4cOhY5/6tQpzZkzR+PGjavMZgMAAAAAUGpcvnGLiIuLU3p6urp161bisOfPn9eQIUN033336fnnn6+E1gEAAAAAUHaEEreZhIQEPfnkk7r77rv18ccfy8rKyuwmAQAAAABQKEKJW4Svr6/s7e0VFRVV5DDx8fEaMGCAfH19NXv2bNnacnUOAAAAAKDq4qj1FuHs7KyRI0dq6tSpsre3l7+/v5KTk7V//34NHz5c586d04ABA1S3bl1NnjxZFy5cMMatVauWbGxsTGw9AAAAAAAFEUrcQkJDQ+Xq6qoZM2bo7Nmz8vDw0IABAyRJ27dv1/Hjx3X8+HF16tQp33g7d+5U/fr1zWgyAAAAAABFuqNDiT4Di39cZs6JI8V+btXI14KtKWj06NEaPXq08dra2lohISEKCQkpMOzAgQM1cODACm0PAAAAAACWxD0lAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglqrDs7GxNmDBBrVq1kre3t6Kjo81uEgAAAAAAFmNrdgPM9PHHH9/kFL4p09AvvfRSmYbfvHmzwsPDFRERoYYNG8rNza3IYbOzs/Xss8/q4MGDSkpKkqurq7p166bXX39d9erVK9P3AgAAAABQGegpUYWdOHFCnp6e6tixozw9PWVnZ1fs8F27dtWcOXP03//+V/PmzdMff/yhUaNGVVJrAQAAAAAomzu6p0RVNn78eEVEREiSvL295ePjo507d2ru3LlasmSJzpw5I3d3dw0YMEChoaGytrbWc889Z4zv4+OjkJAQjRgxQmlpaXJwcDDrpwAAAAAAUChCiSpqypQp8vHx0cqVKxUZGSkbGxtNnTpVixcv1qRJk+Tv76+kpCTFxMQUOn5ycrK++uortWvXjkACAAAAAFAlEUpUUS4uLnJ2dpaNjY08PT2Vmpqq+fPna/LkyRo0aJAkqXHjxurQoUO+8d555x0tWrRIV69elZ+fnxYvXmxG8wEAAAAAKBH3lLhFxMXFKT09Xd26dSt2uDFjxui7777TihUrZGNjo3HjxiknJ6eSWgkAAAAAQOnRU+I24+7uLnd3d911111q2rSpOnbsqJ9//ln+/v5mNw0AAAAAgHzoKXGL8PX1lb29vaKioko9Tm4PifT09IpqFgAAAAAA5UZPiVuEs7OzRo4cqalTp8re3l7+/v5KTk7W/v37NXz4cO3evVsxMTHq2LGjXF1ddeLECb3//vuqX7++OnXqZHbzAQAAAAAogFDiFhIaGipXV1fNmDFDZ8+elYeHhwYMGCBJcnBw0IYNG/T+++/r6tWr8vT0VI8ePfTpp5/y9A0AAAAAQJV0R4cSL730UrGf55w4UuznVo18LdiagkaPHq3Ro0cbr62trRUSEqKQkJACw7Zu3VqrV6+u0PYAAAAAAGBJ3FMCAAAAAACYglACAAAAAACYglACAAAAAACYglACAAAAAACYglACAAAAAACYglACAAAAAACYwionJyfH7EZYypkzZwp9/8qVK3Jycqrk1hTO1tZWmZmZZjcjn6pUn7Ly8PBQYmKi2c24LVBLy6KelkU9LYdaWhb1tCzqaTnU0rKop2VRT8u6Ferp5eVV5Gf0lAAAAAAAAKYglAAAAAAAAKYglKjCsrOzNWHCBLVq1Ure3t6Kjo42u0kAAAAAAFiMrdkNMJPn0dBK/b6EpmFlGn7z5s0KDw9XRESEGjZsKDc3t1KNl5aWpkcffVSHDx9WZGSk2rZtW57mAgAAAABQoegpUYWdOHFCnp6e6tixozw9PWVnZ1eq8f7v//5P9erVq+DWAQAAAABwcwglqqjx48dr8uTJOn36tLy9veXv76+cnBzNmTNHXbt2VePGjdW+fXuFheXvffHdd98pOjpa//rXv0xqOQAAAAAApXNHX75RlU2ZMkU+Pj5auXKlIiMjZWNjo6lTp2rx4sWaNGmS/P39lZSUpJiYGGOcM2fOKDQ0VEuWLJGDg4OJrQcAAAAAoGSEElWUi4uLnJ2dZWNjI09PT6Wmpmr+/PmaPHmyBg0aJElq3LixOnToIEnKysrSuHHj9Pzzz6tVq1b6888/zWw+AAAAAAAl4vKNW0RcXJzS09PVrVu3Qj//+OOPVa1aNb3wwguV3DIAAAAAAMqHnhK3iR9//FG7du1Sw4YN873fp08f9e3bV5988olJLQMAAAAAoHCEErcIX19f2dvbKyoqSk2aNCnw+fTp03XlyhXjdXx8vAYPHqyZM2eqY8eOldlUAAAAAABKhVDiFuHs7KyRI0dq6tSpsre3l7+/v5KTk7V//34NHz5cDRo0yDd89erVJUmNGjWSl5eXGU0GAAAAAKBYhBK3kNDQULm6umrGjBk6e/asPDw8NGDAALObBQAAAABAuVjl5OTkmN0ISzlz5kyh71+5ckVOTk6V3JrC2draKjMz0+xm5FOV6lNWHh4eSkxMNLsZtwVqaVnU07Kop+VQS8uinpZFPS2HWloW9bQs6mlZt0I9i+u9z9M3AAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKe6IUOI2updnhaA+AAAAAAAz3BGhhMSBd1GoCwAAAADALHdEKOHg4KDU1FQOwG+Qk5Oj1NRUOTg4mN0UAAAAAMAdyNbsBlQGGxsbOTo66sqVK5IkKysr09pib2+v9PR0074/V25A4+joKBsbG5NbAwAAAAC4E90RoYR0PZioXr262c2Qh4eHEhMTzW4GAAAAAACmuyMu3wAAAAAAAFUPoQQAAAAAADAFoQQAAAAAADAFoQQAAAAAADAFoQQAAAAAADCFVU7usyEBAAAAAAAqET0lKtnEiRPNbsJthXpaDrW0LOppWdTTcqilZVFPy6KelkMtLYt6Whb1tKxbvZ6EEgAAAAAAwBSEEgAAAAAAwBQ2kydPnmx2I+40TZo0MbsJtxXqaTnU0rKop2VRT8uhlpZFPS2LeloOtbQs6mlZ1NOybuV6cqNLAAAAAABgCi7fAAAAAAAApiCUAAAAAAAApiCUwG1n7Nix+vrrr81uhmnCw8P13HPPKTg4WNu2bStyuMmTJ2vBggWV1zCgAh07dkzBwcFKSEgo03jbtm3TsGHDKqhVFSshIUHBwcE6duxYhX9XeHi4Xn311RKHY70CMwUHB2vnzp1mN8PiDh48qODgYP31118WnW5pl+tbQWnWh5W5zryV7dy5U8HBwWY3AybbtGmTxowZo4EDByo8PLzCv8+2wr/hFvXXX38pPDxce/fuVXJysqpXr6769eurX79+atOmjdnNMwQHB+uVV15RQEBApX/3rFmztH37dgUHB2vAgAHG+wcPHtS///1vffbZZ3JxcSlxOpMnT1b9+vU1cuRIi7QrLCxM9vb2FplWVfX7778rNDRUzZo10//93/8Z7588eVKrV6/Wa6+9pmbNmsnJyanIabz22muysbGpjOZWWbnzsCRZW1urZs2a8vPz01NPPSVnZ2eTW2euouaxW9nYsWPVu3dv9e3b1+ymFGvWrFm6dOlSgWeOHzt2TKGhofrkk0/k6ekpDw8PzZs3TzVq1Cj3d+Xk5Gjr1q3aunWrTp48qezsbHl4eKhVq1Z6+OGH5e3tLUnq27evHn744Zv6XVXVjfUuqv7BwcFG7W8XJR143H///Ro7dmyZppmZmanIyEhFRUXpzJkzqlatmry8vNSjRw/16NFD1apVK3EaCQkJCgkJUVhYmO66664yfX9VlnebI0k1atSQr6+vhg0bZixrhbn77rtvelkvzK2yXJdmPn3yySdLnI4l1pnS9f3WQ4cOFdj/laQPP/xQP/30k3r37m2x/VpLuBW26ZY+Hqhqblz+c/n6+uqdd96xyHdY6rjw8uXLWrBggZ5++mkFBATI0dHRIu0rDqFEEf7zn/8oPT1do0ePVt26dXXx4kUdOnRIly5dMrtpVUq1atW0fv169erVq1QBREXKzMyUra2t6e2oDFu2bFHv3r21fft2nTp1Sj4+PpKkc+fOSZI6duwoKyurQsfNrdOdftCd65577tG4ceOUlZWlU6dO6dNPP1VqaqrGjx9vdtNMVdQ8VpTc+QqVx9raWm5ubuUePycnRzNnztSuXbv0+OOPa9iwYXJ3d1dycrL27Nmj8PBw/f3vf5ckOTg4yMHBochp8fe/Nc2bN8/49549ezR37tx879nZ2ZVpepmZmXrnnXd0/PhxBQcHq0WLFqpevbqOHj2qjRs3ysvLS61atbJY+29FudscSbpw4YKWLl2qDz74QB9++GGhw+cuWzezrBelpOW6qijNfHr58uUSp3Oz68y8atWqpe3bt+uJJ54w9rcuXbqk3bt3q1atWhb5Dksq6zb9VldVt0l5l/9cVbGdiYmJysrKUvv27VWzZs1K+c6qV4UqIDU1VYcPH9abb76pe+65R5JUu3ZtNW3a1BimsDNuNyZ8Y8eO1d/+9jclJSXpxx9/lKOjo4KCgvKN88MPP2jDhg1KTEyUg4ODmjRpookTJxpnsLdu3aqvv/5aCQkJ8vDw0IMPPqigoCBZW1sbZy+mT59utHHWrFkVW5wbtG7dWklJSVq9erWeffbZQoc5dOiQli5dqj/++ENOTk7q2rWrhg4dKltbW82aNUuHDh3SoUOH9N1330mScSaquPGk6/X29vaWvb29tm/fLk9PT4WFhRX42yQmJmrRokU6cOCAJKlNmzYaMWKEsdEIDw/Xrl279J///Mdo87Zt27RgwQItWbLEmMbChQt1+PBhXbt2TR4eHnryySfVtWvXiilsMTIyMhQVFaUpU6YoPT1dW7Zs0dNPP63w8HCtXr1akjRw4EDjt+We9WvevLm+/fZbZWZm6rPPPiswv2ZmZio8PFxRUVFKSUmRu7u7goKCFBQUpOzsbM2dO1cxMTFKSUlRrVq19MADD6hPnz6ytr5+FVju97Rp00br1q1TRkaGOnbsqJEjR1bpnivVqlUzdlJq1aqlLl26GJe9ZGdn66uvvtLmzZt18eJF1atXT4MGDVLHjh0l/f+zeS+//LK+//57HT16VN7e3ho7dqysrKw0b948/fHHH2rUqJHGjRtnnGE9d+6cFi9erCNHjigtLU1eXl4KDg5W+/btjXaVZv1RUYqax3Ll/u6XXnpJmzdvVlxcnIYNG6ZevXqVql43nv3Mm+znDvPKK6/ohx9+0G+//abatWtrxIgR+Xqp7du3T59//rnOnz+vu+66S7169Sr2N02ePFnnz5/X0qVLtXTpUknK1x3xwIED+vzzz5WQkKCmTZtqzJgx+c6I7969WxERETp16pTc3NzUrVs3Pfnkk6buTNxYz8zMTC1evFi7du3SpUuX5Orqqm7dumnIkCGFjv/TTz8pKipKEyZMUIcOHYz3PTw85Ovrq7wP57pxPVnUeuVWFx4ebpzJyj07O2nSpAIH0mWt9aVLl7RgwQLFxsbq0qVLqlOnjvr06aOePXtW7A8qQd4DtOrVqxd474cfftDXX3+txMREeXh46LHHHlNgYGCR09u4caMOHTqkd999N98y7unpqYCAAKWlpUm6vvx+9dVX+vPPPyVJTZs21fDhw40DpZCQEElSaGioJKlly5bKfYL9tm3btH79ep09e1bVq1dX27ZtjeGl62f4pk+frr1798rV1VXBwcG67777imzz0aNHtXLlSh0/flyZmZlq0KCBhg0bpmbNmpVcwHLIu81xc3PTI488omnTpikjI0MpKSmFrlvr16+frwdq7j7KhAkTil1vrVmzRpGRkUpLS5O/v7/q1Kmjbdu2GfuKRS3XxW3H09LS9Nlnn2nXrl1ycHBQUFCQfvvtN9WoUaPMvWpKq6T5VJIRSpw/f17Lly8vdNtx4zozt2fvW2+9pRUrVujkyZPy8fHR888/X+KjFdu1a6dffvlFBw8eVOvWrSVJO3bsUNOmTQucFCppfi/tdq+8Stqmb9++XatWrdJff/2l1q1b69577zU+O3PmjMaPH68PPvhADRo0MN7ftGmTVqxYoblz58rW1lanTp3SkiVLdPjwYdnZ2al169Z65plnjL9TSfNWUccD58+fL9D7uqi/48SJExUREaETJ07otddek5+fn77++mtt2rRJFy5cUN26dfXYY4/lWx+sXr1aW7ZsUUpKSqHrE0vLu/wXZsOGDdq2bZvi4+Pl5OSkdu3aadiwYcZ8f+XKFS1YsEC//vqrrl69qpo1a+rhhx/WI488UqbjwuKOjbZt26bZs2dL+v/r4sroJUgoUYjc5Hj37t1q3rx5mc8U5LVx40YFBwerb9++2rt3rxYtWqTmzZurWbNmOnbsmBYsWKCxY8eqefPmSk1NVUxMjDHupk2bFB4ermeffVZNmjTRyZMnjYX/oYceUlhYmEaNGqUXXnhB7du3Nw4OK5OVlZUGDx6s999/X0FBQapbt26+zy9cuKCwsDB1795dL774ouLj4zVnzhxZW1vr6aef1ogRI3T27Fl5eXlp8ODBkiQXF5cSx8u1Y8cOBQYGasqUKSrs6bbZ2dl67733ZGdnp0mTJkmSFi5cqPfff19hYWFF9ia40WeffaZr165p0qRJcnJy0pkzZ8pbspu2c+dO1a5dWw0aNNB9992nDz/8UIMHD1bfvn1Vq1atAmcQpOvBkJOTk15//fUip/vJJ58oNjZWzzzzjBo3bqzz588rKSlJ0vU6uru76+9//7tcXFx09OhRowvk3/72N2Mahw8flpubm9566y0lJSXpww8/VL169fT4449XTDEsLD4+Xvv27TNCwcjISK1fv17PPfecmjRpoh07duiDDz7QtGnT1KhRI2O88PBwDR8+XHXq1NFnn32mjz76SK6urho0aJBcXV01a9YsLVy40OgOnpaWpnvvvVeDBg2SnZ2doqOj9cEHH+iDDz7I14W3uPVHRSpqHrvxAHzFihUaNmyYxowZIxsbm1LXqzRWrlypoUOHatSoUfryyy81Y8YMzZ49Ww4ODkpMTNT777+vBx54QL1799Yff/yhxYsXFzu91157Tf/4xz/Us2fPAgFGZmam1q5dqzFjxqhatWqaNWuW5s+frzfeeEPS9R3KmTNn6plnnlGLFi2UmJio+fPn69q1a/nWR2b75ptv9Msvv+jll1+Wp6enkpKSil1XRUVFycvLK18gkVdJ68fSrFduNX379tXp06d1+fJl42xWYb3Kylrra9euqUmTJurXr58cHR114MABzZs3Tx4eHsbJj6rm559/1sKFCzV8+HC1adNGv/76qxYsWCA3N7ci55moqCjdc889hV5yYW1tbVxSmJaWpqCgIDVs2FAZGRn68ssvNW3aNH344YeytbXVu+++q9dff12vv/66GjVqZKwQniVoAAAgAElEQVR7fvjhB33++ed66qmn5Ofnp7S0tHz7TdL1g4zBgwdr8ODB2rJliz799FO1bNlSHh4ehbY5LS1N9913n5555hlZWVnp22+/VVhYmD7++GOLXy5xo6tXryo6OloNGjTIt69547o1txdkXiWtt3788UfjhFGLFi20a9curV27tsRekiVtxxcvXqxDhw7pH//4h2rWrKkvv/xShw8fVqdOnSxYmfIrbttRlOXLl2vIkCGqWbOmPv/8c82cOVPTp08vdh1oY2Oj++67T1u3bjVCia1bt+rRRx/V1q1b8w1b0vx+M20vjeK26UeOHNHs2bM1cOBABQQE6ODBg1qxYoUxrpeXl+666y7t2LEjX+i6Y8cOde7cWba2tkpOTtakSZPUs2dPDRs2TFlZWVqxYoXee+89vf3228bxSXHzVlHHA+fPny/171y2bJmefvpp1a1bV46Ojlq5cqV27typkSNHysvLS3FxcZo7d66cnZ3l5+ennTt3av369Xr55ZfVoEEDXbx4UUeOHLmpWt8sKysrPfPMM/L09DROiC5cuNDYHq1cuVInT57UxIkT5erqqoSEBONeM6U9Lizp2KhLly5yc3PTu+++q3fffVceHh6V0gudUKIQNjY2evHFFzV37lxt3rxZjRo10t13363OnTvL19e3TNNq06aNHnroIUnSww8/rG+++UYHDhxQs2bNlJiYKHt7e3Xo0EGOjo6qXbt2vh33L7/8UkOHDjWuC/L09FR8fLy+++47PfTQQ8YMUr169Qrp1ldafn5+at68uVauXFmgy/t3332nmjVratSoUbK2tpaPj4+GDBmiefPmaeDAgXJycpKtra3s7e3z/YaSxstN7D09PYs9KIiJidEff/yhmTNnGgnfSy+9pJdeekkHDhwodQKdmJgof39/4+9j5jXFW7ZsUffu3SVdP3tkb2+v3bt3KyAgoMgzCNWqVTN2XApz9uxZRUdH6/XXXzcS8jp16hif29raGr0vpOu///jx4/rxxx/zhRJOTk56/vnnjb9ZQECAYmJiqnQosW/fPg0bNkzZ2dm6du2aJBnz1Pr169WnTx9169ZN0vUeKIcPH9bXX3+tl156yZjGo48+Kj8/P+Pf06ZN08CBA40dlYceeijfzf8aNWqUb1nv37+/9uzZo507d+qJJ54w3i9u/VGRipvH8nrooYfyvVfaepXGI488Yhz4DB48WP/973914sQJNW/eXN9//708PDw0YsQIWVlZydvbW2fPntWqVauKnJ6zs7Osra3l4OBQYPnIysoydlokqU+fPvr000+Vk5MjKysrrVmzJt9Z7bp162rIkCGaOXOmhg0bVupwsyxy58u8Cgte8zp//rzq1aunFi1ayMrKSh4eHrr77ruLHD53BzCvpUuXGmepJBm9xQpT0nrlVuTg4CA7O7tCz2bl7VlT1lq7u7vn6+VUp04dxcTE6Mcff6yyocT69evVvXt3Yx3k5eWl33//XevWrSsylDh79qxatmxZ4rRvXJe8+OKLGj58uI4eParmzZsb+zc1atTI93f48ssvFRQUpEcffdR478Yz2vfdd59xJnTgwIGKjIzUoUOHiuwtkbuezvXss89q165d2rt3b7E9LMor77Kdnp6uWrVqGT1Cct24bi0slChpvRUZGan7779fDzzwgCTp8ccf18GDB3X27Nli21fcdjwtLU1bt25VSEiIsf80evRojRkzpvwFsbDith1Fybu9fuKJJ/Svf/1LFy5cKPEyjJ49eyo0NFRXrlzR2bNnlZCQoICAgAKhREnz+820vTSK26ZHRkaqdevW6t+/v6Try/mxY8e0ZcsWY/zu3btrw4YNGjx4sKysrJSYmKjY2FgjPPj+++/VsGFDDR061BgnJCREzz77rH7//Xejp3lx81ZRxwNl8eSTT6pt27aSrgdBGzZs0JtvvqkWLVpIur7vevToUX333Xfy8/NTYmKi3Nzc1KZNG9na2srDw6PC72FT2La9d+/eRu0eeeQR431PT08NHTpU7733nsaOHStra2udP39ejRs3Nmpau3ZtY/jSHheW5tgoN5B1cXGptGNMQokiBAQEyM/PT7GxsYqLi9O+ffu0YcMGDRo0yFhwS6Nhw4b5XtesWVMXL16UdP2Ao3bt2goJCVHbtm3Vpk0b+fv7y9HRUX/99ZeSkpI0b948zZ8/3xg/Ozu7xB1TMwwZMkRvvPFGga7lp0+flq+vb760rnnz5srMzNS5c+cK1Kes45XUve7UqVNyd3fPFyLUqVNHNWvW1KlTp0odSgQFBWn+/Pnat2+f7rnnHnXq1KnE764I586dU2xsrHGAZ2VlpW7dumnLli3F3tSmQYMGxR44HD9+XFZWVsVe6/v9999ry5YtOn/+vDIyMpSVlZVvZShJPj4++f5m7u7uOnr0aGl/nilatGihF154QRkZGdq0aZPi4+MVFBSkK1euKDk5ucCBRvPmzbV379587+Wdj11dXSUpXzdHV1dXpaenKz09Xfb29kpLS9Pq1au1Z88epaSkKDMzU9euXcs3zo3TlfKvPypKWeaxvBvvstSrNPL+9tzrGXN/e+76IW8YcDNBTe6N+PJ+X2ZmplJTU+Xs7Kzff/9dR48e1bp164xhcnJyjO7WFXG9Ze58mdfJkyf1wQcfFDlOjx499Pbbb+vll19WmzZt5Ofnp3vvvbdMvej69u2rwMBA7du3TwsXLix22JLWK7ezstY6Oztba9euVXR0tC5cuKBr164pMzOzSt9f4dSpUwUuL2nevLl2795d5Dil3T85d+6cVq1apaNHj+qvv/4y9m0SExOLHOfixYu6cOFCiSFO3vWojY2NXFxcin1qxcWLF7Vq1SodPHhQKSkpys7OVkZGRrFtuRl5l+3Lly/r+++/1zvvvJPvRnelOTAqab115swZI5DI1bRp0xJDieK24+fOnVNWVla+y5kdHBxUv379EttbWYrbdpRmHHd3d2OckkIJHx8fNWzYUD/++KNOnDihrl27Fnq5amnn9/K0vSQlbdNPnz6d79JR6fr2NG8o0bVrV+PSjJYtWyoqKkqenp7G9v7333/X4cOHC32S1blz54z5paL3EfMuN6dOndK1a9f07rvv5hsm775rbiiTexx27733qkOHDhW6XSts2573pvQxMTFas2aNTp8+rStXrig7O1uZmZnGZdW9evXS9OnTdfz4cd1zzz3q0KFDqYLgvCx1bGRphBLFsLOzU5s2bdSmTRsNGDBAc+bMUUREhPr27VvombGsrKwC7934dAMrKytjo+3o6Khp06bp8OHD2r9/v9auXasVK1YoLCzMWGife+65Ys++VBVNmzaVv7+/li5dmu9Mb3HKe3Yx73g3c6+C3OlYW1sX2JHKzMzM9/pvf/ub2rZtq71792r//v1688031a9fv0p/ZNLmzZuVnZ2tF1980Xgvt+3F7UDd7D0doqOj9cUXXxjX2To5Oenbb7/VL7/8km+4wp7mURVDtLzs7e2Ny46effZZ/fvf/9bq1avznYkrSd7fnTtfFfZebi2WLFlipOX16tWTvb29PvnkkwLzXXHrj4pS0jyWtwt0Weer3PVa3t9w42/OVVz9LO3GA8nc78vOzjb+P2DAAHXu3LnAuBXVpTHvfJkrNTW12HGaNGmiWbNm6ddff9WBAwc0a9YsNWzYUG+++WahB8v16tXT6dOn873n4uJS6jMjVfleMRWtrLX++uuvtX79eo0YMUINGjSQg4ODli9fbvFHPFaG4rbdXl5eBeapwkybNk3u7u567rnn5O7uLhsbG73yyitFrg/K4sbLzKysrIxluTCzZs3SxYsXNXz4cNWuXVvVqlXTlClTLNKWwty4bDdp0kTDhw/Xpk2bjJ6HpVm2SlpvldetuB3Pqzzbjpv5zT179tT333+v+Ph449KZG5V2fq+I7V559xvzcnV1VZs2bRQVFWWEErk9InOn165du0J7LueeqJHKV+fSHm9J+Zeb3On+85//LHDpVm47PDw8NGPGDMXExGj//v1avHixVq9erXfeeafCbgBb2LY91/nz5xUWFqYHHnhAAwcOlLOzs44fP66PPvrImFfatWunWbNmad++fTpw4IDCwsLUuXPnfH/fm1ERPT9Li1CiDHx8fIwE3cXFRcnJycZnGRkZOn36dJmvm7axsVHr1q3VunVrBQcHa9SoUfrf//6nwMBA1axZU/Hx8br//vuLHf9mN0CWMnjwYP3973/Xvn37jPe8vb31008/KTs729iAxsbGytbW1rg8wNbWtsBvKM14peHj46MLFy4oISHBSATj4+OVnJxs3GDIxcVFFy9eNLo8StKJEycKTKtWrVoKDAxUYGCg1q5dq2+++aZSQ4msrCxt375dgwcPNi4VyPXJJ59o27Zt5b6bcqNGjZSTk6ODBw/mu8FRrtjYWDVt2tToxitdr+PtaMCAAXr33XeNZfC3337Ld2YuNjb2pu9aHRsbq/vvv9/oeZCRkaH4+HjVq1fvpqZ7s0ozj934+LNcTk5OJdYr9wA+JSXF+LywZa0k3t7e2rVrV75ltjTXgRa2rimNJk2a6PTp00XuSFQljo6OCggIUEBAgHr06KE33nhD586dK3CZhnT97NdHH32kXbt2yd/f34TWVk2lnU/KUuvY2Fi1b9/euBwgJyfHuFFjVeXj46PY2Nh8l+iVtP7r2rWrVqxYoWPHjhU425+dna20tDRlZWXp9OnTGjlypNFl/vfff893oJEbLOT9O7i6usrd3b1Ml16WRmxsrEaMGGGs81JSUvLt31UGa2trZWRkWHSaud3w8/79jh07dlPTrFu3rmxsbHTs2DFjXyw9PV1//vlnmfbNbiddunTR559/Lk9Pz0Iv8b506VKJ83tFKc023dvbu8D2My4ursC0unfvrgULFigwMFAnT57UK6+8YnzWuHFj/fTTT/Lw8Lipmz8Xtu7N3W9ITk42/l2a/QYfHx9Vq1ZN58+fL3CJVl52dnby8/OTn5+f+vXrp+eff16//fabcRlIZTp27JgyMzP1zDPPGMc+//vf/woM5+LiYlym1q5dO3300Ud67rnnVK1atVIdF5bm2MgMhBKFuHTpkqZPn66ePXuqYcOGcnR01LFjx7Ru3Tq1bt1aTk5Oat26tbZu3aoOHTrIxcVFX331VZlXMHv27FF8fLxatGghZ2dnHTx4UFevXjVudBccHKyFCxfKyclJfn5+yszM1PHjx3XhwgXjGn1PT08dOHBALVu2NP0xj3Xr1lVgYKAiIyON93r37q3IyEh99tlnCgoKUkJCgpYtW6aHHnrISDRr166to0ePKiEhQQ4ODnJ2di7VeKVxzz33qGHDhsZN6qTrN3Np3LixsZJq2bKlLl++rDVr1qhLly46dOiQdu3alW86ixYtUrt27VSvXj1dvXpVv/76a6UvuP/73/906dIlPfDAAwVuvtWlSxf98MMP+a7nKwsvLy917txZc+bM0TPPPKMmTZooKSlJ58+f13333ad69epp27Zt2rt3r+rWrasff/xRhw4dui0fK9qqVSv5+Pjoq6++Ut++fRUeHq66desaN248fPiwpk2bdlPfUa9ePf3888/q0KGDbG1tFRERYfEd0vIozTxWXE+okuplZ2cnX19frVu3TnXq1NGVK1e0fPnyMrezV69e2rBhgz7//HP17t1bJ0+e1A8//FDieLVr11ZsbKwuXLhQpscHP/HEE5o2bZpq166tzp07y8bGRn/++aeOHj1a7mWuImzYsEFubm7GjQGjoqLk6OhYZBfkLl266JdfftHHH3+sxx57TPfee6/c3NyUlJSk//73v6aeMTFT7dq1tW/fPp05c0bOzs7Gtc55lbXWXl5eio6OVmxsrGrUqKFvvvlGCQkJaty4cWX8pHLp06ePPvzwQzVp0kRt27bVvn37FBUVpVdffbXIcR555BHt3btXb7/9tp588km1bNlSTk5OOn78uNavX6+nnnpKLVq0UI0aNbR582Z5eHjowoULWrJkSb6zqK6urrKzs9Ovv/6q2rVry87OTk5OTurfv7+++OILubq6ys/PTxkZGTpw4ID69OlT7t9Zr1497dixQ76+vkpLS9OyZcsq9Kk6165dM4LZy5cv69tvv1VaWlqBLvQ3KygoSLNnz9Zdd92lFi1a6Oeff9aRI0duKghzcHBQz549tWzZMtWoUcO40WV2dvYdu75wdHTU3Llzi7x0q3r16iXO7xWlNNv0l19+WW+99ZbWrFlj3Ojyxl6w0vVHzc+bN0+ffvqp7rrrrnzha+/evbV582bNmDFDjz32mFxcXBQfH6+ffvpJTz/9tBwdHUvV3sKOB+rWratatWopIiJCgwcP1vnz5/XVV1+VOC1HR0f16dNHS5YsUU5Ojlq2bKm0tDTFxcXJ2tpagYGB2rZtm7KysuTr6ysHBwdFR0fLxsamQk8Q5V3+c1lbW8vFxUX16tVTTk6ONm7cKH9/f8XFxWnjxo35hl21apUaN26s+vXrKysrS7t27ZKnp6dxyUlpjgtLc2xkBkKJQjg4OMjX11fffPONzp07p2vXrsnd3V3dunUzdsj79eunhIQEvffee3JwcFD//v3LnKxXr15dv/zyi1avXq309HTVrVtXo0ePNm7I8sADD8je3l7r16/XihUrZGdnJx8fn3xnq4cNG6bFixdrzJgxcnd3r/RHgt5owIAB2r59u3HDQHd3d4WGhmrp0qWaMGGCqlevrq5du+qpp54yxunTp49mzZqlV155RRkZGcZjZ0oarzSsrKw0YcIELVy4UP/+978lXV8Yn332WWMD6uPjo1GjRmnNmjVas2aN2rdvr8cffzzf3YdzcnK0cOFCJSUlycHBQffcc0+l33V/y5YtatWqVaF3A+/cubOWL19+U938QkJCtGrVKi1atEiXLl1SrVq1jBvuPPjggzpx4oQ+/vhj5eTkyN/fX3369ClwM6fbRZ8+fTR79mx99NFHunr1qpYtW6aUlBR5eXnp1VdfLXOPqBsNHz5cc+bM0aRJk1S9enUFBQUZy4yZSjOP7d+/v8gN9sMPP1xivcaMGaO5c+cqNDRUderU0ahRo4y7P5eWh4eHXnvtNX3xxRfatGmTmjRposGDB2vmzJnFjhccHKz58+dr3LhxunbtWr4bFxbn3nvv1cSJE/Xll19q/fr1xk5Ljx49ytTuiubg4GA8KtHKykqNGjXS66+/XmSQa2VlpZdfflmbN2/W1q1btWHDBmN717p165sO325VgYGBOnTokCZOnKi0tLRCHwla1lr3799fCQkJevfdd2VnZ6cePXqoe/fuOnXqVGX8pHLp1KmTRowYofXr1+uLL76Qh4eHRo4cWeRNLqXr9zl48803tXHjRm3dulXLli2TnZ2dvLy81KNHD919992ytrbW3//+dy1atEivvvqq6tatq2HDhuV7LLeNjY1GjBih1atXKyIiQi1atNDkyZPVq1cv2draav369Vq2bJmcnZ3Vrl27m/qdY8aM0bx58/TPf/5T7u7uevLJJyv0spoDBw7o+eef/3/t3XlYVdXi8PEvM5ogEDKDqIA445CamFylxOEtu4mzlOZwVdSbaSXXvIITiSVWTpiFJRKoaY7liKIYDiUOiAxXkUEZFEFQGQ6c9w+es38cDygoeA62Ps/j83j22XuftRdrr7X22msAKh+cbGxsmDNnDh06dCAnJ6fefsfd3Z3s7GzCw8MpKSmhV69evPXWW0+cE6Q23n//fb777jup/jt06FAKCgr+tvPLgPKcAI+rTXqvLV9fX9q3b1/rpVdrU6Y/evSIadOmScvKd+jQgREjRqjMKWRgYEDPnj2Jjo6WHmQVzMzMWLJkCeHh4SxfvpzS0lLMzc3p0qVLndJFTc8DH330EZs2beKTTz7B0dGRMWPG8MUXXzz1fKNGjaJ58+bs3buXTZs20aRJExwdHRk2bBhQ+XfbvXs3W7Zsoby8HDs7O+bNm9egk9lXvf8VzMzM2LBhAy1btmTChAns3r2biIgI2rZti4+PD6tXr5b21dPTIyIigpycHPT09HBxceGzzz6Tvq/Nc2Ftno3UQUvemAaKCYIgCIIgCIJQZytXrqS8vFxanro+lJWVMWPGDN55553n6rEiPFlJSQkffvgh06dPV5rPQRBeFqKnhCAIgiAIgiC8REpKSjh06BBubm7o6OgQGxvL+fPnnzj8pjZu3LhBZmYmTk5OPHr0iN27d1NcXEyfPn3qKeRCdeLj43F2dhYNEsJLS/SUEARBEARBEISXSGlpKStWrODGjRuUlpZibW3NsGHDnvuh9saNG4SEhHDr1i10dHRwdHTEx8dHLcukC4Lw8hCNEoIgCIIgCIIgCIIgqEX1U8UKgiAIgiAIgiAIgiA0MNEo0QBGjhxJbGzsc51j7ty5tZ4ZvrEoKipiypQpZGVlqTsoKlatWsXevXvVHYyXXn3cG5pk7dq1SjNAP/65Ol988YXaV8lpbI4fP46Pj88T99mzZ89TZyR/2dJfTXx9fdmzZ88T9/Hx8eH48eMvJkAvudjYWEaOHFkv58rMzGTBggWMGzeu1jPsQ+NJ2zk5OYwcOZL//e9/z7VPdbZt28aUKVMYOXKkSNvPqTGkp2dNJ7V1/PhxaWUCTVNQUMCkSZO4e/fuC//ta9euMW/ePMaMGYO/v3+D/U5D5hUvi5etHBcTXdbR2rVrOXHihPTZyMgIZ2dnfHx8sLW1VWPINN+uXbvo2rUrVlZWAISGhpKYmEh6ejomJibVPqidPn2aXbt2cfv2bYyNjRk0aBDvvPNOtee/du0a/v7+2Nraqiy19PDhQyIiIjhz5oy03OWYMWOkiZm8vb1ZtGgRnp6eT1zaqT7k5eWxfft2Lly4QEFBAcbGxnTt2pURI0bUuMa9pvP19SU3N7fG79u3b9+ghVddrV27lsLCQpUZyP/3v//h5+cnLUNVVxMnTnyuZVkbi8OHD/PTTz8RGhqKrm5lMSKTyZgwYQKWlpZK919WVhazZ89m4cKFdOrUSV1BBmDjxo288sorag3Ds3i83FEICgqqdnnawMDAGpemfFk01jT4NBERERgYGBAcHIyhoaG6g1MnT2uY8fDwYMSIEU89j7m5ORs3bqx2GcOapKWlsWPHDubNm4eLi0uDl+PPyt/fH3t7eyZNmqS0/fjx43z//fds2bLlmc+dk5PDzJkzpc+6urq0aNECT0/PGutNmqq+0tKzkslkREREMHv2bGlbeno627Zt48aNG+Tk5ODt7a0SzkePHhEZGcnZs2cpKCigVatWTJgwAScnJ2mfiIgIYmNjuXv3Lrq6urRq1YpRo0bRtm1blXDI5XICAwOJi4vj448/pnfv3gA0b94cDw8Ptm3bxvTp0xsoFqq3efNmWrZsyfz582vMo6qWWdra2piamtKtWzfGjBlDs2bN6i0sz5JXaKqaynlnZ2eWLVumhhC9eKJR4hl06tSJWbNmAZUPmGFhYXz55ZcEBwerOWSaq6SkhGPHjimtpSuXy/Hw8CAtLY1Lly6pHHPhwgW++eYbJk6ciJubG5mZmYSEhKCvr8+gQYOU9i0qKmLNmjV06tSJvLw8pe9kMhlLly6lWbNmzJkzBzMzM/Ly8qSKLICDgwOWlpZER0ernLs+5eTk8Pnnn2NhYYGvry/W1tZkZWURERGBn58fS5cubdD1kRtKYGAgFRUVAKSmprJ8+XKWL1+Oubk5gFJcv8w0tSJc3zp06EBJSQkpKSm4uroCkJycTNOmTbl9+zb379/H2NgYgCtXrqCnp1dthas2ZDJZvYXbxMSk3s71olUtdxQer4jJZDJ0dXWluH+Zvcg0+CIo/nZZWVm89tprjbIc2Lhxo/T/P//8k5CQEKVt+vr6FBUVPfU82tradb5XFT0wX3vtNbS0tOp07MvmP//5D46OjpSVlXHlyhU2btyIubl5o1odo77S0rOKjY1FX1+f9u3bS9tKSkpo0aIFvXr1IiIiotrjNmzYQFpaGr6+vrz66qtER0ezZMkSgoODMTMzA8DGxoZJkyZhYWFBaWkp+/fvZ/ny5Xz99dcq6X7v3r01pud//OMf+Pn54ePjU68P+k+TlZWFl5eXVL+riaLMKi8vJyMjg/Xr1/PgwQM++uijegvLs+QVmqy6cv7vUn8G0SjxTPT09KSbwMTEhKFDh7JixQpKS0vR19dX2vfzzz/HxcWF999/X9r28OFDpkyZwuzZs+nVqxcFBQWEhIRw8eJFmjdvjre39wu9nhfhwoULAEqVwg8//BCo7HZdXaNEdHQ03bt3x8vLCwBLS0veffdddu/ejZeXl1JGvWHDBjw8PJDL5Zw5c0bpPMePH+f+/fssXrxYurmrq/D16NGDmJiYBm2U+P7779HS0mLhwoXSm0xzc3MWLlzI7Nmz+f777/Hz8wOqf6Py+Bt+uVzOnj17OHLkCHl5eVhZWTFs2DD69esnHZOXl8dPP/3ExYsXAXBxcWHChAlYW1sDlV1ez5w5w3vvvUdERAQFBQV07NiRadOm1frhpup+igclY2PjaguLoqIiVq1axYULF2jevDkjR46sU3hfpKtXrxIWFsbNmzdp2rQp7u7ujB8/vsZC4vG/T0lJCZs2bSI2NhZDQ0MGDx6sckx0dDS//fYbmZmZUiVowoQJmJmZIZfLmT17Nm+99ZbSm67bt2/z73//my+++EItM57b2NhgamrKlStXpAfC+Ph4OnbsSG5uLvHx8bz++uvSdhcXF/T19SkrK2Pr1q3ExMTw8OFDadb2qucICAhg/vz5bN++ndTUVObNm1dtGHbv3s2+ffsoLi6mV69etXqIGzlypPS2SfFW8eOPP+bw4cMkJibSokULJk6cSOfOnesppupP1XJHQdEzzMDAgBMnTmBhYUFgYCC+vr54eXlJaSYrK4sNGzaQnJyMubm5UnmksHXrVs6ePTlWV3wAACAASURBVMudO3cwMTHh9ddfZ+TIkejr65OTk8OsWbNYvnw5bdq0kY45cuQIP//8MyEhIS+84vQsaVBLS4vNmzc/Nf0tXLiQn3/+mbS0NOzs7Jg6darSfXbixAkiIyO5f/8+HTt2xM3NTSV858+fZ/v27WRkZGBiYkLfvn0ZMWKEFE++vr54eHhw584dzp49S+fOnaXu8jdv3mTHjh14e3vzj3/8g5kzZxIYGKgU91XTsqaomj4VPZIeT7OKB8nc3FzCw8Orve8U92bVa87IyGDLli0kJCSgr69Px44dmTBhAiYmJmzbto0dO3YAMGrUKIBGP/xVUZY4Ozvz+++/U1xczOuvv87kyZNV6pmPMzIykuK9f//+HDx4kOvXr0uNEikpKURERHDjxg1kMhkODg74+Pjg4uKidJ78/HwCAwOJj4/H2NiY0aNHS2V1QEAAdnZ2SvWThw8fMnXqVGbNmkWvXr2e6/rrKy3Bk9NOTU6dOkW3bt2Utjk5OUk9Hnbt2qVyTGlpKWfOnGHu3Ll06NABqLxP//zzTw4dOsTo0aMBlOo7AO+//z7Hjh0jNTVVKS9JSUnht99+44svvmDKlCkqv+fg4ICpqSlnzpzB09OzxmupiyeV0VV74qxfv57169czY8YM/vGPf1R7rqpl1quvvkqfPn2UhhpUVFSwc+dOjh49SkFBAdbW1owePZrXXntN6Ty3b99m8+bNXL9+Xfr7dunSBVDNK2qbh2uq6sp5hdqU4y+ivt+QxJwSz+nRo0ecPn0aBweHaguKN954g5iYGOktMsCZM2fQ19eXMrx169aRlZXFwoUL+eSTT4iOjiYnJ+eFXcOLkJCQQOvWrev0BqOsrAw9PT2lbfr6+ty9e1dpqMDBgwcpKChg+PDh1Z7n3LlztG3blh9++IEpU6YwZ84ctm3bpvIG1snJiZSUFEpLS+twZbVXVFREXFwcXl5eKl2rDQwM8PLyIi4urk6t/xERERw7doxJkyYRHBzMP//5T7777jv++usvoPKhOCAgAD09Pfz9/Vm6dCmmpqYsWbKEkpIS6Tw5OTmcPn2aefPm8fnnn5Oamlrjm4DntWPHDnr06MHKlSvp06cP69ev586dO3UK74uQl5dHYGAgjo6OrFixgmnTphETE0N4eHitz7FlyxYuXbrE3LlzWbhwIampqSQkJCjtI5PJGDFiBCtXrmT+/PkUFhby9ddfA6ClpcWAAQNUxgxGRUXh6Oio1kK2Q4cOxMfHS5/j4+Pp0KED7du3V9p+9epVqYIWFhbG6dOnmT59OitWrMDe3p5ly5Zx7949pXNv3bqV0aNHs3r1apydnVV++/Tp00RERDBy5EhWrFiBjY0N+/fvf6briIiIYPDgwaxcuZI2bdqwevVqiouLn+lc6nDy5EkAFi9eXO0cBBUVFaxcuRK5XM7SpUuZPn0627dvV8n/DAwMmD59OsHBwUyaNImYmBh27twJVDbidu7cmaioKKVjoqKieOONN9T2JqeuabC26S88PJyxY8eyYsUKjIyM+Pbbb6VhWcnJyaxbt44333yToKAgunfvrvIAHBcXx7fffsugQYP46quvmD59OrGxsSp5x/79+7G1teWLL75gzJgxbNy4ERsbG/7f//t/bNy4sdF1ua+Lutx39+7dY9GiRdjb27N8+XIWLlxIcXExQUFBVFRU8M477/Cvf/0LqHzDXvWNemN29epVbt68ycKFC5k7dy4XL14kLCys1sfL5XKuXbtGZmamUj5aXFxMv379CAgIYPny5Tg6OhIYGEhhYaHS8du2baNHjx4EBQXx5ptvsnbtWmnsvqenJ6dOnaKsrEzaPyYmBkNDQ7p37/6cV143T0pLT0s7Nbl27ZpSI2BtlJeXU1FRUW299dq1a9UeI5PJOHLkCE2aNFEaivfo0SO++eYbpk6dSvPmzWv8TScnJ65evVqncD7Jk/JIxVAJAwMDJkyYwMaNG2vd+yY7O5u4uDh0dHSkbQcOHGDv3r2MGzeOL7/8kp49e/Lll1+SmpqqEqbBgwcTFBRE586dCQoKUukR/bgn5eGNUW3KcU2s79eVaJR4BnFxcfj4+ODj48MHH3zA1atXlcadVdWnTx/u37+vVEE6deoUvXv3Rk9Pj1u3bnHhwgWmTp2Kq6srrVq1wtfXt8EejNUlNzcXU1PTOh3j5ubG+fPnuXjxIhUVFdy6dYt9+/YBlS348H/jSGfNmoW2dvXJOTs7m9jYWGQyGX5+fowaNYrDhw+rVBBNTU0pLy9/amb3rG7fvo1cLsfOzq7a7+3s7JDL5bWeCLS4uJh9+/Yxbdo03NzcsLCwoG/fvnh6enLw4EGgspIgl8uZMWMGLVu2xNbWlqlTp1JcXMyff/4pnauiogJfX19atmyJi4sLb775JpcvX37+i65Gv3796NevH1ZWVowaNQodHR2pUK1teOtD1ftY8W/RokXS9wcPHsTU1JTJkydjZ2dH9+7dGTduHL///nutGkiKi4s5duwY48ePx83NDQcHB2bMmKHSMDdgwAC6deuGpaUlTk5OTJ48mYSEBGkCq/79+3P79m2SkpKAyr/ViRMnGDBgQD3GRt117NiRpKQkysrKKC0tJSkpSeWBMDMzk3v37tGxY0eKi4s5dOgQ48aNo1u3btLbCxMTEym9KowYMYIuXbpgaWlZbev9gQMH8PDw4K233sLGxob33ntPacxuXQwdOpQePXpgbW3N2LFjKSoqUqkUaYLH0+vy5cuBygaD999/H1tb22rzlsuXL5ORkcGsWbNo1aoVrq6uTJgwgfLycqX9vL29cXV1xcLCgm7duvHPf/6TmJgY6XtPT09iYmKksikjI4Pk5GS1psO6pMH27dvXOv2NGjWKjh07Ymtry/Dhw8nMzJTKhQMHDtCxY0fee+89bGxseOutt+jZs6fS8bt27eLtt9+mf//+WFlZ0bFjR8aNG8fhw4eVKsbt2rVj2LBhWFlZYW1tjYmJCTo6OhgaGmJiYtLo5pSoi7rcd4cOHaJly5aMHz8eOzs7WrZsycyZM0lJSeH69esYGhoqvU1/Wbpza2trM2PGDBwcHHBzc2PcuHEcOXLkqY2mixYtwsfHh7Fjx/Lf//4XT09PpZ4LHTt2pF+/ftjZ2WFra8uHH36Inp6e1KNVoWfPnkp5bMeOHaXG3169eqGtrc3Zs2el/aOioujXr98Lb6R8Ulp6WtqpzoMHD3j48GGd66xNmjTBxcWFnTt3kpeXR0VFBdHR0SQlJak0fP7555/4+Pgwbtw49u/fz8KFC5XS7XfffYebmxtdu3Z94m+ampo+cT6vunhaGV11qETTpk0xMTF5Yq8dRZk1btw4Zs2aRUZGBsOGDZO+37t3L2+//TZ9+/bFxsaGUaNG0a5dO5VJmgcOHEifPn2wtbVlwoQJmJubc+jQoSdey5PycE1WXb00LCysVuW4Jtb360oM33gG7dq1k1rli4qKOHToEMuWLWPZsmUqY6yMjIxwc3Pj5MmT0nwHV65ckR5+MjMz0dLSUqpQt2jRQhp79rIoKyt7apfDx3l6epKVlUVQUBDl5eU0adKEIUOGsH37drS0tCgrKyM4OBgfH58ndt2Wy+UYGxszbdo0tLW1ad26NUVFRfz444/4+PhID4mK8Km7Qai2BXpGRgZlZWXSw4lCeXk5LVq0AOD69evk5OSodPMqLS0lOztb+mxubq40H4KpqSn3799/1kt4IgcHB+n/Ojo6GBsbS79V2/DWh6r3sUJaWhpffvklgPR2qWpjl6urKzKZjKysLFq2bPnE82dlZSGTyZS6xBoaGipdP1Re844dO0hNTaWoqEh6aLlz5w6vvvoqJiYmdOvWjaioKFxcXKTeNH379n2u639eHTt2pKysjKSkJOkes7KywsTEhKysLPLz84mPj8fAwAAnJycyMzMpLy9XGsKlra2Ns7MzGRkZSud+2huqzMxMlYdhZ2fnZ1rZp+rfUVEJLSgoqPN5Gtrj6VVfX59vvvnmqb1lMjMzMTMzUyqbnJycVBrHYmNj2b9/P1lZWRQXF1NRUaH0JrFHjx58//33nD17lr59+xIVFYWTk5NKen6R6pIGmzZtWuv0VzVNKMrigoICXn31VTIzM1XeBLu4uHDs2DHp8/Xr10lJSWH37t3SNrlcTmlpKfn5+VI6q+ub2JdJXe6769evk5CQUO0KPFlZWc/cIKnpWrZsqdQw5eLigkwmIzs7+4nlz+zZs3FwcEAmk5Gens4PP/yAoaGhNHygoKCAyMhI4uPjyc/Pp6KigtLSUqnHYtXfq8rZ2VlquNDT0+ONN94gKioKd3d30tPTSUlJYcaMGfV1+bX2pLT0LGlHUQesa50VYObMmaxfv16qb7Zq1Qp3d3du3LihtF+HDh1YuXIl9+/f5+jRowQHB0tvtqOjo7l58yaBgYFP/T19ff16q7NmZ2fXOo+sDUWZVVpaypEjR8jOzmbIkCFA5VCfe/fuqczz4+rqqtI4VjUdamtr4+Tk9NTwPCkP12TV1UubNm3KqVOnnlqOa2J9v65Eo8QzMDAwkFaQAGjdujUffPABR44ckTL9qt544w1CQkKYPHkyp0+fxtzcnHbt2int87JPzGRkZFTnSYm0tLQYP348Y8eOJT8/H2NjY6k1z9LSknv37pGZmcm6detYt24dUFnxk8vljB49Gj8/P7p06YKJiQm6urpKD5e2traUlJRQWFgovYlVhK+hxlVZW1ujpaVFRkaGyps1qGxk0NHRkRpYtLS0VLqbVW0VVXz32WefqTSGKbrIyeVyHB0dq51YqOrESI83hGhpaT2xa+PzeNJv1Ta89eHx+xgq35DURn3dr8XFxSxbtoxOnToxc+ZMmjdvTmFhIf/973+VuuV5enry9ddfM2HCBI4dO0bPnj1f6MRW1bGwsKBFixbSG2lFnmZoaEjr1q2Jj48nPj4eV1fXOr85e5ErR1TtTqr4u2piN8/q0qti+/NKSkpi9erVeHt788EHH/DKK69w/vx5pZUAdHV16devH1FRUbz++utER0dL4/fVpS5psC73bNU0oVCXNFFRUYG3t7c0p0VVVcuX2vztFOVW1d+vz8lf1aUu951cLqdr167VjqF+Utd2TdSkSRMePnyosv3Bgwf1NlHyq6++KuUVdnZ2ZGdnExkZyXvvvYe+vj5r166loKCADz74gBYtWqCnp8fixYvrnK48PT2ZN28ed+7ckRrNa+oJ2pCelJaeJe0YGRmhpaX1TBNpWllZERAQQHFxMY8ePcLU1JTg4GCVF2eGhoZYWVlhZWWFi4sLs2fP5ujRo3h7e0tvxR8Pc3BwMC4uLixZskTaVlRUpBFzAVSnapn14YcfEhAQwI4dO+pt+eQned48XF1qKudrQxPr+3UlGiXqiba2do2tlT169CAkJIS//vqLkydP4u7uLmWctra2yOVyUlJSpBbDO3fuNIpuRnXh6OhY7VI3taGtrS21dMbExODi4oKxsTFNmzaV3morHDp0iEuXLjFv3jypEGjbtq00r4eignf79m0MDAyUZq9PT0/HzMyswbp+NmvWDDc3Nw4ePMjQoUOVKqQlJSUcPHiQ1157TaqYGBsbS8NUFG7evCn1grCzs0NPT4/c3Fw6duxY7W+2atWKmJgYjIyMGsVSiJoUXltbW/744w+ldHPt2jV0dXWxtLR86vFWVlbo6OiQnJws7V9cXEx6err0+datWxQWFjJ27FgpvT4+UStUDmVq2rQphw8f5s8//5QmQ1W3qmP6q07e1aFDB65cucLVq1cZOnQoUNmQqKurS2JiolToVlRUkJycjLu7e51+19bWVmXoQHJy8vNezkvJ1taWvLw87ty5IzVepqSkKFXQEhMTMTMzU5pkubouwZ6ensyZM4eDBw9SXFysEbP51zYN1lf6U6S9qhRDqxRat25NZmbmM1cuq1I8cFQtCzRxeFFDatWqFX/88Qfm5uaNfiZ6GxsbLly4gFwuV2oou3HjBjY2Nkr7pqWlUVxcLPWWSE5OrnX5U5W2tjbl5eXIZDJpfoOJEydK85rl5+erDC9Q/N7jeaytra302d7eHmdnZ44cOcLJkyerfSmnbs+SdnR1dbGzsyMjI0NlssvaMjQ0xNDQkKKiIi5evMj48eOfuL9cLpcahcaMGcPbb7+t9P28efPw8fFRmQQyPT1d5SXns6rPMro63t7eLF++nDfffBMzMzNMTU1JTExUWqb52rVrKg1bSUlJUh1X8bykSRP8vgi1Kcc1qf78rMScEs+grKyM/Px88vPzycjI4IcffqC4uLjGyX309fXp1asXv/zyCzdu3FCqONnY2ODm5sbGjRtJSkoiNTWVtWvXqnQbW7NmDWvWrGnQ62pIbm5uZGRkKE2klJWVRWpqKvfu3UMmk5GamkpqaqqUMd+/f59Dhw6RkZFBamoqoaGh/PHHH0yYMAGoLDgcHByU/hkbG6Onp4eDg4NUkA8cOJCioiI2b97MrVu3iIuLY9u2bQwcOFCpUpCQkCDN6NtQJk2aREVFBUuWLOHKlSvcuXOH+Ph4li5dio6OjrQiCVR2Tb5w4QLnz5/n1q1b/Pjjj0rdK5s0acLbb7/Nli1bOHbsmBSfhw4d4siRI0BlL53mzZsTFBTE1atXycnJ4erVq/z000/cvn27Qa/1WWhSeL28vLh37x6bNm0iIyODv/76i61btzJo0KBaveE0NDRkwIABbN26lUuXLpGens769euVWqTNzc3R09Pj999/Jzs7m7/++ovIyEiVc2lra9O/f3/Cw8MxMzNTKsTVqUOHDiQnJ5OcnCxNZgnQvn17Tp8+Lc3sDJXxMXDgQLZu3cpff/1FRkYG3333Hfn5+dIKO7U1ZMgQTpw4wZEjR7h9+za7du0iJSWlXq/tZdGpUydsbW1Zu3YtqampJCUl8eOPPyq9SbK2tiYvL4+TJ0+SnZ3NoUOHlOaTULCxscHV1ZWwsDB69eqlEUvg1jYN1lf6Gzx4MJcvX2bXrl3cvn2bI0eOcO7cOaV9hg8fTkxMDJGRkaSlpZGZmUlsbGydJilU0NfXx9nZmd27d5Oenk5iYqJSD5a/Ay8vLx4+fMjq1atJTk4mOzubS5cuERISwqNHj9QdvDoZOHAg2dnZ/PDDD6SmpkpzZcXExKhMbFpeXs769etJT0/n0qVLhIeH4+np+dS5RgoLC8nPz+fu3btcuHCBAwcO0KFDB+l+tba25uTJk2RkZJCSksLXX39d7QP72bNnlfLYK1euSN3vFTw9PdmzZ4/GNFI+7lnTTpcuXVQmp6xaT1UMxUpNTVUaNhgXF8eFCxfIycnh0qVLBAQEYGtrK61Q8fDhQyIiIkhOTubOnTtcv36ddevWcffuXalnlZmZmUrdFirrC1UbpEpKSrh+/Xq1q/88i/oso6vToUMH7OzspAmU33nnHfbu3cupU6e4desWkZGRJCQkqDTIHD58mNjYWG7dusXmzZu5c+cOAwcOfO7waKKqz5eKf/fv369VOa5J9edn1bibnNXk8uXLTJ06Fah8MLSxsWHOnDlKFaLH9evXj+PHj9OqVSuVVsAZM2YQEhJCQEAAxsbGeHt7q4zveXysX2Pj4OCAk5OT0pKbGzZsUJo1+NNPPwUqG2AUb41PnDghVcBcXFzw9/ev8/hRc3NzFixYwE8//cQnn3yCiYkJ/fv3V1qto7S0lLNnz7JgwYLnus6nsbCw4IsvvmD79u18++235OfnI5fLcXV1JSgoSKmLVf/+/bl58ybr168HKgvXnj17KjXsjBo1iubNm7N37142bdokzeCsmEzIwMCAgIAAwsPDWbVqlTR5U4cOHerUkqpYZmnRokVPTOfPq77CWx/MzMzw8/MjLCyMTz/9lFdeeQV3d3fGjBlT63P4+PhQUlLCypUrMTAwYNCgQUqTZBobG+Pr68vPP//MwYMHcXBw4P3331eZJwQq08OOHTvo37+/xgz36tChAzKZTKm7MFSOCy0tLaVJkyZKcx6MGzcOQFqvvFWrVixYsKDOE4r16dOH7OxsIiIiKCkpoUePHgwdOvSZe2O9zLS1tZk3bx4hISH85z//kZYSU6zwApW9+d555x02b95MaWkpXbp0YdSoUWzatEnlfAMGDCAhIUHtE60q1CUN1kf6c3FxYdq0adIylB06dGDEiBH88MMP0j5ubm7Mnz+fX375hb1796Kjo4O1tXWNS+c9zfTp0wkJCcHPzw9LS0smT56sNCnvy87MzIwlS5YQHh7O8uXLKS0txdzcnC5duqisdKDpLC0tCQgIIDIykmXLllFaWoqtrS1z5sxRmdSwffv22NvbExAQQElJCb169XrqG3dAKj+0tbUxNTWla9euSuXW9OnT2bhxI5999hlmZmaMGDGi2jHlI0aM4MyZM4SGhmJsbMz06dNV6l99+vQhNDSU3r1706RJk2eJkgb1rGnH09OTTz/9lKKiIqlelpeXJ9VToXIOhiNHjtC+fXv8/f2BykaHn3/+mbt379KsWTN69erFmDFjpEYfHR0d0tPTiYqKorCwECMjI9q0aUNAQMBT56l63Llz56odDv486quMrsnbb7/NunXrGDZsGIMHD+bRo0ds3bqV/Px8bGxsmDt3rtIqJABjx45l37593LhxA3Nzc+bNm6fxc0M8q6rPlwpmZmZs2LDhqeW4JtWfn5WWvDEMshFeCnFxcYSGhhIcHFzjShnq8vvvv3P+/Hk+//xztfz2Tz/9xMcff0yPHj1e+O/XRlRUFOHh4axevbrRZG4vm+TkZBYuXMiaNWtU5hARhBfl119/JSoqSqkyJAhC/Vq7di2FhYXMnz9f3UF5ory8PGbMmIG/vz+urq7qDk69Wr16NXZ2dkrD2jSJn58fQ4cOVfuk14JQXzTryVB4qbm5ueHl5SUtdahJdHV1lYZOvEiDBg1i5syZpKenq33lj5pcuHCBcePGiQYJNSgrK+Pu3btERkbSs2dP0SAhqIViPpTffvuNwYMHqzs4giCokUwmIz8/n59//llaovBlM378eI0YoladgoICevfuXS9zPQiCphA9JQRBEDTY8ePHWb9+PY6OjnzyySeiUUJQi7Vr1xITE0OPHj3497//Xe3s5oIg1A9N7ymhGNJpbW3NnDlzVLrcC4Ig1JVolBAEQRAEQRAEQRAEQS3E8A1BEARBEARBEARBENRCNEo8QVFREVOmTFFa7kdT+Pn5ERsbq+5gCGqiyWlz1apV7N27V93BqBMRn4Km0uS0+Xcth3x9fdmzZ4/0OT8/n6VLl+Lj48PIkSPVGLIXT5PTZ2PLO0VcCn8H//vf/xg5ciQ5OTn1el5/f3++//77ej1nQxL3uyqxJOgT7Nq1i65du0rLjIWGhpKYmEh6ejomJiasXbtW5ZjTp09L65cbGxszaNAgpbWnFePwHhccHIytra30OTY2lsjISLKzs7G0tGTMmDH07NlT+n748OH89NNP9OzZU+NWshAanrrSZnp6Otu2bePGjRvk5OTg7e2tUgn39vZm0aJFeHp6auwkUY9TV3z+8ccf7N69m6ysLMrLy7GysmLo0KFKSwdqanxWVFSwbds2Tp48SX5+PiYmJrzxxhuMGDFCmm9ALpezfft2jh49SlFREc7OzkyaNAl7e/saz6tYahEql7Rr0qQJtra2dO/encGDB2NoaPhCrk9TNETarOratWv4+/tja2vLV199pfTd37kcetKY/sDAQAwMDKTPe/bs4d69ewQFBWnksogNSV155/Hjx1m3bp3KPmFhYejr6wOam3fWpCHi8urVq4SHh3Pr1i1KSkpo0aIFAwYMUMkPDhw4wKFDh8jNzcXIyIgePXowfvx4Kb9tbHEp1KymvO1///sffn5+rFmzBgsLCzWFTtXatWufutz3tm3bXlBo6o/IO1WJRokalJSUcOzYMT777DNpm1wux8PDg7S0NC5duqRyzIULF/jmm2+YOHEibm5uZGZmEhISgr6+PoMGDVLad9WqVdLaxwDGxsbS/5OSkli9ejUjR46kZ8+enD17llWrVrFkyRKcnZ0B6NatGyEhIcTFxdGtW7f6vnxBg6kzbSoqNb169SIiIqLa8Dk4OGBpaUl0dLTKuTWROuPTyMiI9957D1tbW3R0dPjrr7/YsGEDxsbG0n2tqfH566+/cvDgQXx9fXFwcCAtLY21a9eiq6srLaG2e/du9u3bx4wZM7CxsWHHjh0sXbqU1atXP/HhzcbGBn9/f+RyOUVFRVy7dk1ainLx4sWYmJi8qMtUq4ZOm0VFRaxZs4ZOnTqRl5en9J0oh2pW9R4GyMrKolWrVlhbW6spROqhzrwTwMDAgG+//VZpm6JSDZqbd1anoeLS0NCQwYMH4+DggIGBAdeuXeO7777DwMAALy8vAE6dOkVYWBjTpk3D1dWVnJwc1q9fT1lZGdOnTwcaV1wKL5eJEycybtw46fOsWbMYM2YMffr0UWOono/IO6snGiVqcOHCBQDatm0rbVMsGblnz55qE0x0dDTdu3eXMnpLS0veffdddu/ejZeXF1paWtK+xsbGKolEYf/+/XTo0IH33nsPADs7O+Lj49m/fz8fffQRUPkGsWvXrpw6depvVRkU1Js2nZyccHJyAipbeWvSo0cPYmJiGkXlRZ3x2bFjR6XPQ4YM4cSJE1y7dk3pvtbE+ExKSqJ79+706NEDAAsLC7p3705KSgpQWcAeOHCAd999l969ewMwc+ZMJk+ezKlTp3jrrbdqPLeOjo7U8GBqaoq9vT09evRg7ty5hIWFMXPmTADi4uLYuXMn6enpQGX6/OCDD7CzswMgICAAOzs7Jk2aJJ374cOHTJ06lVmzZtGrVy/OnDnD9u3buX37Nvr6+jg4ODBnzhyNaPho6LS5YcMGPDw8kMvlnDlzRuk8ohyqma+vL15eXrzzzjv4+vqSm5sLVMa9h4cHvr6+PHz4kC1bHeumAQAAF5tJREFUtnDu3DlKS0tp1aoV77//Pm3atFFz6OuPOvNOhafdp5qYd1anoeKydevWtG7dWjrGwsKCs2fPkpCQIB2XmJiIs7Mz/fr1k/bx8PBQyRMaS1wK9efq1auEhYVx8+ZNmjZtiru7O+PHj0dXt/LxsaysjK1btxITE8PDhw9xdHTEx8dHaYnYuLg4Nm/eTG5uLm3atGHgwIF1CkPTpk1V3tY3bdq02ntfLpcTHh7O0aNH0dLSol+/fowfP17qySeTyYiIiODUqVMUFRVhb2/PqFGjcHNzq2vUPBeRd1bv5epvWY8SEhJo3bq10h/5acrKytDT01Papq+vz927d6VKi4Kfnx9Tp05l8eLFXLlyRem7pKQkunTporStS5cuJCUlKW1zcnIiISGh1uETXg7qTJu15eTkREpKCqWlpc90/IukKfEpl8u5fPkyt27dol27dkrfaWJ8urq6Eh8fT2ZmJgAZGRnEx8fTtWtXAHJycsjPz6dz587SMfr6+rRr147ExMQ6/56pqSl9+/bl3LlzVFRUAFBcXMyQIUNYvnw5/v7+NGnShBUrViCTyQDw9PTk1KlTlJWVSeeJiYnB0NCQ7t27k5+fz+rVq/Hw8CA4OJiAgACpYq4JGjJtHjx4kIKCAoYPH17teUQ5VDuBgYF06tSJ119/nY0bNzJx4kTkcjmBgYHk5eUxf/58goKCaNeuHYsXL+bevXvqDnK9UXfeWVpayowZM5g2bRpffPEFN27cUNlHE/PO6jR0XCrcuHGDxMRE2rdvL21zdXUlNTVVurfv3LnD+fPnpbxcobHEpVA/8vLyCAwMxNHRkRUrVjBt2jRiYmIIDw+X9gkLC+P06dNMnz6dFStWYG9vz7Jly6R87s6dO6xcuZLOnTsTFBTEoEGDCAsLa7Awnzx5Eh0dHZYsWcKHH37IgQMHOH36tPT9unXrSEhIYPbs2Xz11Vd4eHiwYsUKUlNTGyxM1RF5Z/VET4ka5ObmYmpqWqdj3Nzc2Lx5MxcvXqRTp05kZWWxb98+oHIiLAsLC0xNTZk8eTJOTk7IZDKio6NZsmQJ/v7+0oNIfn4+zZs3Vzp38+bNyc/PV9pmZmZGXl4e5eXlYs34vxF1ps3aMjU1pby8nLy8PGm8nKZSd3w+fPiQf/3rX8hkMrS1tZk0aZJKZVAT43PYsGE8evSIjz/+GG1tbcrLy3nvvfekVnxFfvV4a3zz5s2f+cHMzs6OR48eUVhYSPPmzaUeGAozZszggw8+ICUlBVdXV3r16kVoaChnz57F3d0dgKioKPr164eurq6Uf/bu3ZsWLVoAld0WNUVDpc20tDR27NjBsmXLapwLQpRDtWNsbIyenh76+vpSWr9y5Qqpqal8//33UpfY0aNH8+effxIdHc2wYcPUGeR6o86808bGhunTp+Po6MijR484cOAACxcuZOXKlUrDaDQx76xOQ8WlwrRp07h//z7l5eWMGDFC6W21u7s7hYWFLFq0CIDy8nL69eun1GUeGk9cCk8XFxeHj4+P0ja5XK70+eDBg9K9qK2tjZ2dHePGjWPjxo2MGjUKuVzOoUOHmDZtmtRTburUqcTHx3Pw4EFGjx7NoUOHMDc3Z+LEiWhpaWFra8vt27eJjIxskOuys7Nj1KhRQGUecfToUa5cuULfvn3JysoiJiaGtWvXYm5uDsCgQYO4dOkSR44cYfLkyQ0SpuqIvLN6olGiBmVlZUrja2rD09OTrKwsgoKCKC8vp0mTJgwZMoTt27dLrWE2NjbY2NhIx7i4uJCbm8uePXvq/OCnr6+PXC6nrKzsb1kZ/LtqLGkTaBRvVNQdn4aGhqxcuZLi4mIuX77Mjz/+SIsWLejUqZO0jybG5+nTp4mOjmb27NnY29uTmppKaGgoFhYWDBgwoEF/WxHHWVlZREZGkpKSwv3796moqEAul3Pnzh0A9PT0eOONN4iKisLd3Z309HRSUlKYMWMGAI6OjnTq1Im5c+fSuXNnOnfuTO/evZ/a7fFFaYi0WVZWRnBwMD4+PvUymZkoh1Rdv36d0tJSpWFDUPn3zM7OVlOo6p86804XFxdcXFykfdq2bcsnn3zCb7/9JnWDBs3MO6vTUHGpsHjxYoqLi0lKSmLr1q1YWFhIvcKuXr3KL7/8wuTJk3F2diYrK4vQ0FC2bdsmPeBB44lL4enatWvHv/71L6VtaWlpfPnll9LnzMxMnJ2dlRquXV1dkclk0ooR5eXlSkMQtLW1cXZ2JiMjQ+kcVdNj1fu2vrVs2VLps6mpKQUFBUBlLyG5XM6cOXOU9pHJZCpDaRuayDurJxolamBkZERRUVGdjtHS0mL8+PGMHTuW/Px8jI2NuXz5MlA59qcmTk5OSt2LTExMpJtIoaCgQOWNY1FREXp6en+72ej/7tSZNmtLET5Nebh7EnXHp7a2ttQK7ejoSGZmJrt27VJqlNDE+AwLC+Ptt9+WeiA4ODiQm5vLrl27GDBggJRf5efnS28loDIve/wNfG1lZGTQpEkTaQKnFStWYGZmxpQpUzAzM0NHR4ePP/5YGr4BlQX5vHnzuHPnDlFRUbi4uEhzTmhra/P555+TnJzMxYsXOXbsGOHh4fj7++Po6PhMYaxPDZE27927R2ZmJuvWrZNm4JbL5cjlckaPHo2fnx9dunQR5dBzqKiooHnz5ixevFjlu5dpdQ51551VaWtr06ZNG5Xl9TQx76xOQ8elogHSwcGBgoICtm/fLjVKRERE4O7ujqenp7RPcXExISEheHt7S42NjSUuhaczMDBQefv94MGDWh+vpaWl0rNCEzzeMF41nHK5HC0tLQIDA6U5MRTq2kDwvETeWUNYXtgvNTKKh4Nnoa2tjZmZGbq6usTExODi4vLEP2pqaqpSRc/FxUVlkpNLly6ptC6mpaUpTWAk/D2oM23WVnp6OmZmZhoxWeDTaFp8VlRUKM2BAJoZnyUlJSpd/7W1taUKgIWFBSYmJkp5WWlpKdeuXVN6s1Jb9+7d49SpU/Tq1QttbW0KCwvJzMzkn//8J507d5aGdpSXlysdZ29vj7OzM0eOHOHkyZP0799f6XstLS1cXFwYMWIEgYGBmJqaPlNDXENoiLRpZmbGl19+SVBQkPTvrbfewsrKiqCgIOlvI8qhZ9e6dWsKCgrQ0tLCyspK6d+zNshpIk3KO+VyOTdv3lTZRxPzzuq8yLhU9GxSeFpertBY4lKoH7a2tiQnJ0tzOEHlEtK6urpYWlpiaWmJrq6u0hxRFRUVJCcnSw3/inNUTUvJyckv7iKqcHR0RC6Xk5+fr5Ivm5mZvfCwiLxTlegpUQM3Nze2bt1KYWEhRkZGQGVX4eLiYu7du4dMJpMmRrGzs0NXV5f79+8TGxtL+/btkclkREVF8ccffyitGbt//35atGiBvb09MpmMkydPcu7cOebOnSvtM2TIEBYtWsSvv/7Ka6+9xtmzZ4mPj1d563Lt2jWViciEl58606ZMJpO65ZWWlpKfn09qaiqGhoZKre4JCQmNJm2qMz537tyJk5MTlpaWlJWVceHCBU6ePMnEiROVwqiJ8dm9e3d+/fVXLCwssLOzIzU1lX379uHh4QFUPuwPGTKEXbt2YWtri7W1NTt37sTQ0JC+ffs+8dzl5eXk5+dLS4ImJiaya9cumjVrxtixYwF45ZVXMDIy4ujRo5ibm5OXl8eWLVuqHULg6enJd999h46OjtIyYklJSVy+fFnqGXDjxg3u3r0rVajUrSHSpq6ursq8GYp5EapuF+UQPHr0SGUCtNqs2d6pUyfatm1LUFAQ48ePx9bWlvz8fOLi4ujUqVOdh8NpKnXmndu3b8fZ2Rlra2tpXHRaWhpTpkxRCqMm5p3Vaai4/O2337CwsJC6dCckJLB3716lOSW6d+/O/v37adOmjTR8IzIykm7duinlp40lLoX64eXlxYEDB9i0aRNDhgwhJyeHrVu3MmjQIAwMDAAYOHAgW7duxcjICAsLC/bv309+fr40t9TAgQPZt28fmzdvxsvLi7S0NA4fPqyW67GxsaFv376sW7eO999/n1atWlFUVER8fDyWlpb06tXrhYVF5J3VE40SNXBwcMDJyUlpOZQNGzZw9epVaZ9PP/0UgDVr1khd406cOMGWLVuAyjdN/v7+0hKKUPlQFxYWxt27d9HX18fe3p758+crLafWtm1bPvroIyIiIoiMjMTKyoqPPvpIWhseKmfFTUxMZNasWQ0XCYJGUmfazMvLk84NkJ2dzZEjR2jfvj3+/v5AZWPF2bNnWbBgQcNEQD1TZ3wWFxezadMmaR9bW1t8fX2VHto1NT4//PBDIiMj2bRpEwUFBZiamuLp6Ym3t7e0z7BhwygtLeX777/nwYMHODk5sWDBgqd2Yb916xZTp05FS0uLpk2bYmNjg6enJ4MHD5aO1dbWZs6cOYSGhjJ37lysrKzw8fHhq6++Ujlfnz59CA0NpXfv3kq/3bRpUxITE/n999958OABr776KsOHD9eYFTgaKm3WhiiHKitlVfM7oFYVVy0tLfz8/IiIiCAkJEQa9tK2bVuNSVv1QZ1554MHD9i4cSP5+fk0bdqUVq1aERAQoHQeTc07q9NQcVlRUcHWrVvJzc2VhgqOHTtWaUnm4cOHo6WlRWRkJHfv3sXY2Jju3bszevRoaZ/GFJdC/TAzM8PPz4+wsDA+/fRTXnnlFdzd3RkzZoy0j2Iy1PXr1/PgwQNatWrFggULpEkczc3NmTdvHj/++CNHjhyhdevWjB07lm+//Vbpt0aOHIm3tzcjR45s0GuaMWMGO3fulPKXZs2a4eTk9MLnlBB5Z/W05Jo4KEhDxMXFERoaSnBwcI0zlKvLli1bpFn7hb8fTU6bv//+O+fPn+fzzz9Xd1BqTcTnyy0vL48ZM2bg7++vtH56Y6DJaVOUQ4Imp8/GlneKuBT+jnJycpg1axYBAQGNrnx+HuJ+V6Xjr3i9KaiwsrJCLpdjamrKK6+8ou7gKLl58yZDhw4Vk4v9TWly2kxNTWXgwIFSl7TGQMTny0kmk3H//n3Cw8PR0dFRevPXWGhy2hTlkKDJ6bOx5Z0iLoW/o5MnT2JsbMyQIUPUHZQXStzvqkRPCUEQBOGlFB8fT0BAANbW1syZM0cjVtQQBEEQBEEQlIlGCUEQBEEQBEEQBEEQ1EKzBrEIgiAIgiAIgiAIgvC3IRolBEEQBEEQBEEQBEFQC9EoIQiCINSroqIipkyZQlZWlrqDomLVqlXs3btX3cEQ1ESkTUEQhLoTeafQ0HTVHQBBEATh5bJr1y66du2KlZUVAKGhoSQmJpKeno6JiQlr165VOeb06dPs2rWL27dvY2xszKBBg3jnnXek7xWTVj4uODgYW1tb6fPDhw+JiIjgzJkzFBYW8uqrrzJmzBj69OkDgLe3N4sWLcLT05OmTZvW96ULGq4h0mZV165dw9/fH1tbW7766iul70TaFAShsWqIvPPq1auEh4dz69YtSkpKaNGiBQMGDFDJXw8cOMChQ4fIzc3FyMiIHj16MH78eGnlJ5F3vhxEo4QgCIJQb0pKSjh27BifffaZtE0ul+Ph4UFaWhqXLl1SOebChQt88803TJw4ETc3NzIzMwkJCUFfX59BgwYp7btq1SqaNWsmfTY2Npb+L5PJWLp0Kc2aNWPOnDmYmZmRl5eHru7/FXUODg5YWloSHR2tcm7h5dbQabOoqIg1a9bQqVMn8vLylL4TaVMQhMaqofJOQ0NDBg8ejIODAwYGBly7do3vvvsOAwMDvLy8ADh16hRhYWFMmzYNV1dXcnJyWL9+PWVlZUyfPh0QeefLQgzfEARBEOrNhQsXAGjbtq207cMPP2Tw4MFYW1tXe0x0dDTdu3fHy8sLS0tLunXrxrvvvsvu3bt5fIEoY2NjTExMpH/a2v9XjB0/fpz79+/z6aef4urqioWFBa6urjg5OSmdo0ePHsTExNTXJQuNREOnzQ0bNuDh4YGzs7PKeUTaFAShsWqovLN169a4u7tjb2+PhYUF/fr1o0uXLiQkJEjnSUxMxNnZmX79+mFhYUHHjh3x8PAgJSVF6fdE3tn4iUYJQRAEod4kJCTQunVrtLS0an1MWVkZenp6Stv09fW5e/cuubm5Stv9/PyYOnUqixcv5sqVK0rfnTt3jrZt2/LDDz8wZcoU5syZw7Zt25DJZEr7OTk5kZKSQmlpaR2vTmjMGjJtHjx4kIKCAoYPH17teUTaFAShsWrocl3hxo0bJCYm0r59e2mbq6srqampJCUlAXDnzh3Onz9P165dlY4VeWfjJxolBEEQhHqTm5uLqalpnY5xc3Pj/PnzXLx4kYqKCm7dusW+ffsAyM/PB8DU1JTJkyczd+5c5s2bh42NDUuWLFF6o5KdnU1sbCwymQw/Pz9GjRrF4cOHCQ8PV/o9U1NTysvLVbrYCy+3hkqbaWlp7Nixg1mzZin13KlKpE1BEBqrhso7FaZNm8bYsWOZP38+Xl5eDBw4UPrO3d2dMWPGsGjRIsaMGcOMGTNwcHBg3LhxSucQeWfjJ+aUEARBEOpNWVkZ+vr6dTrG09OTrKwsgoKCKC8vp0mTJgwZMoTt27dLb2ZsbGywsbGRjnFxcSE3N5c9e/bQrl07oHKMq7GxMdOmTUNbW5vWrVtTVFTEjz/+iI+Pj3QuRfjEG5W/l4ZIm2VlZQQHB+Pj44OFhUWN5xFpUxCExqqhynWFxYsXU1xcTFJSElu3bpWGckDlZJi//PILkydPxtnZmaysLEJDQ9m2bRujRo2SziHyzsZPNEoIgiAI9cbIyIiioqI6HaOlpcX48eMZO3Ys+fn5GBsbc/nyZQAsLS1rPM7JyYnTp09Ln01MTNDV1VV6W21ra0tJSQmFhYXSpJiK8FWdJFN4+TVE2rx37x6ZmZmsW7eOdevWAZUNEHK5nNGjR+Pn50eXLl1E2hQEodFq6HJd0aDr4OBAQUEB27dvlxolIiIicHd3x9PTU9qnuLiYkJAQvL290dHRAUTe+TIQjRKCIAhCvXF0dOTEiRPPdKy2tjZmZmYAxMTE4OLi8sQKRmpqKiYmJtLntm3bEhMTQ0VFhfTwd/v2bQwMDDAyMpL2S09Px8zMTOlY4eXXEGmzadOmfPnll0r7Hjp0iEuXLjFv3jypsi3SpiAIjdWLLNflcjllZWXS55KSEpVhcdra2ioTDYu8s/ETjRKCIAhCvXFzc2Pr1q0UFhZKD1tZWVkUFxdz7949ZDIZqampANjZ2aGrq8v9+/eJjY2lffv2yGQyoqKi+OOPPwgICJDOu3//flq0aIG9vT0ymYyTJ09y7tw55s6dK+0zcOBADh48yObNmxk0aBA5OTls27aNgQMHKnUXTUhIoEuXLi8mQgSN0RBpU1dXFwcHB6XfMTY2Rk9PT2m7SJuCIDRWDVWu//bbb1hYWEhDMxMSEti7d6/SnBLdu3dn//79tGnTRhq+ERkZSbdu3aReEopjRd7ZuGnJH29qEgRBEITnsGDBAt544w1pvXB/f3+uXr2qst+aNWuwsLDg/v37rFixgrS0NKByvojRo0crLa24e/dujh49yt27d9HX18fe3p53332Xbt26KZ0zKSmJn376iRs3bmBiYkK/fv0YPnw4urqVbfClpaVMmTKFBQsW4OLi0lBRIGiohkibj9u2bRtnzpzhq6++Utou0qYgCI1VQ+Sd+/fv5+jRo+Tm5qKtrY2VlRUDBgzgrbfeknpHlJeXs3PnTk6ePMndu3cxNjame/fujB49mmbNmgEi73xZiEYJQRAEoV7FxcURGhpKcHBwjasRqMvvv//O+fPn+fzzz9UdFEENRNoUBEGoO5F3Cg1Nx9/f31/dgRAEQRBeHlZWVsjlckxNTXnllVfUHRwlqampDBw4UGkcv/D3IdKmIAhC3Ym8U2hooqeEIAiCIAiCIAiCIAhqoVn9bwRBEARBEARBEARB+NsQjRKCIAiCIAiCIAiCIKiFaJQQBEEQBEEQBEEQBEEtRKOEIAiCIAiCIAiCIAhqIRolBEEQBEEQBEEQBEFQC9EoIQiCIAiCIAiCIAiCWvx/UODBlE7aE4YAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":676},"id":"Z8sdZC7-wwdh","executionInfo":{"status":"ok","timestamp":1633623150138,"user_tz":-330,"elapsed":1393,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"2288028a-5149-4e5b-9380-9c12e72378ad"},"source":["plot_components(components_df, 'fc1', ascending=False)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABCUAAALlCAYAAADzMFwcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVyVdf7//+eRI6Agi+JxAcU1t3JJEVMLV3AL0/ik6TSYptlt0JxuMxY1JqkljjOl5TT5s8Q117Fc0lxwKTMrLUVzTy0RNwTcRYXr90dfTh45oByOXi6P++3WrXhf7+u6Xud9ruvEeXJd78tiGIYhAAAAAACAO6yE2QUAAAAAAIAHE6EEAAAAAAAwBaEEAAAAAAAwBaEEAAAAAAAwBaEEAAAAAAAwBaEEAAAAAAAwBaEEANyn+vXrJ4vFosOHD5tdSrG58lrWr18vi8WihISEYu//8OHDslgs6tev3y2vM23aNFksFk2bNu2O7/teM2fOHDVp0kRlypSRxWLRsGHDzC4JcKsH4TwGAFcRSgCACywWiywWi0qUKKFffvmlwH5t27a19y3ul1M44pd8c7gaduWFNOvXr3do//bbb9W3b1+dO3dOL730kkaOHKlOnToVua7vv/9e8fHx6ty5sypWrCiLxaKQkJAC++cdP+4IrXD3uJ/CWAB4UFjNLgAA7lVWq1XXrl3TJ598onfeeSff8v3792v9+vX2fnfa2LFj9dprryk4OPiO79vd7sXX0qNHD7Vo0UKVKlUyu5S72hdffCHDMDRjxgy1bNnS5e18+umnmjhxokqWLKn69evrxIkTbqwSAADcLlwpAQAuqlChgpo1a6akpCSnocPHH38sSXryySfvdGmSpEqVKqlu3boqWbKkKft3p3vxtfj7+6tu3bry9/c3u5S7WlpamiSpcuXKxdpOv3799OOPP+r8+fPatm2bO0oDAAB3AKEEABTDwIEDdfz4cS1btsyh/erVq5o2bZpatmyp+vXrF7j+/v379ec//1nBwcHy9PRU5cqV9ec//1n79+936Dd48GBZLBYtXrzY6Xa+++47WSwWxcTE2NsKu4z5u+++U0xMjCpWrChPT09VqVJFL774ov0L4vUOHjyoQYMGqVatWipVqpTKli2rRx55RIMHD9bp06cLGx5Jv3/ZdHaFQ2hoqCwWi0aPHu3QvmLFClksFr355psFvpaEhARVr15dkjR9+nT7LTIF3Sazbds2de3aVQEBASpdurQiIiK0adOmm9buzOHDh9W7d28FBQXJ29tbzZo1y/f+S4XPKbFy5Uq1atVKPj4+Klu2rJ566int2bPnppee3+q+88yZM0dt27ZVQECAvL29Va9ePY0ZM0bZ2dn5+n799dd68sknFRISIi8vL1WsWFEtWrTQW2+9Ze9jsVg0ffp0SVL16tXtY16tWrXCB82JvPFJSkrKt73rX39qaqqGDh2q2rVr24+/5s2b5ztuGjdurCZNmsjT07PItRTFvHnz1L59e5UtW1be3t6qVq2ann32WW3ZssWhX3Z2thITE/XII4+odOnS8vPz0+OPP6758+fn2+b1tyL98ssviomJUbly5VSmTBlFRkZq586dkqRTp05p0KBBqlSpkry9vRUWFqZ169bl215CQoL9Vpnp06erSZMmKlWqlGw2m/r376/jx487fW23+nl04z4WLlyo5s2bq3Tp0ipbtqx69+6to0ePOt1HRkaG4uPjVa9ePZUqVUr+/v5q3769Vq1ala/v9efQunXr1KZNG5UpU0Z+fn7q2rWrdu/e7dDf1eMzMTFRFotFEydOdLo8LS1NVqtVzZo1c2gbNWqUWrVqZf8crVy5svr06aNdu3YVur/rtWnTRhaLxemywj5DUlNTFRcXpxo1asjLy0vlypVTdHS0fvjhh3x9z507p9GjR+vhhx+Wn5+fypQpo5o1a6pXr17aunXrLdcKALcLt28AQDE8++yzeuWVV/Txxx/rqaeesrcvWbJEJ0+e1Lhx43TgwAGn6/7www/q0KGDzp07p+joaNWvX1979uzRrFmztHjxYq1Zs0ZhYWGSpNjYWE2ePFkzZsxQ9+7d820r7xfxW5lfYerUqRo0aJC8vLwUHR2tKlWqaP/+/fr444+1dOlSbd68WVWrVpUkHTt2TGFhYTp79qy6dOmip59+WpcvX9ahQ4c0c+ZMxcXFqVy5coXur127dpo9e7b27NmjunXrSpIOHDig3377TZKUnJysESNG2PsnJydLktq3b1/gNtu0aaOsrCxNnDhRjRo1chj7xo0bO/TdsmWL/vnPf+qxxx7TCy+8oN9++03/+9//1L59e23btk116tS56Zjl+fXXX9W8eXPVqFFDzz33nDIyMjRv3jx1795da9asUdu2bW+6jblz56pPnz7y9vbWM888o0qVKmnTpk167LHH1KhRI7ftu3///kpKSlJISIiefvppBQQEaPPmzRoxYoSSk5O1evVqWa2//xrw5ZdfqmvXrvLz81N0dLSCg4OVkZGh3bt368MPP9TIkSMlSSNHjtTnn3+u7du36+WXX1ZAQIAk2f9dFI0bN77p9rZs2aKoqChlZGToiSeeUM+ePXXx4kXt2rVLCQkJDsfN7WYYhp5//nlNnz5dQUFB6tmzp8qXL6/U1FStW7dOderUsX9pvXLliqKiorRhwwbVrVtXf/nLX3Tx4kUtXLhQvXr10rZt25ze8nX48GGFh4erXr166tevnw4fPqzPPvtMbdq00bfffqtOnTrJz89PvXr1UkZGhubOnavOnTtr37599nP2eu+9955WrVqlXr16qVOnTtq4caOSkpK0fv16fffddypfvry9b1E+j6734YcfasmSJYqOjlZERIS+++47zZs3T9u3b9e2bdvk5eVl7/vrr7+qTZs2Onz4sB5//HF16tRJFy5c0LJly9SpUydNnjxZAwcOzLePZcuWafHixercubMGDx6sXbt2afny5frhhx+0a9cuBQUFSXL9+Hzuuef0xhtvaMaMGXr55ZfzLZ81a5ZycnIcPl+/+uorJSYmqm3btnr66afl6+ur/fv3a+HChVqyZIm++eabQs/n4vjxxx8VGRmpjIwMRUVFqWfPnkpPT9fnn3+u1q1b67PPPlOXLl0k/X7cdurUyf4Z88ILL8hqtdqP28cff1xNmza9LXUCwC0zAABFJskIDg42DMMwBgwYYHh4eBhHjhyxL4+KijL8/PyMCxcuGG+88YYhyUhKSrIvz83NNerWrWtIMmbNmuWw7blz5xqSjDp16hg5OTn29oceesjw9PQ0Tp8+7dD/8uXLRmBgoGGz2YyrV6/a22NjYw1JxqFDh+xte/fuNUqWLGnUrFnTSE1NddjOmjVrjBIlShhPPfWUve399983JBkTJkzINwbnz583Ll68eNOx+uSTTwxJxqRJk+xtH330kSHJ6Nixo+Hp6WlcuHDBvqxx48ZGqVKljOzs7EJfy6FDhwxJRmxsrNP9rlu3zpCUb+yv3/9LL7100/qv35ckIyEhwWHZl19+aUgyOnfu7NCelJSUb99nz541AgICDE9PT2Pbtm0O/V999VX7Ppy9Tlf23aNHj3zv0ciRI/O9pz179jQk5avJMAzj1KlTDj87ey+Ko6DtZWdnG9WqVTMkGbNnz8633vXnmzPXn6PuMHnyZEOSERYWZmRlZTksu3btmpGWlmb/+Z133rG/L9efkydOnDBCQ0MNScY333xjb7/+PR4zZozDtkeNGmVIMgIDA40XX3zR4TNhxowZhiRj2LBhDuvkvcclS5Y0fvzxR4dlw4YNMyQZ/fv3t7e58nmUt48yZcoYKSkpDus8++yzhiRj3rx5Du0RERGGxWIx5syZ49CemZlpNGrUyPD29jaOHz9ub887jj08PIw1a9Y4rPPaa68Zkoxx48Y5tLt6fEZGRhqSjB07duRbVr9+fcPT09NIT0+3t504ccI4e/Zsvr7btm0zfHx8jE6dOjm0F/R5FRERYRT067izz5CrV68aNWvWNLy8vIz169c79D969KhRuXJlo2LFisbly5cNwzCMlJQUQ5LD53qenJwcIyMjw+m+AeBO4vYNACimgQMHKicnR1OnTpX0+18DV69erb59+6p06dJO19m0aZP27Nmjxx57TH379nVY1qtXL7Vu3Vp79+7Vxo0b7e2xsbG6cuWK5syZ49B/6dKlyszMVN++fe1/+S7If//7X129elUTJ07Md0tF+/btFR0draVLl+rcuXMOy0qVKpVvWz4+Pk7bb5R3xUPeFRB5/12hQgUNHTpUV65csb/O06dPa/v27WrdurXbLsNv1apVvitI+vfvL6vVqu+//75I2woNDdU//vEPh7aoqChVrVr1lra1ePFiZWVlqW/fvvn+ivqPf/yj0L/oFmXfEydOlNVq1dSpU/O9RyNGjFC5cuU0e/bsfPtw9n7m/RX6Tlu6dKkOHz6s6Oho9enTJ9/ywp6scTt88MEHkqTJkyfnmyfEw8PDYULTqVOnymKx6N1333U4J202m/3qjrw5Z65XrVo1vfbaaw5tsbGxkn6/HWT8+PEqUeKPX9369Okjq9Va4Bwazz33nJo0aeLQlpCQIH9/f3366af223hc+TzKM3ToUD3yyCMObXlXO1x/XG7fvl0bNmzQ008/rd69ezv0DwgI0FtvvaXLly/rf//7X7599O7dO9+VU4MGDcq3j+LIG+e8q87ybNmyRbt27VLXrl0drgqz2WwqU6ZMvu00atRI7dq107p163T16lW31Ha9L774Qr/88ouGDBmiiIgIh2WVK1fW8OHDdfz4cYfPW8n5uV2iRAkFBga6vUYAKCpu3wCAYgoPD9cjjzyiqVOn6h//+Ic+/vhj5ebmOr0MOc+PP/4o6fdbG5xp166dNm7cqJ9++klPPPGEJOnPf/6zRowYoenTp+svf/mLvW9Rbt349ttvJUkbNmxweu/xyZMnlZOTo3379qlp06aKjo7W66+/rr/85S9auXKloqKi1KpVK9WvX7/A+6BvFBoaqho1amj9+vXKzc2134feoUMHRUREyGq1Kjk5WZGRkVq3bp0MwyhwXFxx/X3geUqWLKkKFSooMzOzSNtq3LixPDw88rVXqVLFPraF+emnnyRJrVu3zrfM19dXjRs3zvfIzKLu++LFi9q+fbuCgoI0YcIEp9vy8vJyuB+/b9++WrRokcLDw9WrVy+1bdtWrVq1uuNf/K+3efNmSVLnzp1NqyHPhQsXtHPnTlWoUCHfl/wbnTt3TgcOHFBwcLD9dqXr5R3becfC9Zy9x3kTgD700EP5vgR7eHioQoUKSk1NdVrLjV9apd8nYG3cuLE2bNig3bt3q3Hjxi59HuVxdn5VqVJFkhzOr7xj9MyZM04fw3rq1ClJyjdPRFH2URw9evSQv7+/Zs+ercTERPv7UNjn6xdffKGPPvpIW7ZsUXp6er4Jj9PT093+9J28cfz111+djmPe/B+7d+9Wly5dVL9+fTVu3Fhz5szRr7/+qu7du6t169Zq1qzZbZ9/BQBuFaEEALjBwIEDNXToUK1YsUJJSUlq2rRpoV9ezpw5I0kF/sKa156VlWVvCwkJUfv27bV69Wrt3r1b9erV08mTJ/Xll1+qcePGatiw4U3rzJuYcvz48YX2O3/+vKTfA4Xvv/9eCQkJ+vLLL7Vo0SJJv38h+Nvf/qahQ4fedJ/S71dLTJkyRT/++KNKliypU6dOqX379ipTpozCwsLsf9W7lfkkiqqgqw+sVqtycnLctq3c3Nybrp/3vleoUMHp8oLai7LvzMxMGYahU6dOOUxSWZiePXtq2bJl+ve//62pU6dq8uTJkqSmTZtq7Nix6tix4y1tx53yjv274TGwRanFlXM7j7MnteRdaVHQU1ysVmuBf5Ev6HiqWLGiQ63FqdnZcZlX8/XnV95nz+rVq7V69Wqn+5H++OxxZR/FUapUKT3zzDOaMmWKVq1apc6dO9uvTCtfvny+cGzixIkaNmyYAgMD1bFjR1WtWlWlS5eWxWKxz2vhbELZ4sobxwULFhTaL28cPTw8tHbtWo0aNUoLFy7Uq6++KkkqU6aMYmNjNXbsWPn6+rq9TgAoCm7fAAA3eO6551SqVCkNHjxYR48etV9aXJC8LxgFzYJ/7Ngxh355brzEePbs2bp27Zq9/WbytnfmzBkZhlHgP9f/hbVevXqaN2+eTp8+rS1btigxMVG5ubl6+eWX9cknn9zSfvP+ArtmzZp8wUO7du30008/KSMjQ8nJyfL399ejjz56S9u91/j5+UmSTpw44XR5Qe1FkfceN2nSpND32DAMh/W6du2qtWvXKjMzU8nJyfrrX/+qn3/+Wd26dSvS0wTcJe+LaEFPcbiTilKLq+f27VDQ8ZRXW14Nd6LmvHUnTpxY6DGZ9zQWM9z4+frFF1/o9OnT6tOnj8PjiK9du6aEhARVrFhRP//8s+bNm6fx48frrbfeUkJCQqHh4o3ybsdx9ljpwoKrxYsXFzqOeZPTSlJgYKDee+89HTlyxD6pcd26dTVp0iS99NJLt1wrANwuhBIA4AYBAQGKiYlRamqqfHx89OyzzxbaP+8qioIu1c97zN+NX8579uwpPz8/zZo1S7m5uZo+fbqsVqvTe+6dadGihaTfH/9YVFarVU2bNtWrr75qn9fi888/v6V127VrJ4vFouTkZK1du1Y1atSwP6avffv2ys3N1YwZM7R//361adPG6W0KN8rr466/lN4Jee+7s3vzz58/X+DcAEXh6+urBg0a6Oeff1ZGRkaR1/fx8VG7du307rvv6vXXX9eVK1e0YsUK+/I7Ne55x+r1+zaLj4+PHn74YZ04ccLpbRfXy3vc4tGjR50+SrOgc/t22LBhQ762M2fOaNu2bfZHxEqufx4VRXE+e4qiOMdnq1atVLt2bS1evFhnzpyxhxM3hr7p6enKyspSy5Yt811dcv78efvtMLcib06HI0eO5Ft242NmpeKPY61atTRgwABt2LBBvr6+BT5mGgDuJEIJAHCTMWPG6LPPPtPKlSudToB2vVatWqlOnTrauHGjFi5c6LBs4cKF+vrrr/XQQw/lm3sg7xLjo0eP6r333tP27dvVpUsX2Wy2W6oxLi5OJUuW1F//+lft27cv3/IrV644/LK7detW+6Xd18v7C2xBE3neyGazqUGDBvrmm2/01VdfOdye0bJlS3l7e2vs2LGSCr6v/UaBgYGyWCz2R4veC7p3726/b3379u0Oy8aMGeP0L6OueOWVV3TlyhX179/f6TYzMzMdvjh99dVXTv9S6+x9zpvs73aP+5NPPqlq1appyZIl+SZ3lVTgPAq3S96tSi+++GK+cyI3N9d+NYH0+0SqhmHo73//u8OX4/T0dI0ePdre53abOXNmvhAlISFBZ86c0bPPPmt/XKern0dF0axZMz3++ONatGiRfVLgG+3YsUMnT550eR9S8Y/P2NhYXb58WR9++KGWL1+uhg0b5rsVz2azqXTp0tq6davD7SZXr17Vyy+/rPT09FveX/PmzSVJU6ZMcWhPTk52etx3795dNWvW1H/+8x8tX77c6Ta//fZbXbx4UZJ06NAhHTx4MF+fzMxMZWdn39JkxQBwuzGnBAC4SdWqVVW1atVb6muxWDR9+nR17NhRvXr1Uvfu3VW3bl3t3btXn3/+ucqUKaMZM2Y4zLSfJzY2Vh9//LHi4+PtP9+qunXraurUqerfv78aNGigTp066aGHHtLVq1f122+/6euvv1b58uW1Z88eSb9/qZk8ebJat26tmjVrKjAwUL/88ouWLl0qLy8vDRs27Jb33b59e+3cudP+33m8vLzUqlWrIs8n4evrq/DwcH399dfq27evHnroIXl4eCg6OvqW5tcwg5+fn/7zn//oueeeU8uWLfXMM8+oUqVK2rRpk7Zv366IiAht2LDB6fteFP3799fWrVv14YcfqmbNmvandGRkZOjQoUP66quv9Pzzz+ujjz6S9PsX7qNHj6pVq1aqVq2aPD09tXXrVq1du1ahoaEOT0to3769xo8fr4EDB+rpp59WmTJlFBAQoLi4uGLVfCNPT08tWLBAkZGR6tOnjyZPnqwWLVro8uXL2r17t5KTkx2ClD179igxMdFhG5mZmQ4TFP7rX/9y+WkiL7zwgr7++mvNnDlTtWvXVvfu3VW+fHmlpaVp7dq16t+/v33iwb/97W9asWKFFi9erEaNGqlLly66ePGiFixYoJMnT2r48OHF+oJ/qzp37qxWrVrZj7ONGzdq48aNqlatmsNYFefzqCg+/fRTtWvXTgMGDND777+v8PBwBQQEKDU1VSkpKdq5c6e+/fbbWw5ZnSnu8fncc8/pzTff1MiRI3X16lWnn68lSpTQ0KFDlZiYqEceeUTdu3fXlStXtG7dOmVkZKht27b2q0tu5vnnn9f48eM1duxYbd++XfXr19e+ffu0YsUK9ejRI9/TSEqWLKlFixYpKipKXbt2VcuWLdW4cWOVLl1aR44c0Q8//KCDBw/q2LFjKl26tLZv366ePXsqLCxM9erVU+XKlXXq1CktXrxYV69etc8xAQCmuv1PHQWA+48kIzg4+Jb6vvHGG/meNZ9nz549xp/+9CejYsWKhtVqNSpWrGj07dvX2LNnT6HbrFWrliHJKFu2rJGdne20T2xsrCHJOHToUL5lKSkpRmxsrFG1alXD09PTCAwMNBo0aGAMGjTISE5OtvfbvHmzMXjwYKNhw4ZGYGCg4e3tbdSsWdPo16+fsWPHjlt6/XmWLFliSDIsFotx4sQJh2XvvPOOIcmoUKFCkV7L/v37jW7duhlly5Y1LBaLwzivW7fOkGSMHDnS6TZDQ0ON0NDQW6r90KFDhiQjNjbW6fKIiAjjxv+lJiUlFfi+L1++3HjssceMUqVKGQEBAUZ0dLSxe/duo2vXroYkIzMzs1j7zrN06VKja9euRvny5Y2SJUsaFSpUMMLCwow33njD2L17t73fvHnzjN69exu1atUyfHx8jDJlyhgNGjQwXn/9dePkyZP5tvvvf//bqFu3ruHp6WlIuuVxdKaw49QwDOPXX381XnrpJaNatWpGyZIljbJlyxrNmzc33n77bYd+ee93Yf8UtI+imDVrlvHEE08Yfn5+hpeXl1GtWjWjT58+xtatWx36Xbp0yXj77beNBg0aGN7e3oavr6/RqlUr49NPP823zZu9x5KMiIgIp8ucHccjR440JBnr1q0zkpKSjEaNGhne3t5GUFCQ0a9fPyMtLc3ptoryeXT9Poryes6ePWu8/fbbxqOPPmr4+PgY3t7eRrVq1YwuXboYkydPNs6fP2/vW9g5VNi4FPf4bN++vSHJsFqtxvHjx532uXr1qvHvf//bqFevnuHt7W1UqFDB+NOf/mQcPnzY6TFd2Jjs3LnT6Ny5s+Hr62v4+PgYERERxvr16wt9/SdOnDBeffVVo0GDBkapUqUMHx8fo1atWsbTTz9tzJw507h69aphGIZx5MgRIz4+3mjZsqVRoUIFw9PT0wgODjY6depkLF++vEjjAgC3i8UwbpjpCgAA3HE5OTmqUaOGrly54nArAFBUCQkJeuutt7Ru3Tq1adPG7HIAACgUc0oAAHAHZWVl2e/3zmMYhsaMGaPffvtNPXr0MKkyAACAO485JQAAuIM2b96sXr16KTIyUtWqVdP58+e1efNmbdu2TVWqVLHPSwAAAPAgIJQAAOAOqlOnjrp166ZvvvlGy5cv17Vr1xQSEqKhQ4fq9ddfL9YkfwAAAPca5pQAAAAAAACmYE4JAAAAAABgCkIJAAAAAABgivtqTom0tDSzS7ipoKAgpaenm13GfYPxdB/G0r0YT/diPN2HsXQvxtO9GE/3YSzdi/F0L8bTve6F8axcuXKBy7hSAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmOK+evoGAAAAAAC3S05Oji5fvixJslgsJlfzuxMnTig7O9u0/RuGIQ8PD3l7e7u0PqEEAAAAAAA3kZOTo0uXLsnHx+euCSQkyWq1ysPDw9QaLl++rKtXr6pkyZJFXpfbNwAAAAAAuInLly/fdYHE3cLLy0tXrlxxaV1CCQAAAAAAbgGBhHPFGRdCCQAAAAAAboJAonCujg+hBAAAAAAAMAWhBAAAAAAA97Hc3FwNHz5cDRo0UHBwsDZt2mR2SXY8fQMAAAAAABflDIy+o/vzmLKkyOskJydr/vz5WrBggUJDQxUQEFBo/6ysLI0YMUKrV6+WJHXs2FFjxoyRv7+/SzUXhislAAAAAAC4jx0+fFg2m01hYWGy2Wzy9PQstH9cXJx27typWbNmadasWdq5c6eGDh16W2rjSgkAAAAAAO5Tw4YN04IFCyRJwcHBCgkJ0ebNmzV58mTNnDlTaWlpKlu2rGJiYhQfH6/9+/dr3bp1+vzzz9WsWTNJ0rhx49SjRw8dOHBAtWrVcmt9hBIAAAAAANynRo0apZCQEM2dO1fLly+Xh4eHEhMTNWPGDI0cOVLh4eE6ffq0du7cKUnaunWrfHx87IGEJIWFhal06dLaunUroQQAAAAAALg1fn5+8vX1lYeHh2w2my5cuKApU6YoISFBvXv3liRVr17dHkKcPHlS5cqVc3jEp8ViUVBQkE6ePOn2+phTAgAAAACAB8S+ffuUnZ2t1q1bm12KJEIJAAAAAADw/9hsNp0+fVqGYdjbDMNQenq6bDab2/dHKAEAAAAAwAOidu3a8vLy0saNG50ub9q0qS5cuKAtW7bY27Zs2aKLFy+qadOmbq+HOSUAAAAAAHhA+Pr6asCAAUpMTJSXl5fCw8OVmZmplJQUxcbGqnbt2mrbtq1ee+01jRs3TpL02muvqUOHDm6f5FIilAAAAAAA4IESHx8vf39/TZgwQceOHVNQUJBiYmLsyydNmqQRI0aob9++kqTIyEiNGTPmttRiMa6/UeQel5aWZnYJNxUUFKT09HSzy7hvMJ7uw1i6F+PpXoyn+zCW7sV4uhfj6T6MpXsxnu51r47nxYsXVbp0abPLyMdqteratWtml1Ho+FSuXLnA9ZhTAgAAAAAAmIJQAgJu4lwAACAASURBVAAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJHghZD99l7btpncd+6d6CS+8PNxpOxLBrG03041wEAAIDbgyslAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAAC4j+Xm5mr48OFq0KCBgoODtWnTJrNLsmOiSwAAAAAAXHQrk6K7kysTrCcnJ2v+/PlasGCBQkNDFRAQUGj/iRMnau3atfr555916dIlHT161NVyb4orJQAAAAAAuI8dPnxYNptNYWFhstls8vT0LLT/lStX1LlzZ73wwgu3vTaulAAAAAAA4D41bNgwLViwQJIUHByskJAQbd68WZMnT9bMmTOVlpamsmXLKiYmRvHx8ZKkv//975KkZcuW3fb6CCUAAAAAALhPjRo1SiEhIZo7d66WL18uDw8PJSYmasaMGRo5cqTCw8N1+vRp7dy505T6CCUAAAAAALhP+fn5ydfXVx4eHrLZbLpw4YKmTJmihIQE9e7dW5JUvXp1NWvWzJT6mFMCAAAAAIAHxL59+5Sdna3WrVubXYokQgkAAAAAAGASQgkAAAAAAB4QtWvXlpeXlzZu3Gh2KZKYUwIAAAAAgAeGr6+vBgwYoMTERHl5eSk8PFyZmZlKSUlRbGysJOno0aPKzMxUamqqJNknwaxevbp8fHzcWg+hBAAAAAAAD5D4+Hj5+/trwoQJOnbsmIKCghQTE2NfPn78ePtjRCUpKipKkrRgwQK1bNnSrbUQSgAAAAAA4KLFfeuaXcJNDR48WIMHD7b/XKJECcXFxSkuLs5p/wkTJmjChAl3pDbmlAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAA4D6Wm5ur4cOHq0GDBgoODtamTZvMLsnOanYBAAAAAADcq5bOy7qj+3uyV0CR10lOTtb8+fO1YMEChYaGKiCg4G0cOXJEEyZM0KZNm3Ty5EnZbDZFR0dr2LBhKlWqVHFKd4pQAgBwR3WfvafQ5Yv71r1DlQAAADwYDh8+LJvNprCwsJv2PXDggHJycjR27FhVr15d+/fv16uvvqrMzEz985//dHtthBIAANzDCHnc52ZjKTGeRcF4uhfnunsxnu7DuX73GzZsmBYsWCBJCg4OVkhIiDZv3qzJkydr5syZSktLU9myZRUTE6P4+Hi1bdtWbdu2ta8fGhqqIUOGaPz48YQSAAAAAADg1o0aNUohISGaO3euli9fLg8PDyUmJmrGjBkaOXKkwsPDdfr0ae3cubPAbZw/f77QWz6Kg1ACAAAAAID7lJ+fn3x9feXh4SGbzaYLFy5oypQpSkhIUO/evSVJ1atXV7NmzZyun5qaqo8++khDhgy5LfXx9A0AAAAAAB4Q+/btU3Z2tlq3bn3TvqdOnVLfvn31xBNPaNCgQbelHkIJAAAAAADg4OTJk/q///s/1alTR++//74sFstt2Q+hBAAAAAAAD4jatWvLy8tLGzduLLDPiRMnFBMTo9q1a+vDDz+U1Xr7Zn5gTgkAAAAAAB4Qvr6+GjBggBITE+Xl5aXw8HBlZmYqJSVFsbGxOn78uGJiYlSxYkUlJCQoIyPDvm65cuXk4eHh1noIJQAAAAAAeIDEx8fL399fEyZM0LFjxxQUFKSYmBhJ0oYNG3To0CEdOnRIzZs3d1hv8+bNqlKliltrIZQAAAAAAMBFT/Zy36MyD5y+VOjyWuVKubTdwYMHa/DgwfafS5Qoobi4OMXFxeXr26tXL/Xq1cul/biCOSUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApivX0jZUrV2rJkiXKyspSSEiI+vXrp3r16hXYf9euXZo+fbpSU1MVGBio6OhoRUZGOvTJzMzU7Nmz9dNPP+ny5cuy2WwaOHCg6tevX5xSAQAAAADAXcblUGLTpk2aNm2aBgwYoLp162rVqlV655139N577ykoKChf/5MnT2rs2LFq27athgwZoj179uiTTz6Rn5+fWrRoIUm6cOGCRowYobp16yo+Pl5+fn46ceKE/Pz8XH+FAAAAAADgruRyKLFs2TJFRESoQ4cOkqT+/ftr27ZtWrVqlfr06ZOv/6pVqxQYGKj+/ftLkkJCQnTgwAEtXbrUHkosXrxYgYGBDs9KtdlsrpYIAAAAAADuYi6FEteuXdPBgwf15JNPOrQ3bNhQe/fudbrO/v371bBhQ4e2Ro0aacOGDbp27ZqsVqt++OEHNW7cWO+9955+/vlnBQYGqn379oqKipLFYnGlVAAAAAAAcJdyaaLLs2fPKjc3V/7+/g7tAQEBysrKcrpOVlaWAgICHNr8/f2Vk5Ojc+fOSfr9Fo9Vq1apQoUKeuONN9SlSxfNnj1bK1eudKVMAAAAAAAeeLm5uRo+fLgaNGig4OBgbdq0yeyS7Io10aW75ebmqmbNmvbbP6pXr65jx45p5cqV6tSpU77+a9as0Zo1ayRJiYmJTueyMNuNNVmt1ruyznuBs3FjPF3HselejKf7cK67F8emezGe7sV4ug9j6V6Mp3vdL+N54sQJWa35v0K/++67d7SOV155JV+bs7qut2rVKs2fP1+fffaZQkNDFRAQUOA6ubm5io2N1c8//6z09HT5+/vr8ccf14gRI1SpUqUC9+Hl5eXS++pSKOHn56cSJUrozJkzDu3OrobI4+wqijNnzsjDw0NlypSRJAUGBiokJMShT0hIiFasWOF0mx06dLDPaSFJ6enpRX4tt9uNNQUFBd2Vdd4LnI0b4+k6jk33Yjzdh3PdvTg23YvxdC/G030YS/diPN3rfhnP7OxseXh4mF2Grl275vCz1WrN13ajX375RTabTU2aNClwO3lyc3PVsmVLxcXFqUKFCjp27JhGjx6tfv366YsvvihwH9nZ2QW+r5UrVy5wPZdu37BarapRo4ZSUlIc2nfs2KE6deo4Xad27drasWOHQ1tKSopq1KhhT2jq1KmjtLQ0hz5paWn3ZIoGAAAAAIDZhg0bpoSEBB09elTBwcEKDw+XYRj66KOP1KpVK1WvXl1NmzbV2LFjJUklSpTQwIED1bRpU4WEhCgsLExxcXHatm2bLl++7Pb6XAolJKlbt25av369kpOTlZqaqqSkJGVkZKhjx46SpEmTJmnSpEn2/pGRkcrIyNC0adOUmpqq5ORkrV+/3mGyzK5du2r//v1atGiRjh8/rm+//VYrVqxQVFRUMV4iAAAAAAAPplGjRumvf/2rKlWqpJ9++knLly9XYmKiJk6cqCFDhmjt2rWaPHlygbdmZGZmatGiRWrSpIm8vb3dXp/Lc0q0bNlS586d06JFi5SZmakqVaooPj5e5cuXl5T/Eh2bzab4+HhNnz7d/njQ559/3v44UEmqVauW/v73v2vOnDn63//+p6CgIPXq1YtQAgAAAAAAF/j5+cnX11ceHh6y2Wy6cOGCpkyZooSEBPXu3VvS7/M5NmvWzGG9t99+W0lJSbp06ZIeffRRzZgx47bUV6yJLqOiogoMDBISEvK11a9fX+PGjSt0m48++qgeffTR4pQFAAAAAACc2Ldvn7Kzs9W6detC+7300kvq3bu3jh49qnfffVdDhgzRrFmzZLFY3FrPXfX0DQAAAAAAYL6yZcuqbNmyqlmzpmrVqqWwsDB9//33Cg8Pd+t+XJ5TAgAAAAAA3Ftq164tLy8vbdy48ZbXMQxD0u9P2HA3rpQAAAAAAOAB4evrqwEDBigxMVFeXl4KDw9XZmamUlJSFBsbqy1btmjnzp0KCwuTv7+/Dh8+rPHjx6tKlSpq3ry52+shlAAAAAAA4AESHx8vf39/TZgwQceOHVNQUJBiYmIkSd7e3lq2bJnGjx+vS5cuyWazqU2bNvrvf/97dz19AwAAAACAB93QoUPdtq0Dpy8VurxWuVIubXfw4MEaPHiw/ecSJUooLi5OcXFx+fo+/PDDWrhwoUv7cQVzSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAcB/Lzc3V8OHD1aBBAwUHB2vTpk1ml2RnNbsAAAAAAADuVbYD8e7b1s06ZEona40t8naTk5M1f/58LViwQKGhoQoICLil9S5fvqxu3bpp9+7dWr58uRo1alTkfd8MV0oAAAAAAHAfO3z4sGw2m8LCwmSz2eTp6XlL640ePVqVKlW6rbURSgAAAAAAcJ8aNmyYEhISdPToUQUHBys8PFyGYeijjz5Sq1atVL16dTVt2lRjxzpegbFy5Upt2rRJb7755m2tj9s3AAAAAAC4T40aNUohISGaO3euli9fLg8PDyUmJmrGjBkaOXKkwsPDdfr0ae3cudO+TlpamuLj4zVz5kx5e3vf1voIJQAAAAAAuE/5+fnJ19dXHh4estlsunDhgqZMmaKEhAT17t1bklS9enU1a9ZMkpSTk6MhQ4Zo0KBBatCggY4cOXJb6+P2DQAAAAAAHhD79u1Tdna2Wrdu7XT5+++/r5IlS+rFF1+8I/VwpQQAAAAAAJAkffPNN/ruu+8UGhrq0P7kk08qOjpakyZNcuv+CCUAAAAAAHhA1K5dW15eXtq4caNq1KiRb/m7776rixcv2n8+ceKE+vTpow8++EBhYWFur4dQAgAAAACAB4Svr68GDBigxMREeXl5KTw8XJmZmUpJSVFsbKyqVq3q0N/Hx0eSVK1aNVWuXNnt9RBKAAAAAADwAImPj5e/v78mTJigY8eOKSgoSDExMabUQigBAAAAAICLTtYa67ZtHTh9qdDltcqVcmm7gwcP1uDBg+0/lyhRQnFxcYqLi7vpulWqVNHRo0dd2u+tIJQw2fvvv1/o8qFDh96hSu4PjKf73GwsJcazKDg2AQAAgPx4JCgAAAAAADAFoQQAAAAAADAFoQQAAAAAADAFoQQAAAAAADdhGIbZJdzVXB0fQgkAAAAAAG4BwYRzxRkXQgkAAAAAAG7C29tbFy5cIJhwIjs7W56eni6tyyNBAQAAAAC4CQ8PD5UqVUoXL16UJFksFrfvY++xM4Uur1wqfyDi5eWl7Oxst9dyqwzDkIeHh0qWLOnS+oQSAAAAAADcAg8PD/n4+Ny27f9/238rdHnXhyvlawsKClJ6evrtKum24/YNAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCmtxVl65cqWWLFmirKwshYSEqF+/fqpXr16B/Xft2qXp06crNTVVgYGBio6OVmRkpNO+n332mebMmaOoqCgNGDCgOGUCAAAAAIC7kMtXSmzatEnTpk1Tjx49NG7cONWpU0fvvPOO0tPTnfY/efKkxo4dqzp16mjcuHF66qmnlJSUpM2bN+fru2/fPq1Zs0ahoaGulgcAAAAAAO5yLocSy5YtU0REhDp06KCQkBD1799fgYGBWrVqldP+q1atUmBgoPr376+QkBB16NBBERERWrp0qUO/ixcv6oMPPtBLL70kHx8fV8sDAAAAAAB3OZdCiWvXrungwYNq1KiRQ3vDhg21d+9ep+vs379fDRs2dGhr1KiRDh48qGvXrtnbJk+erPDwcD388MOulAYAAAAAAO4RLoUSZ8+eVW5urvz9/R3aAwIClJWV5XSdrKwsBQQEOLT5+/srJydH586dkyStWbNGx48fV+/evV0pCwAAAAAA3EOKNdGlO6WlpWnOnDkaPXq0rNZbK2vNmjVas2aNJCkxMVFBQUG3s0SX3FiT1WotUp1342syi7OxYDxdV9xj09k2HmSc6+7jjnMdf3DHuY4/MJ7uxXi6D2PpXoynezGe7nM//p7kUijh5+enEiVK6MyZMw7tzq6GyOPsKoozZ87Iw8NDZcqU0fbt23Xu3Dm98sor9uW5ubnavXu3Vq9erZkzZ6pkyZIO63fo0EEdOnSw/1zQJJtmurGmoKCgItV5N74mszgbC8bTdcU9Np1t40HGue4+7jjX8Qd3nOv4A+PpXoyn+zCW7sV4uhfj6T736u9JlStXLnCZS6GE1WpVjRo1lJKSoscee8zevmPHDoWHhztdp3bt2vrhhx8c2lJSUlSjRg1ZrVaFhYXpX//6l8Py//73v6pYsaJ69Ohxy1dPAAAAAACAe4PL3/S7deumDz74QLVq1VKdOnW0evVqZWRkqGPHjpKkSZMmSZLi4uIkSZGRkVq5cqWmTZumDh06aO/evVq/fr1efvllSZKPj0++p214eXnJ19dXVatWdbVMAAAAAABwl3I5lGjZsqXOnTunRYsWKTMzU1WqVFF8fLzKly8vKf9lJTabTfHx8Zo+fbr98aDPP/+8WrRoUbxXAAAAAAAA7knFuiciKipKUVFRTpclJCTka6tfv77GjRt3y9t3tg0AAAAAAHB/cOmRoAAAAAAAAMVFKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExhLc7KK1eu1JIlS5SVlaWQkBD169dP9erVK7D/rl27NH36dKWmpiowMFDR0dGKjIy0L//ss8/0/fffKy0tTVarVbVr11afPn1UtWrV4pQJAAAAAADuQi5fKbFp0yZNmzZNPXr00Lhx41SnTh298847Sk9Pd9r/5MmTGjt2rOrUqaNx48bpqaeeUlJSkjZv3mzvs2vXLkVGRmr06NEaOXKkPDw8NHr0aJ0/f97VMgEAAAAAwF3K5VBi2bJlioiIUIcOHRQSEqL+/fsrMDBQq1atctp/1apVCgwMVP/+/RUSEqIOHTooIiJCS5cutfd544031LZtW1WtWlVVq1bVkCFDdPbsWe3Zs8fVMgEAAAAAwF3KpVDi2rVrOnjwoBo1auTQ3rBhQ+3du9fpOvv371fDhg0d2ho1aqSDBw/q2rVrTte5dOmSDMOQr6+vK2UCAAAAAIC7mEtzSpw9e1a5ubny9/d3aA8ICNCOHTucrpOVlaVHHnnEoc3f3185OTk6d+6cAgMD862TlJSkatWq6aGHHnK6zTVr1mjNmjWSpMTERAUFBbnycm6rG2uyWq1FqvNufE1mcTYWjKfrintsOtvGg4xz3X3cca7jD+441/EHxtO9GE/3YSzdi/F0L8bTfe7H35OKNdHl7TR9+nTt3btXo0aNUokSzi/o6NChgzp06GD/uaD5LMx0Y01BQUFFqvNufE1mcTYWjKfrintsOtvGg4xz3X3cca7jD+441/EHxtO9GE/3YSzdi/F0L8bTfe7V35MqV65c4DKXbt/w8/NTiRIldObMGYf2rKwsBQQEOF0nICBAWVlZDm1nzpyRh4eHypQp49A+bdo0ffPNN3rzzTdVoUIFV0oEAAAAAAB3OZdCCavVqho1aiglJcWhfceOHapTp47TdWrXrp3v1o6UlBTVqFFDVusfF2wkJSXZA4ng4GBXygMAAAAAAPcAl5++0a1bN61fv17JyclKTU1VUlKSMjIy1LFjR0nSpEmTNGnSJHv/yMhIZWRkaNq0aUpNTVVycrLWr1+vJ5980t7n448/1vr16/Xyyy/L19dXWVlZysrK0uXLl4vxEgEAAAAAwN3I5TklWrZsqXPnzmnRokXKzMxUlSpVFB8fr/Lly0vKf6+LzWZTfHy8pk+fbn886PPPP68WLVrY++Q9TnTUqFEO68bExOiZZ55xtVQAAAAAAHAXKtZEl1FRUYqKinK6LCEhIV9b/fr1NW7cuAK3N3/+/OKUAwAAAAAA7iEu374BAAAAAABQHIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAAD8/+zdeVhWdf7/8ReKJKYCioh6u47KqAmT+SvGvaKNcskKs7RxKS11HGua0qux1CzMLq00LDUzdTRNG3NNu9Asl6/W5MLighsxoIKIN+IKCL8/vDjDLesNt35An4/r8qr75pxzf86bcz7nnNf9OQcYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGCEu+kGAACQ35pl9kLedXyvZz/vm9MYAAAA3FCMlAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGuJtuAAAAzpoxY0axPx89evRNagkAAADKg5ESAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABghHt5Zt64caNWr14tu90um82mQYMGqU2bNkVOv3//fi1YsECJiYny8fFRr1699PDDD5drmQAAAAAAoHIq80iJHTt26KuvvtKTTz6pDz74QAEBAXr//feVmppa6PQpKSkKDw9XQECAPvjgA/Xp00fz58/Xzp07y7xMAAAAAABQeZU5lFi7dq26d++ukJAQ2Ww2DRkyRD4+Pvrhhx8Knf6HH36Qj4+PhgwZIpvNppCQEHXv3l1r1qwp8zIBAAAAAEDlVaZQIjs7W8eOHVNQUJDD+4GBgTp06FCh8xw+fFiBgYEO7wUFBenYsWPKzs4u0zIBAAAAAEDlVaZQ4ty5c8rJyZGXl5fD+97e3rLb7YXOY7fb5e3t7fCel5eXrl69qoyMjDItEwAAAAAAVF7letClaZGRkYqMjJQkTZkyRb6+vi5dfvKTnYr9+faVO0pcxvyII9e94xiwTJo0qdj5q+x8qcTPyAmeW+I0FUF561mwlpKr63mr1FIqSz2dq6V0+9STfd05rqjn9dzd3ZWdnZ3vndujnjdjXz+evLDEz5gcerLYn1eGWko3Z1+nnv9zM+pZUi2lylHPm7GvS+WvZ2WopUTf6Wrs6651M66JKtu+XqZQonbt2qpSpYrS09Md3i9suo9XygAAIABJREFUNESewkY8pKenq2rVqqpVq5YkOb3MkJAQhYSEWK9v9gMxXfF5JS3D7ya1oyKoCPW8VWoplX9dSjP/7VLPirBtuqodFUFZ1sPX19ep+ajn7dcGV6go61FR2lFeFWU9Kko7yqsirEdFaIOrVIR1qQhtcIWKsh4VpR3lVRHW40a0oWHDhkX+rEy3b7i7u6tFixaKiopyeD86OloBAQGFztOqVStFR0c7vBcVFaUWLVrI3d29TMsEAAAAAACVV5n/+sYTTzyhLVu2aNOmTUpMTNT8+fOVlpamhx56SJL06aef6tNPP7Wmf/jhh5WWlqavvvpKiYmJ2rRpk7Zs2aKePXuWepkAAAAAAODWUeZnSnTq1EkZGRn697//rbNnz6px48YaN26c6tWrJ6ngkA8/Pz+NGzdOCxYssP486ODBgxUcHFzqZQIAAAAAgFtHuR50+cgjj+iRRx4p9GcTJkwo8F7btm31wQcflHmZAAAAAADg1lHm2zcAAAAAAADKg1ACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMMLddANudT37eTu89vX1VWpqqqHWVG7X11KinuXBtula1BMAAABwHiMlAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADDCvSwz5ebmavny5dq0aZPOnz+vVq1aaejQoWrcuHGx8+3cuVPLli1TcnKy6tevr/79++vee++VJGVnZ2vp0qXau3evkpOT5enpqXbt2un555+Xr69vWZoJAAAAAAAqsDKNlFi1apXWrl2rwYMHKzw8XLVr19bkyZN16dKlIueJi4vTxx9/rK5du2rq1Knq2rWrpk+frsOHD0uSMjMzdfz4cfXt21cffPCB3njjDZ05c0bvvfeerl69Wra1AwAAAAAAFZbToURubq7Wr1+vPn36KDg4WE2aNNGoUaN06dIlbdu2rcj51q1bp3bt2qlv376y2Wzq27ev2rVrp3Xr1kmSatSoofHjx6tTp05q2LChWrZsqWHDhikpKUlJSUllX0MAAAAAAFAhOR1KpKSkyG63KzAw0HrPw8NDbdq00aFDh4qcLy4uTkFBQQ7vBQUFKS4ursh5Ll68KEm68847nW0mAAAAAACo4JwOJex2uyTJ29vb4X0vLy+lp6cXO5+Xl1eBefKWd73s7GwtWrRI99xzj+rWretsMwEAAAAAQAVX4oMut27dqjlz5livx40bd0MbJElXr17VjBkzdOHCBb3xxhtFThcZGanIyEhJ0pQpU1z+QMzkEn5els9zd3d3br4jJU9SWR4EWhnqeavUUnJ+XZyupXTb1LMibJtlbYcJ1NN1XLOvFx7+u1JlqKXkqm2Teuahnq7Dvu5a1NO12Nddq/z1vPVqWWIo0bFjR7Vq1cp6nZWVJenayIf8jU1PTy8wEiI/b2/vAiMp0tPTC4y4uHr1qj755BMlJCRowoQJqlWrVpHLDAkJUUhIiPU6NTW1pNVxqbJ8nq+vr1Pz+d2gdlREFaGet0otJefXxdlaSrdPPSvCtlnWdlRE1NO1KsJ6VIQ2uEJFWY+K0o7yqijrUVHaUV4VYT0qQhtcpSKsS0VogytUlPWoKO0or4qwHjeiDQ0bNizyZyXevuHp6Sl/f3/rn81mk7e3t6KioqxpMjMzdfDgQQUEBBS5nNatWzvMI0lRUVFq3bq19To7O1sfffSRfv/9d73zzjsFAgsAAAAAAHDrcPqZEm5ubgoNDdWqVau0a9cuJSQkaNasWapevbq6dOliTTdp0iQtWbLEeh0aGqqYmBh99913SkpK0sqVKxUbG6vHH39c0rUREnl/IvRvf/ub3NzcZLfbZbfblZmZ6YJVBQAAAAAAFUmJt28Upnfv3srMzNS8efN04cIFtWzZUm+99ZY8PT2taZKTkx0eUBkQEKAxY8Zo6dKlWrZsmfz9/TVmzBjr1pAzZ87oP//5jyRp7NixDp83YsQI9ejRoyxNBQAAAAAAFVSZQgk3NzeFhYUpLCysyGkiIiIKvBccHKzg4OBCp/fz89M333xTluYAAAAAAIBKyOnbNwAAAAAAAFyBUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwwt10AwAAgDmjR48ueaIj4258Q24R1NO1SqwntXQK9XQd9nXXYtt0rcpWT0ZKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABghLvpBlRkVeeuNt2EWwr1dB1q6VrUEwAAADCDkRIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACPcTTcAAABUbCktw0034ZZCPV2HWroW9XQt6uk61NK1Klo9GSkBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADDC3XQDAAC3lqpzV5tuAgAAACoJRkoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjHAvy0y5ublavny5Nm3apPPnz6tVq1YaOnSoGjduXOx8O3fu1LJly5ScnKz69eurf//+uvfeewudds6cOYqMjNSAAQPUq1evsjQTAAAAAABUYGUaKbFq1SqtXbtWgwcPVnh4uGrXrq3Jkyfr0qVLRc4TFxenjz/+WF27dtXUqVPVtWtXTZ8+XYcPHy4w7c6dO3XkyBH5+PiUpXkAAAAAAKAScDqUyM3N1fr169WnTx8FBwerSZMmGjVqlC5duqRt27YVOd+6devUrl079e3bVzabTX379lW7du20bt06h+lOnz6t+fPna/To0XJ3L9NADgAAAAAAUAk4HUqkpKTIbrcrMDDQes/Dw0Nt2rTRoUOHipwvLi5OQUFBDu8FBQUpLi7Oen316lV98skneuqpp2Sz2ZxtGgAAAAAAqEScHopgt9slSd7e3g7ve3l56ezZs8XO5+XlVWCevOVJ0jfffKNatWrp4YcfLlVbIiMjFRkZKUmaMmWKfH19SzWfSe7u7s6180jJk1SG9b5RXF1Paunk+lPPIrGvu9btWs/kUkxT8nrYi/1pZaiDq5RUz9LVgnrmoZ6uczP29dIt49ZA3+la7OuuVf563nr7eomhxNatWzVnzhzr9bhx425IQ2JjY7VlyxZ9+OGHpZ4nJCREISEh1uvU1NQb0TSX8vX1daqdfqWYpjKs943i6npSS+fWn3oWjX3dtahn0cq7HrdKHVzBFbWgnv9DPV2LeroWfafrsG261q1az4YNGxb5sxJDiY4dO6pVq1bW66ysLEnXRj7kT2DS09MLjITIz9vbW+np6Q7vpaenWyMuYmNjZbfbNWzYMOvnOTk5Wrx4sdavX6/PP/+8pKYCAAAAAIBKpMRQwtPTU56entbr3NxceXt7KyoqSi1btpQkZWZm6uDBgxowYECRy2ndurWioqIc/rxnVFSUWrduLUl65JFHFBwc7DDPe++9p86dOzuMhgAAAKXXs5/j7ZZlGRWF/6GerkU9XYdauhb1dC3q6TrX11Kq/PV0+kGXbm5uCg0N1apVq7Rr1y4lJCRo1qxZql69urp06WJNN2nSJC1ZssR6HRoaqpiYGH333XdKSkrSypUrFRsbq8cff1zStedLNGnSxOGfu7u7vL29ix3qAQAAAAAAKqcy/c3N3r17KzMzU/PmzdOFCxfUsmVLvfXWWw4jKpKTk1W3bl3rdUBAgMaMGaOlS5dq2bJl8vf315gxYxxuDQEAAAAAALePMoUSbm5uCgsLU1hYWJHTREREFHgvODi4wC0axSlsGQAAAAAA4Nbg9O0bAAAAAAAArkAoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBHuphsAAAAKV3XuatNNuKVQT9einq5DLV2LeroW9XQt6lkQIyUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACHfTDQAAwNVSWoabbgIAAABKgZESAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABjhXpaZcnNztXz5cm3atEnnz59Xq1atNHToUDVu3LjY+Xbu3Klly5YpOTlZ9evXV//+/XXvvfc6THPixAktWbJEMTExys7OVqNGjfTXv/5VNputLE0FAAAAAAAVVJlGSqxatUpr167V4MGDFR4ertq1a2vy5Mm6dOlSkfPExcXp448/VteuXTV16lR17dpV06dP1+HDh61pUlJSNH78ePn5+entt9/WtGnT1K9fP1WvXr0szQQAAAAAABWY06FEbm6u1q9frz59+ig4OFhNmjTRqFGjdOnSJW3btq3I+datW6d27dqpb9++stls6tu3r9q1a6d169ZZ03z99dcKCgrSCy+8oBYtWqh+/frq0KGDfH19y7Z2AAAAAACgwnI6lEhJSZHdbldgYKD1noeHh9q0aaNDhw4VOV9cXJyCgoIc3gsKClJcXJwkKScnR7/99ptsNpvee+89DR06VOPGjdOOHTucbSIAAAAAAKgEnA4l7Ha7JMnb29vhfS8vL6Wnpxc7n5eXV4F58pZ37tw5Xb58WStXrlRQUJDGjx+vzp07a8aMGdq9e7ezzQQAAAAAABVciQ+63Lp1q+bMmWO9Hjdu3A1pSE5OjiSpY8eOeuKJJyRJzZo109GjR7VhwwZ16NChwDyRkZGKjIyUJE2ZMqVS3Obh7u7uXDuPlDxJZVjvG8XV9aSWTq4/9SwS+7prlWn7RKGopWtRT9einq5DLV2LeroW9XStyl7PEkOJjh07qlWrVtbrrKwsSddGPuRf8fT09AIjIfLz9vYuMJIiPT3dGnFRu3ZtVa1atcBf2WjUqFGRt3CEhIQoJCTEep2amlrS6hjn6+vrVDv9SjFNZVjvG8XV9aSWzq0/9Swa+7prlWX7ROGopWtRT9einq5DLV2LeroW9XStylDPhg0bFvmzEm/f8PT0lL+/v/XPZrPJ29tbUVFR1jSZmZk6ePCgAgICilxO69atHeaRpKioKLVu3VrStXTnD3/4g06cOOEwzcmTJ1WvXr2SmgkAAAAAACoZp58p4ebmptDQUK1atUq7du1SQkKCZs2aperVq6tLly7WdJMmTdKSJUus16GhoYqJidF3332npKQkrVy5UrGxsXr88cetaXr16qUdO3YoMjJSp06dUmRkpHbs2KFHHnmknKsJAAAAAAAqmhJv3yhM7969lZmZqXnz5unChQtq2bKl3nrrLXl6elrTJCcnq27dutbrgIAAjRkzRkuXLtWyZcvk7++vMWPGONwacu+992r48OFauXKl5s+frwYNGmjkyJGFPk8CAAAAAABUbmUKJdzc3BQWFqawsLAip4mIiCjwXnBwsIKDg4tddo8ePdSjR4+yNAsAAAAAAFQiTt++AQAAAAAA4AqEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAI9xNNwDFS2kZbroJtxTq6VrU03WoJQAAAG5HjJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIt9zc3FzTjQAAAAAAALcfRkrcZGPHjjXdhFsK9XQdaula1NO1qKfrUEvXop6uRT1dh1q6FvV0LerpWpW9noQSAAAAAADACEIJAAAAAABgRNUJEyZMMN2I202LFi1MN+GWQj1dh1q6FvV0LerpOtTStaina1FP16GWrkU9XYt6ulZlricPugQAAAAAAEZw+wYAAAAAADCCUAIAAAAAABhBKIHbRmxsrMLCwnTu3DnTTQFQDhEREZoyZYrpZtxS6B9hwsSJE/XTTz+Zbkahdu/erX/84x/Kycm5aZ85cOBAbdmy5aZ9njPCwsK0c+dO080ok2+++UZ///vfb+pnjhw5UqtXr76pnwkzv+vb3d///nd988035V6OuwvaUqnZ7XZ999132r17t86cOSNPT0/5+/urc+fOuv/++1W9evVC59uyZYvmzZunRYsWlfqzYmNjNXHiRH3xxReqXbu2q1ahUrLb7Vq5cqVV91q1aqlp06Z69NFH1aFDB9PNq/QiIiKUkZHh8r9ZXJbtvqKJiIiwToKrVq2qO++8U40bN9Z9992nkJAQubu7rlssa71SUlI0atQol3TyN1P+2lapUkU+Pj7q0KGD+vfvr5o1a7rscwYPHqzK9DikytDfBQQEaM6cOapVq5bpppTKzdyPTQoLC9Nrr72m4OBgly97//79WrNmjY4dO6azZ89qxIgR6tGjh8M0ly9f1pIlS/TLL78oIyNDvr6+euihh/TEE09I+l9fVZgBAwaoV69eRX7+7t27lZqaqq5du1rvRUZGavv27Tp+/LguXryoTz/9VH5+fg7zHTt2TIsXL9bRo0dVpUoV3XffffrLX/7icL4WHR2tZcuWKSEhQXfccYe6d++u/v37q2rVqpKuXbisWLGi0HbNnTtXXl5e6tChg5YtW6Zt27apW7duRReyEPm3z/xatWql9957z6ll3WgjR47U6dOni/x527ZtVZGfiV/Y+c5vv/2mjz76SE888YSeffZZ9erVS4899tgN+fzKcl5U1uudimLLli2aNWtWsdO88847N6k1RTN5vL+Rx4ub4dY4apdRSkqKxo8frxo1aqhfv35q2rSpPDw89N///lebNm1SrVq11KVLF9PNvOXk1d3T01P9+/dXs2bNlJOTo5iYGM2dO1efffZZgXmys7NvmZNMmNe+fXv99a9/VU5Ojs6dO6eYmBgtX75cW7du1fjx4yv8wbkiy6vt1atXlZiYqM8++0wXLlzQmDFjXPYZNWrUcNmybrSy9HcmuLu7y9vbu8if531bXKVKxRlgWVn344pSy8uXL6tx48bq3r27Pv3000KnWbBggaKjozVq1Cj5+fnpwIEDmj17tmrXrq1u3brJ19dXc+bMcZjnl19+0bx580o8MV6/fr169OjhUIcrV64oMDBQHTt21IIFCwrMk5aWpnfffVd//vOfNXToUF28eFELFixQRESE9e1ofHy8wsPD1adPH40aNUppaWmaO3eucnJy9MILL0iSevXqpYcffthh2R9//LHc3Nzk5eVlvXf//ffr+++/dzqUkP63feZXEc9jwsPDrW0yPj5e77//vt5//335+vpKqphtLs7PP/+szz//XAMGDFBoaKgkqXr16hW2P7gZboXrnU6dOulPf/qT9XrmzJmqWbOmBg8ebL1Xs2ZNxcbGmmiepMpzvK+oKldP42JffPGFqlSpovDwcIfOys/PT/fcc88N/yYuJydHs2fPVkxMjOx2u+rWrasHH3xQPXv2tA7SeQlwYGCgVq1apczMTP2///f/NHToUN1xxx2SpNzcXK1evVqRkZFKS0uTv7+/evfuXaaD6M0wb948SdKUKVMc6m6z2axvTMLCwjRkyBDFxMRo3759euihhzRgwIAS65WQkKCvvvpKR48eVU5Ojvz9/fWXv/xFd911l/U5v//+u77++mslJCTIZrNp2LBhlfpP6Dhr//79+te//qXff/9dNWrUUOfOnTVgwADrxGP//v1avHixEhISVKVKFTVs2FCvvPKKMjIyrJQ6LCxMkvT0009b/1+ZVKtWzboAq1Onjpo1a6bAwEC9+eabWr16tcLCwvTzzz/r+++/V1JSkjw8PNS2bVsNGjRIderUkfS/kU/jx48vdHuKjY0tsl7Z2dlaunSptm3bpvPnz6tx48bq16+fwwE3v4sXL2revHnat2+fLl26JB8fHz322GN6/PHHb0K1nJO/tnXr1lWnTp2s4cil6fOuXr2qRYsWWd8ydu/eXVlZWUpKSrK+rbv+m7EJEybIZrOpRo0a2rRpk9zc3NStWzcNGDDAWq7dbtfs2bMVFRUlLy8vPfPMM1q7dq3uu+++G7oNl6a/S01N1fz58xUdHS1JCgwM1ODBg1W3bl1J177V3bVrl/r27aulS5cqPT1dd911l15++WVr1F1xfV/etvrmm29q6dKlOnHihGw2m4YPH271fdeP5Mv79u/VV1/V4sWLlZSUpKlTpyozM1NLly7V8ePHlZ2drSZNmmjgwIFq3bq1tW4XL17U4sWL9euvv+rChQvy8/PTM888ow4dOmj48OF65ZVXHC5Yo6KiFB4ers8++6zYYOR6pdmPS9rXsrOztXDhQu3atUsZGRny8vJSly5d9Pzzz0u69i1y9+7dderUKf3666+qXr26evbs6TAC4OLFi1q0aJF+/fVXZWZmqnnz5nrhhRf0hz/8QZLKXMuRI0dKkqZPny5JqlevniIiIiRJ//nPf7R8+XIlJibK29tbXbp00TPPPOPUBWSHDh2sb+7ylnu9uLg4devWzTqG+vn5afPmzTp8+LC6deumKlWqFPid7dq1S+3bty8wwiG/c+fOKTo6WgMGDHB4P69PO3r0aKHz7d69W1WqVNGLL75o7dsvvfSSXn/9dZ06dUr+/v7asWOHbDabtV/7+/vr+eef10cffaRnnnlGnp6eBS7V4T7dAAAgAElEQVRSU1NTdeDAgQIhQseOHfXll19ay3ZG/u2zMKdOndLnn3+uw4cPy9fX1wpM8jt8+LC++OILJSYmqlGjRnr22Wc1ZcoUvfPOO2rXrp0kKTExUYsWLdKBAwfk4eGhu+66S4MGDSr1vpR/5G7eSKnatWsXOv/58+c1ffp07dmzR15eXgoLC3M410xLS9PChQu1b98+SVLr1q01aNAgNWjQoFRtKa9169Zp8eLFevnllx3aldeHTps2TVLx/WVJfUJ+xR3nJSkrK0tz5szR9u3b5enpqdDQUKf6Dldx5npn7dq12rJli5KTk1WjRg3dfffdGjhwoO68805J/+vPXnvtNS1YsECpqalWABcVFaUlS5YoPT1dHTt21PDhw+Xh4SGp/NcqHh4e1rKka/uXh4dHkdv59u3bizxeStKPP/6o1atXKyUlxRoBFhoaWq6wuDTHe6n0x/zQ0FCtWLFC586dU1BQUIF1cEZGRobmzZungwcPKiMjQ/Xr11fPnj11//33W9OU5lwqPT1ds2fP1r59++Tl5aWnn366TO0pzG0bSmRkZGjfvn3q379/kempm5vbDW1DTk6O6tSpo1dffVW1a9fWkSNHrOGzDzzwgDXdgQMH5O3trfHjx+vMmTP66KOP1KBBAz355JOSpKVLl2rnzp0aOnSoGjZsqLi4OM2ePVs1a9asMEOD85w/f1579+5Vv379Cq17XqcnSStWrFD//v01cOBAubm5lapen3zyiZo2bar3339fVatWVUJCgkMnJklLlizR888/Lx8fH3311VeaOXOmpk+ffsN/3xVBWlqawsPD1bVrV40YMULJycn6/PPPVaVKFb3wwgu6evWqPvzwQ91///3Wt93Hjx9XlSpVFBAQoEGDBunrr7/WzJkzJemW+uahSZMm+tOf/qRdu3ZZFzPPPPOMGjVqpIyMDC1evFiffPKJJk6c6DBfUdtTcfWaNWuWkpOTNXr0aNWtW1d79uzRBx98oPDwcDVr1qxA25YuXaqEhASNHTtWXl5eSklJqRT3/icnJ2vv3r3WkOnS7MNr1qzRTz/9pOHDh6tJkybauHGjtm3bpubNmxf7WVu3blVoaKjeffddxcfHa8aMGWrRooX17U9ERITsdrveeecdeXh4aOHChcUOWXaF0vR3OTk5mjp1qjw8PKyhp19++aU+/PBDhYeHW/1SSkqKduzYoddff11XrlzRxx9/rKVLl2rYsGGSStf3LVq0yArWVqxYoSlTpmjmzJlWwH29rKwsffvtt3rppZdUu3Zt+fj46OjRo+rWrZsGDRokNzc3bdiwQeHh4ZoxY4Zq1aql3NxchYeH6/z58xoxYoQaNGigEydOKCsrS9WrV1fnzp31448/OoQSmzdvVocOHZwKJIpy/X5c0r72/fff69dff9Xf/vY3+fn56cyZMzpx4oTDMtetW6fevXvr6aefVmxsrL788kvVr19f9913n7W+NWrU0NixY1WzZk1t2bJFkyZN0scffywfH58y1zI8PFwvvviihg8frnvuucc6Kdy7d69mzpypQYMGqU2bNkpNTdXcuXOVlZVV6IVteQQEBOi3337TAw88IF9fXx06dEjx8fFF3paRnJysmJgYvfrqq8Uu9+DBg3J3d1eTJk2cak9WVpaqVq3qcOGQt50fPHhQ/v7+ys7OVrVq1Rzm8/DwUFZWlo4dO2ZdzOe3efNm1axZU/fdd5/D+76+vvLy8tL+/fudDiWKk5OTow8//FA1a9bU5MmTdeXKFX311VfKzs62prl8+bKmTJmiwMBAjRo1SmfPntVXX33lsJyzZ8/qnXfe0f3336+BAwfq6tWr+vrrrzV16lRNnjzZ5aNxVqxYoeeee07PPfecNm/erM8++0xt27aVr6+vrly5ookTJ6p169aaMGGC3N3dtWbNGr377rv66KOPiuxnXGXp0qVau3atXn/99RLPfYvrL0vTJ+Qp6bxo3bp1CgsLU69evbRnzx7Nnz9ff/zjH9W6detS9x3l5ez1jpubmwYNGiQ/Pz+lpqbqyy+/1JdffukQ2GVnZ2vt2rUaPXq0srOzNW3aNE2bNk3VqlXT3//+d2VkZGjatGnauHGjevbsKenmXquUdLyMjIzUN998oyFDhqhFixZKSEjQ7Nmz5e7urkcffbRMn1na6xtnjvlbt27VG2+8oStXrmjOnDn67LPP9Oabb5apfVlZWWrRooX69OkjT09PRUdHa86cOfL19VX79u2t6Uo6l5o1a5ZOnz6t8ePH64477tCCBQuUkpJSpjZd77YNJU6dOqXc3Fw1bNjQ4f2XX35ZFy5ckCR17drV2oBvBHd3d/Xr18967efnp+PHj2v79u0OoUSNGjU0bNgwValSRTabTcHBwYqJidGTTz6py5cva+3atfrnP/+pNm3aWMs5cuSINm7cWOFCiby622y2Eqft1KmTHnzwQYf3SqpXamqqevbsqUaNGklSoScR/fr1s771eeqpp/T2228rLS3NSihvZRs3bpSPj4/1LZPNZtPzzz+vOXPmqF+/fsrKytKFCxfUsWNHq3Z5tZT+N2zeFRcPFZHNZrOS6/z7YP369fXiiy/q1Vdf1ZkzZxy2leK2p8LqderUKW3fvl0RERHW8NhHH31UUVFRioyM1Isvvig/Pz+H50mcPn1azZs3V8uWLSVd+8a0otq7d68GDhyonJwcZWVlSZJ1oVSaPm/9+vXq3bu3ddE6aNAg7d27t8TPtdls1rIbNmyoTZs2KSYmRl26dNGJEye0b98+TZ482foWesSIEdY30TdKafq7mJgY/f7775o5c6b1zfLo0aM1evRoRUdHKzAwUNK1E5mRI0da21RISIh+/PFHazml6fueeuopa4TAiBEj9PLLL2vbtm0F+tk8OTk5Gjp0qMNIsvyjziRpyJAh2rVrl/bs2aNu3bopOjpacXFxmjZtmrXe9evXt6Z/8MEH9dZbbyktLU116tTR+fPn9euvv+q1114rskbOytuPS7OvnT59Wg0aNFCbNm3k5uYmX19fBQQEOCyvZcuW6tu3r6Rr29bRo0etUTaxsbGKj4/XvHnzrIuaZ599Vr/99pt+/vln9e7du8y1zPtG7M4773ToQ1auXOnwDVfeSICZM2daIb6rDBkyRHPmzNGIESOscHHw4MG65557Cp1+06ZNql27tjp27Fjsck+fPi0vLy+nL5rvuusuLVy4UN99952eeOIJXb58WYsXL5Z07QJdkoKCgrRu3Tr9/PPP6ty5s9LT0/Xtt986TJNfTk6OfvzxR3Xt2rVAmCFdG4VTlpPuvL4wv0ceeUQDBgxQdHS0EhMTHbbNQYMG6e2337am3bp1q3JycvTKK6/Iw8NDjRs3Vt++fTVjxgxrmh9++EFNmzZ1GHEyatQoDRkyRMeOHbOOGa7SrVs365vtfv36af369dq/f7+6deum7du3Kzc3VyNGjLC2wWHDhunFF1/Ub7/9pk6dOrm0LflFRUVp9+7dGjt2bKnOe4vrL0vTJ+Rxd3cv9rwoMDDQush97LHH9P333ys6OlqtW7cudd9RXs5e7+Qfgenn56cBAwZo6tSpGjlypMOIxrxwQZI6d+6sdevWae7cuVa/1bFjR8XGxqpnz543/VqlpOPlt99+qwEDBljnGX5+fkpOTtbGjRvLHEqU9vqmtMf8zMxMjRo1yuofhg0bprffflsnT54s08ijOnXqOITJ9evXV0xMjLZv3+4QSpR0LrVnzx5NmjRJf/zjHyVdG9FX1HOFnHXbhhJFmTRpkjXEOO+E+kb64YcftHnzZp0+fVqZmZm6evVqgQsOm83mcOCuU6eOjhw5IunasL2srCy9//77DvMUtpyKwJlbYgq7paKkej3++OOaPXu2fvrpJ7Vv31733Xefw0W1JDVt2tT6/7yh+Onp6bdFKJGUlKRWrVo5bE9//OMflZ2drVOnTqlp06bq0aOH3nvvPd11111q3769goODrU7xVpebm2udTB07dkwrVqxQfHy8zp8/b227qampDtuKs9vT8ePHlZubW+CbxOzs7AIXKXkefvhhTZ8+XcePH1f79u3VsWNHtW3btuwregO1adNGw4cPV2ZmpiIjI5WcnGzd1ysVvw9fvHhRdrvd4UTazc1NLVu21JkzZ4r93Py/B0ny8fFRenq6pGvbvZubm8OQWF9fX+v3daOUpr9LTExUnTp1HIa6169fXz4+PkpMTLROUHx9fR2epeHj4+MwWqY0fV/+WyyqV6+uJk2aKDExsci2Va1atcDInfT0dC1btkyxsbGy2+3KyclRZmamUlNTJV3bvr29vYs8MfvDH/6gJk2aaMuWLerbt6+2bdummjVr6u677y6hUqWXtx+XZl/r0aOHJk+erL/97W8KDAxUhw4d9Kc//cmhj8xft7zXu3btknStn8jMzNTQoUMdpsnKylJycrL1uiy1LMqxY8d05MgRrVq1ymGdMzMzZbfbXfYNq3TtW+NDhw7pjTfeUL169XTgwAEtWrRIfn5+BW43u3r1qrZs2aLu3buXeBtJZmZmoQFASRo3bqyRI0dqwYIF+vrrr1W1alU99thj8vLysvruoKAgDRw4UPPmzdOsWbNUrVo1PfXUUzpw4EChIcjevXt15swZhYSEFPqZHh4eyszMdLqteX1hfnn7cFJSkurUqeNwbG3ZsqVDoJSUlKQmTZo4jHi6PmQ4duyYDhw4UCD8kK5dJLk6lMg/sqVq1aqqXbu21Q8dO3ZMKSkpBUbrZGZmOuwLN0Ljxo118eJFLV++XAEBAQ6jbgtTXH9Zmj6htIo7LpW277hRirreiYmJ0cqVK5WUlKSLFy8qJydH2dnZstvt1jGzWrVqDiGHt7e3vL29HW4t8PLyso4vN/tapbjj5blz53TmzBnNmTNHc+fOtabJyckp1237pZ23tMf8ovqHpKSkMoUSOTk5+u6777Rjxw6lpaUpKytL2dnZBUaOleZcKn+/Uq9ePZedS922oYS/v7/1y80vbyO50cPMJGnHjh1asGCBdQ9pjRo1tGHDBv36668O0+V9O5Ff3saf998333yzwIVjYfOZ1qBBA7m5uSkxMVH33ntvsdNeP/ypNPUKCwtT165dtWfPHu3bt0/Lly/XSy+95PCtd3H1vJ3lnQyNGDFCoaGh2rt3r/7zn//o66+/1j/+8Y8in3dwK0lMTJSfn58uX76s9957T+3bt9eoUaPk5eWljIwMvf322w7DayXnt6e8C6bw8PACJ+7XD7fPc/fddysiIkJ79+5VdHS0wsPD9ec//1kjRowow1reWHfccYf1rdOQIUM0ceJErVixQmFhYaXu88ri+t+Dm5ub8f3amf6uMPkvUK7fVvJuactTmr7PWe7u7gVOxCMiIpSenq6//OUvqlevnqpVq6ZJkyYV2C+K88ADD+j7779X37599eOPP6p79+4uHWaetx+XZl9r0aKFIiIitG/fPkVHRysiIkJNmzbVP//5z1K1KScnR15eXpo0aVKBn3l6elr/78pa5uTk6Omnn9af//znAj9z5V/2yszM1JIlS/Taa69ZIx+aNm2q+Ph4rVmzpsAx4bfffpPdbi/VNlerVi3rW1pndenSRV26dJHdbrfOE9auXeswIueJJ57Q448/rrNnz6pmzZpKSUnRkiVLCn3ORWRkpAICAooM0s6fP1+muubvC2+U3Nxc3X333YXetpP/gZ2uUlw/lJubq2bNmhX6UGNX/vWlwvj4+OjNN9/UxIkT9e677+qf//xnsZ9ZXH9Z3j4hv+KOS6XtO8rLmeud06dPKzw8XA8++KD69eunmjVr6vjx4/rkk08c+qXC6lDYtmHqWqW47TTvvy+99FKRI2DKorzH+7x23iirV6/WmjVrNHjwYDVp0kTVq1fXkiVLCtwKXJpzqRvVzorzGO2brFatWgoMDNSGDRt0+fJlI204ePCgWrZsqUcffVQtWrSQv7+/0+mozWZTtWrVdPr0afn7+zv8q4gjJWrWrKmgoCBt3Lix0LoXd5JS2no1aNBAoaGhGjdunB544AFt3rzZpetQmTVq1EiHDx92uJjJu7c3/wlds2bN1KdPH02YMEHt2rWzHjro7u5+U/9m+82UkJCgffv2KTg4WCdOnFBGRoaee+45tW3bVo0aNbKSYmcUVq9mzZopNzdXdru9wD5bXNqc97T7kSNH6pVXXtFPP/10U0ZzldfTTz+tVatWKS0trcR9uEaNGvL29rZGgknXTmaKeuhdaTVq1Ei5ubk6duyY9d6ZM2eUlpZWruWWpDT9nc1mU1pamsPw8OTkZJ09e7ZUt7nlV1Lfd/jwYev/L1++rP/+978FRlOU5ODBg9afNmvcuLGqV6/uMCS+efPmstvtxY7A6Nq1q86cOaMNGzbo+PHjDg/aKq/8+3Fp9zVPT08FBwfrpZde0tixYxUTE6NTp05ZP89fN+nawx/zfjctWrRQenq63NzcCnxGSReFJdVSunaCeH0f0qJFCyUlJRX4PH9/f5ee4GdnZ+vq1asFLkCqVKlS6HFg06ZNatu2bYFh4oVp3ry5zp07V65n43h7e6t69erasWOHPDw8rG8Y87i5ualOnTry8PDQ9u3bVbdu3QIjMNPS0rR79+4ib2HKzMzUqVOnXP4w7EaNGiktLc1hVMyRI0ccTv4bNWqkhIQEh1Ea+ftG6VodExMT5evrW2BbcOWFbWk0b95cp06dUq1atQq05UaHEtK1b5YnTJigK1eu6N1331VGRkax0xfXX5bUJ+RX1vOi8vQdznDmeufo0aPKzs7WoEGD1Lp1azVs2LDQW56cVZGuVby9veXj46Pk5ORC+9CyKu31TWmP+UX1D84es/McPHhQ99xzj7p166ZmzZqpfv36OnnypFPLyDuXyt8Ppaamuuxc6rYNJSTpxRdfVG5urt58801t27ZNiYmJOnHihLZt26bff//d4UD86aefFvkns6RrG8+YMWP0yy+/lPrzGzRooOPHj2vPnj06efKkVqxYof379zu1Dp6enurZs6cWLVqkzZs369SpU4qPj9cPP/ygyMhIp5Z1swwdOlS5ubkaO3as/u///k8nTpxQUlKSfvjhB73++utFzldSvTIzM/XFF18oNjZWKSkpOnz4sA4ePOj0if2t4tKlS4qPj3f4d/fdd+vs2bPW07x3796txYsX69FHH9Udd9yhlJQULV68WIcOHdLp06ete9/yalivXj1lZWUpKipK586d05UrVwyvZdlkZWXJbrcrLS1N8fHxWrt2rSZOnKgWLVqoZ8+e8vX1VbVq1bRhwwYlJydr9+7dWrZsmdOfU1i9GjZsqC5dumjWrFnauXOnkpOTdfToUa1evdoaEn69ZcuW6ZdfftHJkyeVmJioXbt2yc/Pr0zDn2+2du3ayWaz6d///nep+rzQ0FCtXr1av/zyi06cOKGFCxfq7Nmz5UrmGzZsqKCgIM2dO1dxcXGKj4/XrFmzdMcdd9zwB9yW1N+1b99eTZs21cyZM3X06FEdPXpUM2bMUPPmzYu8ned6pe37vv32W0VFRem///2vPvvsM7m7uzv9Z+AaNGigrVu3KjExUUeOHNEnn3zi8K3UXXfdpZYtW2ratGnau3evUlJSFBUV5XBsvPPOOxUcHKyFCxeqTZs2ZX4yf0n7cWn2tbVr11rH/1OnTmnbtm3y9PR0uP3q8OHDWrlypU6ePKnIyEj9/PPP1n3X7du3V0BAgKZOnao9e/YoJSVFcXFx+uabb3TgwIFy1VK69m1mdHS07Ha7zp8/L+nas0G2b9+uZcuWKSEhQUlJSdq5c6f+9a9/OVW/y5cvW8eG3NxcpaamKj4+3joRrlGjhtq2baslS5ZY29aWLVv0008/FfgmMDU1VXv37i3y4v56zZs3l5eXlw4ePOjwvt1uV3x8vHWynJiYaN1Cl2fD/2/v3uN6vP/Hjz86vVU6RwdatdaJHBIzPrIkp/E1G6k5Hz7MYj528BG2fTA24+trbGsTFsohmbWUz+IjYcLwRQ5FChNJKaXQ4V3v3x9+XV9v75riXe9387rfbt3oel/X9X5dz17X67qu1/U6JCZy5coVcnJySExMJCIiglGjRik12d+1axfXr18nOzubn376iV9++YVJkyapVLAkJyfTokWLWludwKO/vYGBwTO9Ua3Jn4//1FTCdOzYkbZt2xIWFsa1a9fIyMhg06ZNSpVKvr6+6OrqsmbNGm7cuMHZs2eJjY0F/u9N5cCBA3nw4AGrVq3i8uXL3L59m7NnzxIeHs7Dhw8bnObn0bt3b8zNzVm+fDlpaWnk5eWRlpZGZGRkgx9+npWlpSULFixALpfz+eef11rp9bTysj5lwuOe9b7oecqOhqrv8469vT0KhYLdu3eTl5fH4cOH2b1793N/v7Y9qwQFBREXF0dCQgI5OTlcv36dgwcPSufXs6rP8019r/kymUypfFi3bh0+Pj5PvV7m5eWp3Ps/ePCANm3acP78eS5evMjNmzf58ccfGzxWTps2bfD29mbt2rXSvVRYWFidrXwb6oXtvgGP+vAsX76c2NhYtm/fTkFBAXp6ejg4ODBgwAClwU6e1sdTLpeTk5PDgwcP6lynpga85qLTv39/aWRThULBa6+9xtChQ5UGY6mP4OBgzM3NiY+PZ/369RgZGeHs7Ky2QXLUzdbWlmXLlhEbG8uWLVsoLCzE1NQUJycnlf6Xj3tavHR1dbl//z7ff/89d+/exdTUFB8fn1r7Wr4I0tPTmTNnjtKy1157jXnz5rF582bmzJlDy5Yt6dWrF6NGjQIeFYK3bt1i5cqV0lRYvXv3lvKSh4cH/fv3Z/Xq1ZSUlDTbKUHPnTsnDR7bsmVLXnrpJUaOHEm/fv3Q19fH0NCQGTNmsG3bNvbs2YOjoyPjx49X6Q/5NHXFa/r06fz8889s3ryZgoICTExMcHV1rfMh1MDAgOjoaPLy8jAwMMDd3f2ZR2DWhKFDh/L999+zevXqp5Z5Q4cOpaioiO+//x4dHR369OlD9+7dn6mlyuNmzJjBmjVrWLRoEWZmZgQHB0vxbExPK+90dHSYM2cOERER0swuHTt2ZPLkyfWuMKlv2TdmzBgiIyPJycnhpZdeIjQ0tMEz6ISEhLB27VpCQ0OxsrJi5MiRSjf+urq6zJ8/n6ioKL799lvKysqkKUEf17dvXw4dOvRc3Uuedh4DTz3XDA0NiY+P59atW+jo6ODs7Mz8+fOVmjQPGTKEP/74g59//hlDQ0OCgoKkAdJ0dHSYN28e0dHRhIeHU1xcjIWFBR4eHk+d6u5psQQYN24ckZGRhISEYGVlRVhYGN7e3sydO5edO3cSHx+Pnp4e9vb29OnTp0Hxy8rKUppNKCYmhpiYGPz8/KRBYD/44AO2bt3KN998Q2lpKa1btyY4OFhlMLj9+/djbGysMntFXXR1dfH39+fw4cNKFRx79+7lp59+kn7/6quvgEd/x5rjy8zMJCYmhrKyMtq2bcu7776rEuvTp0/z888/U1lZibOzM3PmzFEZt0ShULB//3569+5dZ5fdlJQUfH19n6lLb03+fJyVlZU049Xs2bMJDw9n/vz50pSgq1evltY1MjIiNDSU9evXM2fOHBwcHBg5ciQrV66Uyi0rKysWL17M1q1b+fLLL6moqKBVq1Z07txZWqdmut/HpxFtDC1atGDRokVs3bqVlStX8uDBAywtLfHy8nrqGA/qZGFhwYIFC1i8eDGLFi1SGjwUnl5e1qdMeNyz3hc9T9nRUPV93nFycmLixInExcURHR2Nh4cH48aNY9WqVc+dhvo8q9RM+13zb2MJCAigRYsWxMfHs23bNmQyGQ4ODs88yGWN+jzf1Peab2NjQ69evVi2bJnSlKBPU1vldGhoKMOHDycvL48vv/wSmUxGnz596N2795+2aqzN9OnTCQ8Pl+6lAgMD1TYbnI5C051uXyCHDx9mzZo1DX6bIQiC8CKbM2cOnp6eTJ48WW37vHfvHtOmTWPWrFlK01P+FdU8lKxfv16tYw48jyNHjrB27VrCw8ObZAynZzVjxgwGDhxY5xSYwrMrLi7mo48+YunSpbWO9aBpxcXFfPjhh3z11Vdak74TJ06wYsUKpVkOniY5OZmtW7eyatWqJq0cEISGmj59Ov379+ftt9/WdFI0KiYmht9//53/+Z//0XRSmtQL3VKiqdT0Sfz111+Vpl0RBEEQlOXn55Oamkr79u2Ry+UkJSXxxx9//Gkrqvo4f/48Dx8+xNHRkeLiYqKjozEzM3shBnDVJuXl5RQVFREbGyu9rRJeTObm5oSEhHDnzh2teeh/XH5+vjRFs6YcOHAAW1tbrK2tyc7OZuPGjXTt2rVBlYunT59mzJgxokJC0GrZ2dkYGBgwdOhQTSdF0BBRKdEEjhw5woYNG/Dw8FCZ+kcQBEH4Pzo6Ohw8eJCoqChpzu/58+crTef5LORyudQFRiaT4ebmxqJFixrcfUF4PnFxccTGxuLp6cmIESM0nRxBw2pm9dBGrq6uap9Ss6GKi4vZsWMHd+/excLCAh8fH8aMGdOgfXz00UeNlDpBUJ+XXnpJqfuS8OIR3TcEQRAEQRAEQRAEQdCIF3r2DUEQBEEQBEEQBEEQNEdUSvyJ0tJSpk6dWufcxJoUFRVFRESEppPRINocz5UrVxIfH6/pZDRIXl4eQUFBZGVl1XubAwcOvLCzkWijmJgYPv74Y00no1kLCgri2LFjdf7+V/Us578gCNpLm++RmuM9pzrMmDGDXbt2/ek6L0pZLPKn+ohY1k6MKfEnYmNj6dKlC3Z2dgBs2LCBS5cukZ2djYWFBWFhYSrbHDlyRJrP3MzMjEGDBqmM2p2YmMiePXvIy8ujVatWDB8+HD8/P+nzhQsXkpaWprJvBwcHVq5cCcCwYcOYOXMmQ4YMwdbWVp2H3Wg0FU+ABw8eEB0dze+//05JSUsDXlgAAB+2SURBVAnW1taMGjWKv/3tbwAEBgayYMECAgICMDY2bqQI1F9YWBglJSXMnTtXaXlWVhbz5s3ju+++o1WrVqxduxZTU1MNpfKvLywsjIMHDwKPpvK1trame/fuBAUFibEIntPjsQUwNTXFzc2NcePG0bZtWw2mTPOeNp2cn5+fyhSbL7KavOTv709ISIjSZ5s3b2bXrl34+PiolKeCoE3EPWfTKioq4pdffuHUqVMUFBRgZGSEnZ0dvXr1wt/fH0NDQ5YuXSoG4/3/RP5UHxHL2olKiTqUl5ezf/9+QkNDpWUKhQI/Pz+uX7/O2bNnVbY5ffo033zzDZMmTcLb25ubN28SHh6OTCaT5r7du3cvW7ZsYdq0abi5uZGZmUl4eDgtW7aUBnyaPXs2crlc2m9lZSWzZ8+mZ8+e0jIzMzM6derE3r17m8Wbb03GUy6Xs2TJEkxMTPjwww+xsrKisLBQmscewNHREVtbWw4dOvTc8xQ3FV1dXSwsLDSdjL+8jh07MnPmTORyORcvXmTNmjWUl5czdepUTSet2auJLUBhYSGbN29mxYoVfP311xpOmWatXbtW+v///u//Eh4errRMJpNRWlqqiaQBj8rUx8tPbWBtbc3Ro0eZNGmSVGFYVVXFoUOHaNWqlYZT93TaGFOh6Yh7zqaVl5fHZ599hrGxMcHBwTg5OSGTycjOziYpKQlTU1N8fX2fOsvJ43H7KxP5U31ELOsmroB1OH36NAAeHh7SssmTJwOwa9euWjPNoUOH6Nq1KwMHDgTA1taWt956i7i4OAYOHIiOjg6HDh0iICAAX19faZ2srCzi4uKkTGNiYqK0399++43y8nL8/f2Vlnfr1o1t27Zp/QkImo3ngQMHuHfvHp9//rl001fbFF/dunUjJSWl2VRK5OXl8f7777N06VJpZoJTp06xadMm7ty5g6urKwMGDGD16tV89913Ssd87tw5Nm7cSF5eHq6uroSEhGBjY0NZWRmTJk1i0aJFuLu7AxASEkKLFi1YtWoVAGfPnuW///u/2bBhA/r6+iQkJHDgwAFu376NsbExXbp0Ydy4cbRs2ZKysjKmTZtGSEgIPXr0kL7/7NmzLF26lB9++EHrK1YMDAykNPr6+nL+/HlOnDjBlClT2LVrF/v27aOwsBA7OzuGDRvG66+/Lm27ZcsWjh8/zp07d7CwsKBnz54EBQUhk8lq/a47d+6wZMkS6W+ip6fXJMeoKY/H1sLCgiFDhrBs2TIqKiooKipSyd/wqBXBRx99pJSf/moePydqpvF78jypqZTIz89n69atXLp0idatWzNp0iQ6deokrXfjxg2ioqJIT09HJpPRoUMHJk6cKO2vurqan3/+maSkJIqLi7G3t+edd97h1VdfBf6vnPnHP/5BUlISGRkZjBkzhu3bt2vVee3k5MTdu3c5evSodK08deoUBgYGtGvXTqkSJzk5mV27dklvk/r378/gwYPR1X3Uo/XPyjR41PLuxx9/JDU1lYcPH2Jpackbb7zBkCFDgNrz6IwZMxg4cKD0ZisoKIjJkydz/vx5UlNT6d+/P+PHj+fkyZPs2LGDGzduYGFhga+vLyNHjpSuXb///js7duzg1q1byGQyHB0d+fDDD7W+HBX+nLjnbFrr169HV1eXpUuXKrV6tLGxoWvXrtTMAVCf8/bJe0a5XE5kZKTUMtfc3BxfX98Gz5qiTUT+VB8Ry7qJSok6pKen4+Ligo6OTr23qaysxMDAQGmZTCajoKCA/Px8bGxsqKysVHkgkclkZGZm1vmmJCkpCW9vb5W3Pa6urhQWFpKbmys1AdJWmozniRMn8PDwICIighMnTmBiYkLPnj0ZPny4UrxdXV3ZuXMnFRUVdT40arM7d+6wYsUKBg4cSP/+/bl+/TqbNm1SWU8ul/PLL78QEhKCgYEBYWFhrFu3jk8++QRDQ0NcXFxIS0vD3d2d3Nxc7t+/z7179ygqKsLCwkL6rCZ2Ojo6TJw4ERsbG+7cuUNERAQRERHMnDkTQ0NDevXqRXJystIN+v79+/Hx8WmWN9IymYyqqiqio6M5duwYf//732nTpg0ZGRmEh4djYmKCj48PAC1atCAkJAQrKytu3LjBunXr0NfX55133lHZ740bN/jiiy/o0aMH48ePb9C58lfw8OFDjhw5gqOjY7M8/zQlOjqasWPHMmXKFHbu3MmqVav4/vvvMTQ05O7duyxYsAB/f3/GjRtHVVUV27ZtY/ny5SxZsgRdXV3+/e9/Ex8fz9SpU3FxceG3335jxYoVLFu2DGdnZ+l7am5QairLbty4oXXntb+/P8nJydINVs3/b9++La2zb98+YmJimDx5Mi4uLly/fp3w8HD09fWlh4s/K9PgUcyvX7/O3LlzMTc3Jy8vj3v37jU4vT/99BOjRo1i3Lhx6OjocObMGb799lsmTpxIu3btuHPnDuvWraOyspLx48dTVFTEqlWrGD16NK+99hplZWVcvnxZDZETNE3cczadkpISUlNTGTVqVJ3dMP/s7/DkefukX3/9lRMnTjBr1ixsbGwoKCggJydHbenXBJE/1UfEsm5ioMs65OfnY2lp2aBtvL29OXnyJKmpqVRXV5OTk0NCQgLwqO8aQOfOnUlOTiYzMxOFQkFWVhZJSUlUVVVRUlKiss+cnBzS0tIICAhQ+awmffn5+Q09vCanyXjevn2bY8eOIZfLmTdvHsHBwfznP/9h69atSt9naWlJVVUVhYWFajji53fmzBnGjRun9LNgwYI619+7dy+2trZMmDCBNm3a0KNHD/r376+yXlVVFX//+99xdXXFycmJoUOHcuHCBenNQPv27blw4QIAFy5cwNPTEzc3N86fPy8ta9++vbS/IUOG0KFDB2xsbGjfvj1jx47l6NGjVFdXAxAQEEBqaqoU19LSUk6cOEHfvn3VE6gmlJmZSUpKCl5eXiQkJPDee+/h7e2NjY0Nvr6+BAQEsGfPHmn9wMBAPD09sbGxwcfHh7fffpuUlBSV/V6+fJkFCxbQv39/JkyY8MJUSDyexydMmEBaWhr/+Mc/NJ2sZmXIkCF069YNe3t7Ro8eTWlpKdeuXQMelQlOTk6MHTsWBwcHnJyceP/998nMzOTKlSsAxMfHM3ToUHx9fWnTpg3BwcG0a9dOZXC3QYMG0aNHD2xsbLC2ttbK89rX15esrCxu3bpFUVERZ86coU+fPkrr7Ny5k7Fjx0rH0q1bN9566y2l8/ZpZVp+fj4vv/wyrq6utG7dGi8vL6Xmr/X1t7/9jYCAAGxtbbGxsSE2NpahQ4fi7++PnZ0dHTp0YMyYMfznP/9BoVBQWFhIVVWVlHZHR0cCAgKaZeWuoEzcczad3NxcFAoFbdq0UVr+3nvvSdejx7vLPenJ8/ZJ+fn52Nvb065dO1q1aoWHh4fKm+jmRuRP9RGxrJtoKVGH2mqcniYgIIDc3FyWL19OVVUVRkZGDB48mB07dkgPGYGBgRQVFfHZZ5+hUCgwNzfHz8+PXbt21fogkpSUhKWlpfTm9XE16auoqHiGI2xamoynQqHAzMyM9957D11dXVxcXCgtLWXTpk1KNd3aFs927doxbdo0pWXXr19nxYoVta5/8+ZNpWbuAG5ubirrGRgYKF2MLS0tkcvl3L9/HxMTE7y8vEhMTEQul3PhwgW8vLwoLy8nLS2NV199laysLKVmiOfPnyc2NpabN2/y4MEDqqurkcvlFBUVYWVlxSuvvIKjoyMHDhxg+PDhHD58GBMTE7p06fI84WkyNQ/ONcf16quvMnToUI4dO8aXX36ptG5VVRWtW7eWfj927Bi7d+8mNzeXsrIyqqurpQebGoWFhSxevJjAwECVQYv+6h7P46Wlpezdu5cvvviCL774QsMpaz6cnJyk/9fcSBQXFwNw5coV0tPTa22CmZubS5s2bbh7965SM1IAT09PqYlpjSfLFm08r01MTOjevTvJyckYGxvj5eWl9Abo3r17FBQUsHbtWtatWyctr66ulipl4ell2oABA1i5ciVXr16lY8eOdOvWTamitr5cXFyUfr9y5QqZmZnExcVJyxQKhdSdydnZmY4dO/Lxxx/TqVMnOnXqRI8ePZ7a713QfuKeU/M+//xzqqurCQ8Pp7Kyss71njxvn9SnTx+WLFnCrFmz6NSpEz4+Pnh7e0vdw5ojkT/VR8SybqJSog6mpqYNHkhMR0eHsWPHMnr0aIqKijAzM+PcuXMA0gimMpmM6dOn8+6771JcXIylpSX79u3DyMhI5cZCLpdz8OBBAgICau1bXpO+5nBDosl4WlhYoK+vr3RBaNu2LeXl5ZSUlEjraVs8W7RoodJs6v79+8+93ycvjDWFVc3DsqenJ3K5nKysLNLT0xk8eDDl5eWsXbuWS5cuoaenh6urK/CoFnXp0qUEBAQQHByMiYkJV69eZfXq1UqD6fTt25dff/2V4cOHk5ycjJ+fX7O5QNc8OOvp6WFpaYm+vr7UZDo0NFSl2VvNuZqRkcGqVasIDAxkwoQJtGzZkpMnTxIVFaW0vqmpKa1btyYlJYW+ffuq9Pn7K3syj7u4uDBhwgT27dtHv379AJQeFl+UQcUa4vFrw+MVsTX/dunShfHjx6tsZ25urhTbp6ltBHptPK/9/f0JCwvD0NCQ4OBgpc9qyripU6eqVMTUqE+Z1qVLF8LCwjhz5gznzp1j6dKl9OzZk+nTpwOP/g5Pxra2vPtk0/Hq6moCAwNrbXVhZmaGrq4un376KZcvXyY1NZX9+/ezdetWFi5cqNTVRmh+xD1n07Gzs0NHR4ebN28qLa9p9fC02TaeNvOWi4sLYWFhpKamcu7cOcLCwnBycuLTTz/VePn4rET+VB8Ry7o1z7OjCTg7O6sUWPWlq6uLlZUV+vr6pKSk4O7urvKH1dfXx9raGl1dXVJSUvDx8VEprI4fP05JSUmdzWGzs7PR09PD0dHxmdLZlDQZTw8PD3Jzc5XeUN+6dYsWLVooTaeZnZ2NlZVVs20K27ZtW5V5sjMzMxu8n5pxJZKSknjw4AEuLi64ublx584dDh8+rDSeRFZWFnK5nIkTJ+Lu7i69eX1S7969KSgoIDExkatXrzarpow1D86tW7eWjtvBwQEDAwPy8/Oxs7NT+qlpKXHp0iWsrKwIDAzE1dUVe3v7WpvCGRgYEBoaiomJCUuWLFFLxVNzpqurS0VFhXSO1zRNBKRuCUL9vPzyy9y4cYNWrVqp5FMjIyOMjY2xtLTk0qVLSttdvHgRBweHp+5fG8/rjh07oq+vT0lJiTRYZw0LCwssLS25ffu2SjxqKsfqW6aZmZnx+uuvM2PGDEJCQjh48KD0dtXMzExpm6KiIqV8XBcXFxdu3rxZa9pqbhx1dHRwd3dn5MiRLF26FEtLS44cOfLM8RK0g7jnbDqmpqZ06tSJxMREysrKGuU7jIyM6NGjB1OnTmXu3LmcP3+e3NzcRvmupiDyp/qIWNZNtJSog7e3N1u2bKGkpER6cK1pgn337l3kcrl0g+zg4IC+vj737t3j2LFjtG/fHrlcTnJyMkePHmXRokXSfnNycsjMzMTNzY379++TkJBAdnY2M2bMUElDUlISHTp0qHOe2PT0dNq1a9cs5lDWZDwHDBjAnj172LhxI4MGDSIvL4+YmBgGDBig1KQpPT2dzp07N01AGkH//v1JSEggMjKSfv36kZ2dzb59+4A/H7SpNu3btychIYHOnTujq6uLTCbDzc2N3377jcDAQGk9e3t7FAoFu3fv5rXXXiMjI4Pdu3er7K9ly5b06NGDyMhI2rVrh729/fMdrIYZGRkxdOhQoqKiUCgUtG/fnrKyMjIyMtDV1aVfv37Y29tTWFjIb7/9hru7O6mpqbWOJwGParhDQ0P56quvWLJkCZ9++qk00v9fWWVlpfSwVlpaKt0kdu3aVcpzcXFx2Nra8uDBA5VxYIQ/N3DgQJKSkli1ahXDhg3DzMyM27dvc/ToUcaPH4+RkRFvvvkmMTEx2NnZSQNdpqens2zZsqfuXxvPax0dHVasWIFCoVAZGAwejZ4fERGBsbExPj4+yOVyrl69SmFhIW+//Xa9yrTt27fz8ssv89JLL1FVVcXvv/+OjY2N9H1eXl7s2bMHDw8PdHV12bZtW61pedKIESNYtmwZrVu3pmfPnujp6ZGdnU1mZiZjx44lIyODc+fO0blzZywsLLh69SoFBQX1qkAStJu452xaU6ZM4bPPPiM0NJSRI0fi7OyMrq4uV65c4Y8//lCawaihEhISsLCwwNnZGX19fQ4fPoyRkRHW1tZqPIKmJfKn+ohY1k1UStTB0dERV1dXpSki16xZQ1pamrTOnDlzAJSmWzx48KDUPNvd3Z2FCxdKTd3hUfPMhIQEcnJy0NPTw8vLiyVLlqgMlnP79m3Onz/PrFmz6kxjSkoKQUFB6jngRqbJeLZq1YpPPvmEyMhI/vnPf2JhYYG/vz8jRoyQ1qmoqOD48eN88sknjReERta6dWs+/vhjIiMj2bNnD6+88gqBgYH88MMP9bohfpyXlxdxcXFK/aTbt29PWloaXl5e0jInJycmTpxIXFwc0dHReHh4MG7cOGn60Mf17duXQ4cONcsBLmsTHByMubk58fHxrF+/HiMjI5ydnRk2bBjwaEqlN998k40bN1JRUUHnzp0JDg5m/fr1te5PJpMxd+7cF6pi4ty5c7z77rvAo4qeNm3a8OGHH0p5LCQkhPDwcObNm4etrS1Tpkz508FeBWVWVlYsXryYrVu38uWXX1JRUUGrVq3o3LmzVCa88cYbPHz4kC1btlBUVESbNm34+OOP690dQBvPayMjozo/CwgIoEWLFsTHx7Nt2zZkMhkODg7Sdak+ZZqBgQHR0dHk5eVhYGCAu7u70pzz48ePZ82aNSxcuBALCwvGjBlTrzdj3t7ezJ07l507dxIfH4+enh729vbSYJ3GxsZcunSJxMRE7t+/j7W1NSNGjFCahlhonsQ9Z9OytbVl+fLlxMbGsn37dgoKCtDT08PBwYEBAwY819TwhoaGxMfHc+vWLXR0dHB2dmb+/Pla/7D8Z0T+VB8Ry7rpKBrSqfQFc+bMGTZs2MDXX3+tdf3ATp06RVRUFCtWrKi1P5A20uZ4JiYmcvLkST799FNNJ0Wt/v3vf7N9+3Y2btyo8Rkdjhw5wtq1awkPD2/WF2dBEP6POK8FQT20+R6pOd5zCuol8qf6iFjWTm/hwoULm/QbmxE7OzsUCgWWlpZa98by6tWr9OnTR2WQPW2mzfG8du0aAwYMUBpjojlKTEwEHg1ic/r0abZu3crrr7+u0RHxy8vLKSgoICIigp49e9Y60q8gCM2LOK8FQb20+R6pOd5zCuol8qf6iFjWTrSUEIS/kI0bN3L06FFKS0uxsrKiV69eBAYGSgM0akJMTAyxsbF4enryz3/+E2NjY42lRRAE9RDntSAIgiAI6iIqJQRBEARBEARBEARB0Ajt6sgiCIIgCIIgCIIgCMILQ1RKCEIzVFpaytSpU7Vy3uuoqCgiIiI0nYwGEfFUL22O57x58zh27Jimk1Fv2hxLkTfVa+XKlcTHx2s6GYLwl6DN53pzLDsFobGJKUEFoRmKjY2lS5cu2NnZAbBhwwYuXbpEdnY2FhYWhIWFqWxz5MgRYmNjuXXrFmZmZgwaNIg333xTaZ3ExET27NlDXl4erVq1Yvjw4fj5+UmfL1y4UGnaohoODg6sXLkSgGHDhjFz5kyGDBlS5xzI2kbEU70aK56HDx8mLi6OW7duYWRkRMeOHRk/fjwWFhbSOseOHWP79u3cvn0bW1tbRo0aRffu3aXPR4wYQWRkJN27d9e6Ua9rI/KmemkqnvBoNqS9e/eSn5+Pqakp3bp1Y+zYsRgaGgIQGBjIggULCAgIEGN0CMJzEmWnIDQvolJCEJqZ8vJy9u/fT2hoqLRMoVDg5+fH9evXOXv2rMo2p0+f5ptvvmHSpEl4e3tz8+ZNwsPDkclk0jzJe/fuZcuWLUybNg03NzcyMzMJDw+nZcuWdOvWDYDZs2cjl8ul/VZWVjJ79mx69uwpLTMzM6NTp07s3buXcePGNVYY1EbEU70aK54XL17k22+/Zdy4cXTv3p2ioiJ+/PFHvvnmG/71r38BkJGRwapVqwgKCqJ79+4cP36clStXsnjxYtzc3ADw8fEhPDycM2fOaP2MESJvqpcm43n48GE2b97Me++9h6enJ3l5efzwww9UVlYSEhICPJq/3tbWlkOHDkn7FgSh4UTZKQjNj/a/JhIEQcnp06cB8PDwkJZNnjyZN954A3t7+1q3OXToEF27dmXgwIHY2tri4+PDW2+9RVxcHDVj3R46dIiAgAB8fX2xtbWlV69e9OvXj7i4OGk/JiYmWFhYSD8XL16kvLwcf39/pe/r1q0bKSkp6j70RiHiqV6NFc+MjAysra35r//6L2xsbHB3d2fQoEFcvnxZ2s/u3bvx8vJi+PDhODg4MHz4cLy8vNi9e7e0jq6uLl26dOHw4cONcfhqJfKmemkynpcuXcLNzY3XX38dGxsbOnTogJ+fH5mZmUrf15ziKQjaSpSdgtD8iEoJQWhm0tPTcXFxQUdHp97bVFZWYmBgoLRMJpNRUFBAfn6+tI5MJlNZJzMzU6nW/3FJSUl4e3urzGfs6upKYWGhVvblfJKIp3o1Vjw9PT25e/cuJ0+eRKFQcO/ePY4cOUKXLl2kbTIyMujcubPSfjp37kxGRobSMldXV9LT0xt6aE1O5E310mQ8PT09uXbtmpQX79y5w8mTJ5XyLzyKZ2ZmJhUVFQ0+PkEQHhFlpyA0P6JSQhCamfz8fCwtLRu0jbe3NydPniQ1NZXq6mpycnJISEgAoKioCHj08JacnExmZiYKhYKsrCySkpKoqqqipKREZZ85OTmkpaUREBCg8llN+mou5NpMxFO9Giue7u7ufPDBB3z77beMHj2aKVOmoFAoeP/996X9FBUVYW5urrRvc3NzaR81rKysKCwspKqq6lkOscmIvKlemoxnr169GDVqFAsWLGDUqFFMnz4dR0dHxowZo/R9lpaWVFVVUVhYqIYjFoQXkyg7BaH5EWNKCEIzU1tN/dMEBASQm5vL8uXLqaqqwsjIiMGDB7Njxw7pTUJgYCBFRUV89tlnKBQKzM3N8fPzY9euXbW+bUhKSsLS0rLWfvk16WsOb/tEPNWrseJ548YNIiIiGDFiBJ07d+bu3bts3ryZtWvXKlVM1IdMJkOhUFBZWYmenl6Dtm1KIm+qlybjmZaWxs6dO5kyZQpubm7k5uayYcMGYmJiCA4Olr6vOcVTELSVKDsFofkRlRKC0MyYmppSWlraoG10dHQYO3Yso0ePpqioCDMzM86dOwcgjfwsk8mYPn067777LsXFxVhaWrJv3z6MjIwwMzNT2p9cLufgwYMEBATU+lBXk74nt9NGIp7q1VjxjI2NxdXVVRoJ3cnJCUNDQ/71r38xatQorK2tsbCwoLi4WGnfxcXFSrNzwKN4GhgYSLMeaCuRN9VLk/GMjo6mV69e0htTR0dHysrKCA8PJzAwUIptc4qnIGgrUXYKQvMjum8IQjPj7OzMzZs3n2lbXV1drKys0NfXJyUlBXd3d5ULor6+PtbW1ujq6pKSkoKPj4/K1InHjx+npKSEvn371vo92dnZ6Onp4ejo+EzpbEoinurVWPEsLy9XiVvN7zWDkLm7u6uMqn727Fnc3d2Vll2/fh0XF5dnSmNTEnlTvTQZz7ryb03erZGdnY2VlZVKRZogCPUnyk5BaH5ESwlBaGa8vb3ZsmULJSUlmJqaApCbm0tZWRl3795FLpdz7do14NG82Pr6+ty7d49jx47Rvn175HI5ycnJHD16lEWLFkn7zcnJITMzEzc3N+7fv09CQgLZ2dnMmDFDJQ1JSUl06NChzvm109PTadeuHS1atFB/ANRMxFO9Giue3bp1Izw8nL1790rdNzZt2sTLL78sDSA2ePBgFixYwC+//MKrr77K8ePHuXDhAp9//rlSGi9evKgyIKY2EnlTvTQZz65du7J7925eeeUVqfvG9u3b8fHxUXqLmp6e3izypiBoM1F2CkLzIyolBKGZcXR0xNXVlZSUFGnu7DVr1pCWliatM2fOHAC+++47bGxsADh48CBRUVHAozfKCxcuxNXVVdqmurqahIQEcnJy0NPTw8vLiyVLlkjb17h9+zbnz59n1qxZdaYxJSWFoKAg9RxwIxPxVK/GimefPn14+PAhiYmJREZGYmxsTIcOHZQGCvTw8OCDDz4gOjqa7du3Y2dnxwcffICbm5u0TmFhIZcuXWLmzJmNFwQ1EXlTvTQZzxEjRqCjo8P27dspKCjAzMyMrl278s4770jrVFRUcPz4cT755JPGC4IgvABE2SkIzY+O4sm2g4IgaL0zZ86wYcMGvv76a5Umg5p26tQpoqKiWLFihVYPIvg4EU/10uZ4RkVF8eDBA6ZNm6bppNSLNsdS5E31SkxM5OTJk3z66aeaToogNHvafK43x7JTEBqb3sKFCxdqOhGCIDSMnZ0dCoUCS0tLWrZsqenkKLl69Sp9+vRRmZNbm4l4qpc2x/OPP/5gyJAhWj/IZQ1tjqXIm+p17do1BgwYIDU3FwTh2Wnzud4cy05BaGyipYQgCIIgCIIgCIIgCBqhXe2ZBEEQBEEQBEEQBEF4YYhKCUEQBEEQBEEQBEEQNEJUSgiCIAiCIAiCIAiCoBGiUkIQBEEQBEEQBEEQBI0QlRKCIAiCIAiCIAiCIGiEqJQQBEEQBEEQBEEQBEEjRKWEIAiCIAiCIAiCIAga8f8AJ8lyv5c7sbAAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":732},"id":"8xp79OV8wwdh","executionInfo":{"status":"ok","timestamp":1633623150144,"user_tz":-330,"elapsed":46,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b8a17815-aa7f-4e6e-d818-4f9d3b13ad12"},"source":["plot_components(components_df, 'fc1', ascending=True)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABCUAAAMlCAYAAABEr0kcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3yMd97/8fdkJgdEMiFCJcQp4rCoolFJG4egVZQ2LaVtiFItuna33ZW7a5PqQdR9d1XdSrWNoAfSbYvSIlm0KD2pOBRRshVUKsmgSIhcvz/8MrfpJEgyMg6v5+Oxj+18r+/3e32uz8yVh/nMdX0vk2EYhgAAAAAAAKqZh7sDAAAAAAAANyeKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAwE1ixIgRMplMys7OdncoVVaZY1m3bp1MJpOSkpKqvP/s7GyZTCaNGDGiynPdTLKysjR48GA1aNBAJpNJVqvV3SEBLte9e3eZTCZ3hwEA1w2KEgDgAiaTSSaTSR4eHvrpp5/K7dejRw973/nz51dfgDcBCgWuVZUiTlnvw/nz5zVo0CCtXLlS/fv3V2JioiZNmlThuW02m6ZPn67hw4erTZs2slgsMplMSk9PL3dM9+7d1aRJkwrvC9cuVxYZAQDuZXF3AABwo7BYLCouLtbbb7+tl19+2Wl7VlaW1q1bZ+9X3aZOnapJkyYpODi42vftajfSsdwsDhw4oF27dmn06NF68803Kz1Pdna2/vrXv0qSQkJCFBgYqKNHj7oqTAAAUM24UgIAXKR+/frq3LmzUlJSyiw6vPXWW5KkAQMGVHdokqRbbrlFrVq1kqenp1v270o30rHcLA4fPixJatiwYZXmCQ0NVXp6uvLy8nTw4EHdfffdrggPAAC4CUUJAHCh0aNH65dfftGnn37q0H7u3DnNnz9f3bp1U5s2bcodn5WVpccee0zBwcHy8vJSw4YN9dhjjykrK8uh39ixY2UymbR06dIy59myZYtMJpNiY2PtbZdah2HLli2KjY1VgwYN5OXlpUaNGumJJ56wf5G82P79+zVmzBi1aNFCNWrUUJ06ddSuXTuNHTtWeXl5l0qPpAtfSsu6wiE0NFQmk0kvvPCCQ/tnn30mk8mkf/zjH+UeS1JSkpo2bSpJSk1Ntd8iU95tMj/88IPuvfdeWa1W1axZU9HR0dq0adNlY78SR44c0bhx49SkSRN5eXmpXr16uv/++/Xdd9859Fu1apVMJpOee+45h/a1a9faYz948KDDtiFDhshkMmn//v0O7bt379aIESPUqFEjeXl5qX79+ho2bJj27NnjFN/Ro0f1zDPPKDw8XLVq1ZLValV4eLhGjBhhn3fEiBHq0aOHJOn55593yOe6desqnBOTyaTo6Gin+S6+9P78+fOaM2eOIiMj5e/vrxo1aqhFixZ6/PHHHT7/AQEB6tWrl+rUqVPhOCpi9+7dio+PV5MmTeTt7a2goCDdeeedeuONN5z6ZmRk6O6771adOnXk7e2tli1batKkSTp+/LhT39L1Bs6dO6cpU6aoefPm8vHxUXh4uObNm2fvN2fOHLVr1041atRQSEiIEhMTVVJS4jDXxbcs7d69W4MGDVKdOnVUq1YtRUVFafXq1WUeW1FRkZKTk9WuXTvVrFlTfn5+uvPOO7VkyRKnvhfvIzs7W0OHDlVgYKB8fHzUuXNnp791F3v//ffVo0cPWa1W+fj4qHXr1nrxxRdVVFTk1NdkMql79+46duyYxowZo1tuuUXe3t5q27atUlJSHPpW9vN56NAhmc1mdezYsdw+99xzj0wmk3bs2GFvmz9/vh544AE1a9ZMNWrUkJ+fnyIjI7Vo0aJy5/m9+fPnX/K2vdLj/73i4mLNnj1bXbt2lZ+fn2rWrKmOHTtq1qxZTp8HSVq2bJl69eplz1/Dhg0VHR2t2bNnX3GsAFDduH0DAFzo4Ycf1p///Ge99dZbGjRokL192bJlys3N1bRp07Rv374yx37zzTeKiYnRyZMnNXDgQLVp00a7d+/WokWLtHTpUqWnp6tLly6SpLi4OM2dO1cLFizQfffd5zRXamqqJF3R+grvvPOOxowZI29vbw0cOFCNGjVSVlaW3nrrLS1fvlybN29W48aNJV34wt2lSxedOHFC/fr10wMPPKDCwkIdOHBACxcu1Pjx41W3bt1L7q9nz5569913tXv3brVq1UqStG/fPv3888+SLnzBmzx5sr1/RkaGJKlXr17lztm9e3fZbDa99tpr6tChg0Pub731Voe+3377rV555RXdcccdevzxx/Xzzz/rX//6l3r16qUffvhB4eHhl81ZeQ4cOKCoqCgdPnxYPXv21MMPP6yDBw8qLS1NK1as0L/+9S/1799fknTnnXfKy8tLGRkZeumll5yOt/S/S99DwzC0du1aNWnSRM2aNbP3+fzzz3X//ffr3LlzGjBggFq0aKGcnBx99NFHWrFihdauXavbbrtNknT69GlFRkbqp59+Uu/evTVgwAAZhqH//Oc/Wrp0qWJjY9WsWTN7/lJTUxUdHe3wZakyazMkJiYqOzvbab7S/z979qz69++vNWvWqFGjRho2bJj8/PyUnZ2tjz/+WFFRUQoLC6vwfitrxYoVevDBB1VUVKS7775bDz/8sGw2m7Zt26ZXXnlFTz75pL3v3Llz9eSTT6pWrVp68MEHFRQUpHXr1mnatGlavny5Nm7cWOaCnkOHDtWWLVvUr18/eXp66sMPP9SYMWPk6empzMxMpaamqn///urVq5eWLVumKVOmqGbNmvrb3/7mNNeBAwd0xx13qF27dnriiSd05MgRLV68WPfcc4/ee+89DRkyxN737Nmz6tu3r9avX69WrVpp3LhxOn36tD788EMNGTJEP/zwQ5m3n/3nP//R7bffrmbNmunRRx9Vfn6+Fi9erPvuu0/p6en2IkGp+Ph4paSkKCQkRA888ICsVqs2b96syZMnKyMjQ2vWrJHF4vjPUJvNpsjISHl5eSk2NlZFRUVKS0tTfHy8PDw8FBcXJ0mV/nwGBwcrJiZGq1ev1vbt29WuXTuH7UeOHNGaNWvUqVMn/eEPf7C3P/nkk2rbtq3uuusu3XLLLcrLy9PKlSv16KOPas+ePU6FVFcpPadXrVql8PBwDRs2TD4+Plq7dq0mTJigLVu2aOHChfb+b775pp544gk1aNBAAwYMUGBgoHJzc5WZmamUlBQ99dRTVyVOAKgyAwBQZZKM4OBgwzAMY9SoUYbZbDYOHjxo3963b1/Dz8/POHXqlPHcc88ZkoyUlBT79pKSEqNVq1aGJGPRokUOc3/wwQeGJCM8PNw4f/68vb1ly5aGl5eXkZeX59C/sLDQCAgIMIKCgoxz587Z2+Pi4gxJxoEDB+xte/bsMTw9PY3mzZsbOTk5DvOkp6cbHh4exqBBg+xtM2fONCQZM2bMcMrBb7/9Zpw+ffqyuXr77bcNScasWbPsbXPmzDEkGb179za8vLyMU6dO2bfdeuutRo0aNYyioqJLHsuBAwcMSUZcXFyZ+127dq0hySn3F+//ySefvGz8l9pXnz59DEnGiy++6NC+ceNGw2w2G3Xq1DFOnjxpb7/zzjsNs9ls2Gw2e1vXrl2Njh07GnXr1jUeeeQRe/sPP/xgSDLi4+Ptbfn5+YbVajXq1q1r7Ny502Gf27dvN2rVqmV07NjR3rZs2TJDkjFx4kSnYyoqKjJOnDhhf12ar8TExCvKyeVcar6EhARDkjFgwACjsLDQYVthYaGRm5tb7ryln4U1a9a4JM5ff/3V8PPzMzw9PY1169Y5bb/4vM7Ozja8vLyM2rVrGz/++KNDvyeffNKQZIwePdqhPTo62pBkdO7c2SgoKLC3//TTT4anp6dhtVqNJk2aOJyPBQUFRt26dY3AwECHc7r0cyjJeOaZZxz288033xgWi8WwWq3G8ePH7e0vv/yyIcm45557HOY6evSoERoaakgyNm7cWOY+kpKSHPbx+eef2+e6WEpKiiHJGDx4sNPfhMTExDL/hpTuY9SoUUZxcbG9fefOnYbZbDZat27t0L+yn8/33nvPkGT85S9/cdr2yiuvGJKMmTNnOrTv27fPqW9RUZHRs2dPw2KxOP3tLH2PL1aak9//7SklyYiOjnZoK83V+PHjHXJSXFxsxMfHG5KMTz75xN5+2223GV5eXsbRo0ed5v/111/L3C8AXAu4fQMAXGz06NE6f/683nnnHUkXfmFcs2aNhg8frpo1a5Y5ZtOmTdq9e7fuuOMODR8+3GHbkCFDFBUVpT179mjDhg329ri4OJ09e1bvv/++Q//ly5eroKBAw4cPd/ol8vfeeOMNnTt3Tq+99prTLRW9evXSwIEDtXz5cp08edJhW40aNZzmqlWrVpntv1d6xcPvrwioX7++nn76aZ09e9Z+nHl5edq2bZuioqLk5eV12bmvRGRkpNMVJPHx8bJYLPr6668rPW9OTo5Wr16txo0b2xdiLNWtWzc9/PDDys/P10cffWRv79Wrl86fP6/169dLkk6ePKlvv/1WvXv3Vo8ePfTvf//b3resK0YWLFggm82m559/3um2oD/84Q8aPXq0tm7dql27djlsK+t98vLyUu3atSt59JV3/vx5zZ49WzVq1NCcOXPk7e3tsN3b21v16tWrtnhSU1N14sQJPfnkk/ZbTi4WEhJi/+9Fixbp7NmzGj9+vP2qn1IvvfSSateurYULF5Z5u0JycrLDFRTNmjVTVFSUbDabJk+e7HA+Wq1WDRgwQMeOHdOhQ4ec5vL393e4vUmSOnfurOHDh8tms+njjz+2t7/zzjsymUx69dVXHf4+BAUF2a9QKl3/5mKhoaH6+9//7tDWt29fNW7c2Om8ee2112SxWPTOO+84fdYmT56sunXr6t1333XaR82aNfXqq6/KbDbb29q0aaPIyEj9+OOP+u2335zGVNSgQYPk7++vd999V+fPn3fYlpqaKk9PTz388MMO7c2bN3eax8vLS+PGjVNxcbHD3zJXKSkp0euvv64GDRron//8p0NOzGaz/ud//kcmk8kpjxaLpcy1dgIDA10eIwC4CrdvAICLRUREqF27dnrnnXf097//XW+99ZZKSko0evTocsd8//33ki7c2lCWnj17asOGDdq6davuuusuSdJjjz2myZMnKzU1VePGjbP3rcitG1999ZUkaf369frmm2+ctufm5ur8+fPau3evOnXqpIEDB+q//uu/NG7cOK1atUp9+/ZVZGSk2rRpI5PJdNn9SRe+3DRr1kzr1q1TSUmJ/T7wmJgYRUdHy2KxKCMjQ3369NHatWtlGEa5eamMzp07O7V5enqqfv36KigoqPS8W7dulXThtoyyvhT07NlTixYt0tatW/XYY4/Z25KSkpSRkaGBAwdq/fr1Ki4uVq9evdSkSRN9+OGH+vHHH9W6dWt7geLiXJS+f9u2bSvz0Yh79+6VJP34449q06aNoqOjFRwcrOTkZH3//ffq16+fIiMjdeuttzp86alOu3fv1vHjxxUREVHlRTBdYfPmzZIurC1wOZc6bwMCAtSxY0d98cUX2r17tzp06OCwvazPYenxd+rUyWlbaZEiJydHoaGhDttuu+22MgtK3bt3V2pqqrZu3aq4uDidPHlS+/btU3BwsFMR5eLjKP0sX6y8z0ijRo3sn0Ppwi1C27ZtU2BgoGbMmOHUX7pQaPrxxx+d2sPCwuTn51fmPiSpoKBAvr6+Zc55pWrUqKGHHnpI8+bN06pVq9SvXz9J0nfffaedO3dq8ODBTl/gf/75Z02bNk0ZGRn6+eefdebMGYftZRWKqmrv3r3Kz89XWFiYXnzxxXKP5eI8Dh8+XH/5y1/Upk0bDR06VNHR0YqMjKzWoh4AVAZFCQC4CkaPHq2nn35an332mVJSUtSpU6dLLq5WuiDeLbfcUub20nabzWZvCwkJUa9evbRmzRr7F9fc3Fx9/vnnuvXWW9W+ffvLxlm6MOX06dMv2a/0F8rQ0FB9/fXXSkpK0ueff27/1b9Ro0Z65pln9PTTT192n9KFX/vnzZun77//Xp6envr111/Vq1cv1a5dW126dLH/8ngl60lUVFn390sXfmH8/S+nFVGZ97Br166qVauWw/F6eXkpKirKfm98RkaGwsLC9MUXX6hNmzZq0KCBfXzp+3fxAollKX3//Pz8tHnzZiUmJmrZsmVatWqVpAu/oj711FP6+9//Xu1PNCnNx7XyeNeKxFOZ97yUv7+/U1vplQuX2nbu3DmnbfXr1y9z/6WfldI4qxLvpc6bixdcLCgokGEY+vXXX/X888+XOaY8l9qHpCqdnxcbMWKE5s2bp9TUVHtRorSYW7puRan9+/fr9ttvV0FBge6880716dNH/v7+MpvN9nVSyroSpqpKz+2srKxL5vHiq0f+/Oc/KzAwULNnz9bMmTM1Y8YM+yKz06dPL7MQBgDXAm7fAICr4NFHH1WNGjU0duxYHTp0SGPGjLlk/9IvIb/88kuZ248cOeLQr1TpP6BL/0H97rvvqri42Okf1pfb7/Hjx2UYRrn/u/gy9tatW2vx4sXKy8vTt99+q+TkZJWUlOiPf/yj3n777Svab+kvsunp6U6Fh549e2rr1q3Kz89XRkaG/P397Qs1Xssq8x56enoqKipKO3fu1C+//KKMjAzdcccdqlmzplq2bKmQkBClp6fr66+/1smTJ51+kS+da9u2bZd8/y7+PISEhOjtt99Wbm6uduzYoZkzZ6pu3bqaMmWKpkyZ4tKcXInSL6JX49fmyqhIPJU9b13t6NGjZbaXxlW6/+qIt3Rsx44dL/mZNAyj0vuoqm7duiksLEzLli2TzWbTuXPn9P777yswMNBepCj16quvKi8vT2+//bbWrVunmTNn6oUXXlBSUpL69u17xfv08LjwT+6yHhd9qaLV4MGDL5nDAwcOOIx77LHHtHnzZuXl5WnFihUaNWqUvvjiC/Xt21e//vrrFccLANWJogQAXAVWq1WxsbHKyclRrVq1nO5R/r3SqyjKe5zd2rVrJcnpy/n9998vPz8/LVq0SCUlJUpNTZXFYtGwYcOuKM6uXbtKkr788ssr6n8xi8WiTp066W9/+5t9XYtPPvnkisb27NlTJpNJGRkZ+ve//61mzZrZrwzo1auXSkpKtGDBAmVlZal79+5XdGtBaR9X/ZpaUaXv4YYNG8r84lHee1hajHn//fe1Y8cOh6tCevbsqXXr1mnNmjUOfUtV5f0zmUxq27atJkyYYJ//4vevuvLZqlUrWa1WZWZmlvkI2upWmtPPPvvssn0vdd7abDb98MMP9kdhXk3ff/+907ovF8dVGmft2rXVvHlzHTp0yOkxw1L5n9GK8PX1Vdu2bbVz507l5+dXep7LqernMy4uToWFhVq8eLFWrFihY8eOadiwYU5XCpU+LemBBx5wmqN0LZgrERAQIElOj/mVLjwR6PdKz4vNmzeXeXXM5VitVvXr10/z5s3TiBEjlJ+fry+++KLC8wBAdaAoAQBXyYsvvqiPP/5Yq1atuuwCgpGRkQoPD9eGDRv04YcfOmz78MMP9eWXX6ply5aKiopy2FZ6f/ShQ4f0z3/+U9u2bVO/fv0UFBR0RTGOHz9enp6e+tOf/mRff+BiZ8+edfjC+91339kvAb9Y6S+15S3k+XtBQUFq27atNm7cqC+++MLhy3a3bt3k4+OjqVOnSip/nY3fCwgIkMlksj9atLqFhISod+/eys7OdrqXfsuWLXrvvfcUEBCgwYMHO2wrPb7k5GQZhuFUlDh+/Lhmz54tDw8Ph0cfStLIkSNltVr1/PPPl7lIZ0lJicMX5p07d5b5q3pZ71/po12vdj7NZrOeeuopnTlzRmPHjnW6FP7s2bPV+gtvXFyc/Pz89MYbb5T5JS4nJ8f+34888og8PT31+uuvOz3qd/LkyTpx4oQeeeQRp8U7Xe348eNOV7l8++23evfdd+Xv7+/wmYuPj5dhGHr22WcdvtAfO3bM/mjL+Pj4KsXz5z//WWfPnlV8fHyZVwEUFBTY1+OorKp+Ph977DF5eHhowYIFWrBggaSy1+EpLZb+vvC0atWqMhcELU/nzp3l4eGh9957T6dPn7a35+fnOy2MK10o+k6YMEFHjhzR008/7bSOhXThypaLF7EtXYPn93JzcyVd+d9nAKhurCkBAFdJ48aN1bhx4yvqazKZlJqaqt69e2vIkCG677771KpVK+3Zs0effPKJateurQULFtgvAb5YXFyc3nrrLSUkJNhfX6lWrVrpnXfeUXx8vNq2bau7775bLVu21Llz5/Tzzz/ryy+/VL169bR7925J0sKFCzV37lxFRUWpefPmCggI0E8//aTly5fL29tbEydOvOJ99+rVSzt27LD/dylvb29FRkZWeD0JX19fRURE6Msvv9Tw4cPVsmVLmc1mDRw48IrW13CFOXPmKDIyUs8++6xWr16tzp076+DBg0pLS5OHh4dSUlKcClQdO3ZUQECAcnNzVbt2bd1+++32baXHnpubq86dOzvdc1+3bl19+OGHGjx4sLp27apevXqpbdu2MplMOnjwoL766ivl5eWpsLBQkrRmzRo9++yzuuOOO9SyZUsFBQUpJydHS5culYeHh5599ln73OHh4QoODtYHH3wgT09PhYaGymQy6dFHH3VaaLGqEhMTtWXLFi1fvlwtW7ZU//79Vbt2bR08eFCrV6/W9OnTHb4wPvPMMzp27Jgk2Z/UMn36dC1atEjShScsDBo0qFKxBAYG6r333lNsbKx69Oihe+65R+3bt9eJEyeUmZmpgwcP2i+Zb9KkiWbMmKFx48bptttu00MPPaR69epp/fr1+uqrr9SqVStNmzatCpm5MnfddZfeeustbdmyRZGRkTpy5IgWL16skpISzZ0712HxyGeeeUafffaZli5dqg4dOqhfv346ffq00tLSlJubq7/+9a9Oxc+Kio+P13fffafZs2erefPm9qd05Ofn68CBA/riiy80cuRIzZkzp9L7qOrns1GjRurRo4cyMjJksVjUrl27Mtf9eeqpp5SSkqIHH3xQsbGxatiwoXbs2KHPP/9cDz30kBYvXnxF8d5yyy0aPny4Fi5cqFtvvVX33nuvTpw4oZUrV+quu+4qc3HRyZMna9u2bZozZ46WL1+unj17Kjg4WLm5ucrKytLGjRv10ksv2Z+8M3jwYPn6+qpr165q0qSJDMPQl19+qW+++UadOnVSTEzMFcUKANXuKj9yFABuCpKM4ODgK+r73HPPlfu8+t27dxuPPPKI0aBBA8NisRgNGjQwhg8fbuzevfuSc7Zo0cKQZNSpU8coKioqs09cXJwhyThw4IDTtszMTCMuLs5o3Lix4eXlZQQEBBht27Y1xowZY2RkZNj7bd682Rg7dqzRvn17IyAgwPDx8TGaN29ujBgxwti+ffsVHX+pZcuWGZIMk8lkHD161GHbyy+/bEgy6tevX6FjycrKMvr372/UqVPHMJlMDnleu3atIclITEwsc87Q0FAjNDT0imI/cOCAIcmIi4tz2paTk2OMHTvWaNy4seHp6WnUrVvXuO+++4yvv/663Pnuv/9+Q5LRr18/p20tW7Y0JBl//etfLxnPuHHjjBYtWhje3t5G7dq1jfDwcOORRx4xPv74Y3u/Xbt2GX/605+MTp06GYGBgYaXl5cRGhpqPPDAA8bGjRud5v3666+Nnj17Gn5+fvZ8rl279tLJKcfl8n/u3Dnj9ddfN7p06WLUqlXLqFmzptGiRQtj9OjRRlZWlkPf0NBQQ1K5/ytvHxWxY8cO49FHHzUaNmxoeHp6GkFBQcZdd91lzJ0716nvqlWrjN69extWq9Xw8vIymjdvbjz77LNGQUGBU9/o6GijvH9+XeocTUxMdMr/xZ/DXbt2GQMHDjSsVqtRo0YNo1u3bsbnn39e5n7OnDljvPTSS0bbtm0NHx8fw9fX14iMjDTee+89p76X+qxf7niWL19u3HvvvUa9evUMT09Po379+kaXLl2M5557zvjxxx8d+koyoqOjK5SXqn4+Fy5caP/M/Pd//3e5/TZu3Gj06NHDsFqt9lx9/PHH5X6my8tJYWGh8cwzzxjBwcGGp6en0bx5c+Pll182zp07V+7xl5SUGAsWLDB69uxpBAQEGJ6enkbDhg2NyMhI46WXXjJ+/vlne9833njDGDRokNG0aVOjRo0aRkBAgHHrrbca06ZNM06cOHHFeQGA6mYyDDeuNAQAAIBKyc7OVtOmTRUXF6f58+e7OxwAACqFNSUAAAAAAIBbUJQAAAAAAABuQVECAAAAAAC4BWtKAAAAAAAAt+BKCQAAAAAA4BYWdwfgSocPH3Z3CJcVGBhof7Y6qo58ug65dC3y6Vrk03XIpWuRT9cin65DLl2LfLoW+XSt6yGfDRs2LHcbV0oAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt7ihnr4BAAAAAMDVcv78eRUWFkqSTCaTm6O54OjRoyoqKnLb/g3DkNlslo+PT6XGV6kosWrVKi1btkw2m00hISEaMWKEWrduXW7/Xbt2KTU1VTk5OQoICNDAgQPVp08f+/aPP/5YX3/9tQ4fPiyLxaKwsDANGzZMjRs3rkqYAAAAAABUyfnz53XmzBnVqlXrmilISJLFYpHZbHZrDIWFhTp37pw8PT0rPLbSt29s2rRJ8+fP1+DBgzVt2jSFh4fr5ZdfLvf5qLm5uZo6darCw8M1bdo0DRo0SCkpKdq8ebO9z65du9SnTx+98MILSkxMlNls1gsvvKDffvutsmECAAAAAFBlhYWF11xB4lrh7e2ts2fPVmpspYsSn376qaKjoxUTE6OQkBDFx8crICBAq1evLrP/6tWrFRAQoPj4eIWEhCgmJkbR0dFavny5vc9zzz2nHj16qHHjxmrcuLEmTJigEydOaPfu3ZUNEwAAAAAAl6AgUbaq5KVSRYni4mLt379fHTp0cGhv37699uzZU+aYrKwstW/f3qGtQ4cO2r9/v4qLi8scczhSEC8AACAASURBVObMGRmGIV9f38qECQAAAACAS1CQuLTK5qdSa0qcOHFCJSUl8vf3d2i3Wq3avn17mWNsNpvatWvn0Obv76/z58/r5MmTCggIcBqTkpKiJk2aqGXLlmXOmZ6ervT0dElScnKyAgMDK3M41cpisVwXcV4vyKfrkEvXIp+uRT5dh1y6Fvl0LfLpOuTStcina12v+Tx69KgslmvzWRHXQlze3t6Vel/dH3k5UlNTtWfPHk2ZMkUeHmVf0BETE6OYmBj76/LWs7iWBAYGXhdxXi/Ip+uQS9cin65FPl2HXLoW+XQt8uk65NK1yKdrXa/5LCoqcvuCkmWxWCzl3n1QnYqKisp9Xxs2bFjuuEoVJfz8/OTh4aHjx487tNtsNlmt1jLHWK1W2Ww2h7bjx4/LbDardu3aDu3z58/Xpk2blJiYqPr161cmRAAAAAAArrrzowdW6/7M85ZVeExJSYkmTZqkFStWyGazKS0tTd26dbsK0VVcpdaUsFgsatasmTIzMx3at2/frvDw8DLHhIWFOd3akZmZqWbNmjlcapKSkqKNGzfqH//4h4KDgysTHgAAAAAA+P8yMjK0ZMkSzZ8/X1u3blXnzp0v2d9ms2nChAlq1aqVWrVqpQkTJjhdlOAqlX76Rv/+/bVu3TplZGQoJydHKSkpys/PV+/evSVJs2bN0qxZs+z9+/Tpo/z8fM2fP185OTnKyMjQunXrNGDAAHuft956S+vWrdMf//hH+fr6ymazyWazqbCwsAqHCAAAAADAzSs7O1tBQUHq0qWLgoKC5OXldcn+48eP144dO7Ro0SItWrRIO3bs0NNPP31VYqv0mhLdunXTyZMn9dFHH6mgoECNGjVSQkKC6tWrJ8l5fYegoCAlJCQoNTXV/njQkSNHqmvXrvY+pY8TnTJlisPY2NhYPfTQQ5UNFQAAAACAm9LEiROVlpYmSQoODlZISIg2b96suXPnauHChTp8+LDq1Kmj2NhYJSQkKCsrS2vXrtUnn3xiv6Ji2rRpGjx4sPbt26cWLVq4NL4qLXTZt29f9e3bt8xtSUlJTm1t2rTRtGnTyp1vyZIlVQkHAAAAAABcZMqUKQoJCdEHH3yglStXymw2Kzk5WQsWLFBiYqIiIiKUl5enHTt2SJK+++471apVy+EWjy5duqhmzZr67rvvrq2iBAAAAAAAuHb5+fnJ19dXZrNZQUFBOnXqlObNm6ekpCQNHTpUktS0aVN7ESI3N1d169aVyWSyz2EymRQYGKjc3FyXx1fpNSUAAAAAAMD1Ze/evSoqKlJUVJS7Q5FEUQIAAAAAAPx/QUFBysvLk2EY9jbDMHTs2DEFBQW5fH8UJQAAAAAAuEmEhYXJ29tbGzZsKHN7p06ddOrUKX377bf2tm+//VanT59Wp06dXB4Pa0oAAAAAAHCT8PX11ahRo5ScnCxvb29FRESooKBAmZmZiouLU1hYmHr06KFJkybZH1QxadIkxcTEuHyRS4miBAAAAAAAlWaet8zdIVRYQkKC/P39NWPGDB05ckSBgYGKjY21b581a5YmT56s4cOHS5L69OmjF1988arEYjIuvlHkOnf48GF3h3BZgYGBOnbsmLvDuGGQT9chl65FPl2LfLoOuXQt8ula5NN1yKVrkU/Xul7zefr0adWsWdPdYTixWCwqLi52dxiXzE/Dhg3LHceaEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALnr5xjQval3DZPrktplZDJDeGy+WTXFYM+XQdznXXIp+uxbnuWuTTdTjXXYvPpmuRT+DKcKUEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKFLgEAAAAAqKT73t1drftbOrxVhceUlJRo0qRJWrFihWw2m9LS0tStW7erEF3FUZQAAAAAAOAGlpGRoSVLligtLU2hoaGyWq2X7P/aa6/p3//+t3bu3KkzZ87o0KFDVy02bt8AAAAAAOAGlp2draCgIHXp0kVBQUHy8vK6ZP+zZ8/qnnvu0eOPP37VY+NKCQAAAAAAblATJ05UWlqaJCk4OFghISHavHmz5s6dq4ULF+rw4cOqU6eOYmNjlZCQIEl69tlnJUmffvrpVY+PogQAAAAAADeoKVOmKCQkRB988IFWrlwps9ms5ORkLViwQImJiYqIiFBeXp527NjhlvgoSgAAAAAAcIPy8/OTr6+vzGazgoKCdOrUKc2bN09JSUkaOnSoJKlp06bq3LmzW+JjTQkAAAAAAG4Se/fuVVFRkaKiotwdiiSKEgAAAAAAwE0oSgAAAAAAcJMICwuTt7e3NmzY4O5QJLGmBAAAAAAANw1fX1+NGjVKycnJ8vb2VkREhAoKCpSZmam4uDhJ0qFDh1RQUKCcnBxJsi+C2bRpU9WqVcul8VCUAAAAAABcNcsX237X4vh6wBBr9QVzFSwd3qra9mXLL5Ytv/h3rY6vrXUu/zU/ISFB/v7+mjFjho4cOaLAwEDFxsbat0+fPt3+GFFJ6tu3ryQpLS1N3bp1q/wBlIGiBAAAAAAAN7CxY8dq7Nix9tceHh4aP368xo8fX2b/GTNmaMaMGdUSG2tKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3sLg7AAAAAAAArlfLF9uqdX939vat8JiSkhJNmjRJK1askM1mU1pamrp163YVoqs4ihIAAAAAANzAMjIytGTJEqWlpSk0NFRWq7XcvgcPHtSMGTO0adMm5ebmKigoSAMHDtTEiRNVo0YNl8dGUQIAAAAAgBtYdna2goKC1KVLl8v23bdvn86fP6+pU6eqadOmysrK0t/+9jcVFBTolVdecXlsrCkBAAAAAMANauLEiUpKStKhQ4cUHBysiIgIGYahOXPmKDIyUk2bNlWnTp00depUSVKPHj00Y8YMde/eXaGhoYqJidGECRO0YsWKqxIfV0oAAAAAAHCDmjJlikJCQvTBBx9o5cqVMpvNSk5O1oIFC5SYmKiIiAjl5eVpx44d5c7x22+/XfKWj6qgKAEAAAAAwA3Kz89Pvr6+MpvNCgoK0qlTpzRv3jwlJSVp6NChkqSmTZuqc+fOZY7PycnRnDlzNGHChKsSH7dvAAAAAABwk9i7d6+KiooUFRV12b6//vqrhg8frrvuuktjxoy5KvFQlAAAAAAAAA5yc3P14IMPKjw8XDNnzpTJZLoq+6EoAQAAAADATSIsLEze3t7asGFDuX2OHj2q2NhYhYWFafbs2bJYrt7KD6wpAQAAAADATcLX11ejRo1ScnKyvL29FRERoYKCAmVmZiouLk6//PKLYmNj1aBBAyUlJSk/P98+tm7dujKbzS6Nh6IEAAAAAACVNGDI1XkqRVls+cUumSchIUH+/v6aMWOGjhw5osDAQMXGxkqS1q9frwMHDujAgQO6/fbbHcZt3rxZjRo1ckkMpShKAAAAAABwAxs7dqzGjh1rf+3h4aHx48dr/PjxTn2HDBmiIUOGVFtsrCkBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANzC4u4AAAAAAAC4Xs2cObNa9/fYI09VeExJSYkmTZqkFStWyGazKS0tTd26dbsK0VUcRQkAAAAAAG5gGRkZWrJkidLS0hQaGiqr1Vpu35KSEsXHx2vnzp3Ky8uTv7+/oqKi9F//9V+65ZZbXB4bt28AAAAAAHADy87OVlBQkLp06aKgoCB5eXldsn9kZKTmzJmjL774Qm+++ab+85//6PHHH78qsXGlBAAAAAAAN6iJEycqLS1NkhQcHKyQkBBt3rxZc+fO1cKFC3X48GHVqVNHsbGxSkhIkIeHh0aPHm0fHxISovHjx2vkyJEqLCyUj4+PS+OjKAEAAAAAwA1qypQpCgkJ0QcffKCVK1fKbDYrOTlZCxYsUGJioiIiIpSXl6cdO3aUOb6goEAfffSROnbs6PKChFTFosSqVau0bNky2Ww2hYSEaMSIEWrdunW5/Xft2qXU1FTl5OQoICBAAwcOVJ8+fRy2L1++XPv371dBQYGeeuopde/evSohAgAAAABw0/Lz85Ovr6/MZrOCgoJ06tQpzZs3T0lJSRo6dKgkqWnTpurcubPDuJdeekkpKSk6c+aMbrvtNi1YsOCqxFfpNSU2bdqk+fPna/DgwZo2bZrCw8P18ssv69ixY2X2z83N1dSpUxUeHq5p06Zp0KBBSklJ0ebNm+19CgsL1ahRI40cOfKy97gAAAAAAICK2bt3r4qKihQVFXXJfk8++aRWrVql999/X2azWRMmTJBhGC6Pp9JXSnz66aeKjo5WTEyMJCk+Pl4//PCDVq9erWHDhjn1X716tQICAhQfHy/pwn0p+/bt0/Lly9W1a1dJ0m233abbbrtNkvS///u/lQ0NAAAAAABUQZ06dVSnTh01b95cLVq0UJcuXfT1118rIiLCpfupVFGiuLhY+/fv14ABAxza27dvrz179pQ5JisrS+3bt3do69Chg9avX6/i4mJZLBUPJT09Xenp6ZKk5ORkBQYGVniO6maxWCoW577Ld7kejvtqcXU+yWUFj598lotz3bXIp+twrrsW+XQtznXX4bPpWuSzqmyX3Hq95OLo0aOV+t5a3cqK0cPDQyaTSRaLRa1bt5a3t7e++uortWzZ8orm9PC4cJPFpb67e3t7V+q9rFRGT5w4oZKSEvn7+zu0W61Wbd++vcwxNptN7dq1c2jz9/fX+fPndfLkSQUEBFQ4jpiYGPuVGpLKvXXkWhIYGFihOIOuoM/1cNxXi6vzSS4rdvzks3yc665FPl2Hc921yKdrca67Dp9N1yKfV9f1kouioiKZzWZ3h3FZxcXFTm0lJSUyDEPFxcXy8fHRqFGj9NJLL8lisSgiIkIFBQXKzMxUXFycvv32W+3YsUNdunSRv7+/srOzNX36dDVq1EidOnUqc37pQn7Key8bNmxYbrzXfpkHAAAAAIBr1NNPP11t+7Lll10QqKiEhAT5+/trxowZOnLkiAIDAxUbGytJ8vHx0aeffqrp06frzJkzCgoKUvfu3fXGG29cO0/f8PPzk4eHh44fP+7QbrPZZLVayxxjtVplszletnP8+HGZzWbVrl27MmEAAAAAAIDLGDt2rMaOHWt/7eHhofHjx2v8+PFOff/whz/oww8/rLbYKvX0DYvFombNmikzM9Ohffv27QoPDy9zTFhYmNOtHZmZmWrWrNl1cV8OAAAAAABwrUo/ErR///5at26dMjIylJOTo5SUFOXn56t3796SpFmzZmnWrFn2/n369FF+fr7mz5+vnJwcZWRkaN26dQ6LZRYWFio7O1vZ2dkyDEPHjh1Tdnb2dXOPEQAAAAAAuHKVvkShW7duOnnypD766CMVFBSoUaNGSkhIUL169SQ5L1YSFBSkhIQEpaam2h8POnLkSPvjQCXpp59+0vPPP29/vWTJEi1ZskTR0dEaN25cZUMFAAAAAADXoCrdN9G3b1/17du3zG1JSUlObW3atNG0adPKna9t27ZasmRJVUICAAAAAADXiUrfvgEAAAAAAFAVFCUAAAAAAIBbUJQAAAAAAABuQVECAAAAAAC4BUUJAAAAAADgFlV6+gYAAAAAADezoH0J1bcvSXvrvFDhcSUlJZo0aZJWrFghm82mtLQ0devWzfUBVgJXSgAAAAAAcAPLyMjQkiVLNH/+fG3dulWdO3e+onGFhYWKiYlRcHCwtm3bdlVioygBAAAAAMANLDs7W0FBQerSpYuCgoLk5eV1ReNeeOEF3XLLLVc1NooSAAAAAADcoCZOnKikpCQdOnRIwcHBioiIkGEYmjNnjiIjI9W0aVN16tRJU6dOdRi3atUqbdq0Sf/4xz+uanysKQEAAAAAwA1qypQpCgkJ0QcffKCVK1fKbDYrOTlZCxYsUGJioiIiIpSXl6cdO3bYxxw+fFgJCQlauHChfHx8rmp8FCUAAAAAALhB+fn5ydfXV2azWUFBQTp16pTmzZunpKQkDR06VJLUtGlT+zoT58+f14QJEzRmzBi1bdtWBw8evKrxcfsGAAAAAAA3ib1796qoqEhRUVFlbp85c6Y8PT31xBNPVEs8XCkBAAAAAAAkSRs3btSWLVsUGhrq0D5gwAANHDhQs2bNcun+KEoAAAAAAHCTCAsLk7e3tzZs2KBmzZo5bX/11Vd1+vRp++ujR49q2LBhev3119WlSxeXx0NRAgAAAACAm4Svr69GjRql5ORkeXt7KyIiQgUFBcrMzFRcXJwaN27s0L9WrVqSpCZNmqhhw4Yuj4eiBAAAAAAAlZTbYurlO7mILb/YJfMkJCTI399fM2bM0JEjRxQYGKjY2FiXzF1RFCUAAAAAALiBjR07VmPHjrW/9vDw0Pjx4zV+/PjLjm3UqJEOHTp01WLj6RsAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAABchmEY7g7hmlbZ/FCUAAAAAADgMsxmswoLCylOlKG4uFgeHpUrL/BIUAAAAAAALsPHx0fnzp3T6dOnJUkmk6naYzj6y9nL9vHy8aqGSP6PYRjy8PCQj49PpcZTlAAAAAAA4Ap4enrK09PTbfvfu/3yRYnwNjWrIRLX4fYNAAAAAADgFhQlAAAAAACAW1CUAAAAAAAAbkFRAgAAAAAAuAVFCQAAAAAA4BY8fQMAAAAAcM0K2pdw2T65LaZWQyQ3hsvls7pzyZUSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANzCUpXBq1at0rJly2Sz2RQSEqIRI0aodevW5fbftWuXUlNTlZOTo4CAAA0cOFB9+vSp0pwAAAAAAOD6VOkrJTZt2qT58+dr8ODBmjZtmsLDw/Xyyy/r2LFjZfbPzc3V1KlTFR4ermnTpmnQoEFKSUnR5s2bKz0nAAAAAAC4flW6KPHpp58qOjpaMTExCgkJUXx8vAICArR69eoy+69evVoBAQGKj49XSEiIYmJiFB0dreXLl1d6TgAAAAAAcP2q1O0bxcXF2r9/vwYMGODQ3r59e+3Zs6fMMVlZWWrfvr1DW4cOHbR+/XoVFxdLUoXnTE9PV3p6uiQpOTlZgYGBlTmcch0d3O2S2+/v/spl53jc0uB3LTaHVweOLrjk+ClT5l12H6496qunqvl0zqXk6nzeKLmUKpPPiuVSunnyybleMdfDuS5dH/nkXHeta+Fcl8jnxfjbeUF1nOsS/066GH87r5wrzvWNf4xyeG2xWOzfASXpH//4xyXH3yjnulT1fP4+l5Lr81nduaxUUeLEiRMqKSmRv7+/Q7vVatX27dvLHGOz2dSuXTuHNn9/f50/f14nT56UYRgVnjMmJkYxMTH21zfibR434jG5E/l0LfLpOuTStcina5FP1yKfrkMuXYt8uhb5/D+/z0VgYGCF8kMu/09Zubge8tmwYcNyt/H0DQAAAAAA4BaVulLCz89PHh4eOn78uEO7zWaT1Wotc4zVapXN5njZ0/Hjx2U2m1W7dm1JqvCcAAAAAADg+lWpKyUsFouaNWumzMxMh/bt27crPDy8zDFhYWFOt2FkZmaqWbNmslgslZoTAAAAAABcvyp9+0b//v21bt06ZWRkKCcnRykpKcrPz1fv3r0lSbNmzdKsWbPs/fv06aP8/HzNnz9fOTk5ysjI0Lp16xwWtrzcnAAAAAAA4MZRqds3JKlbt246efKkPvroIxUUFKhRo0ZKSEhQvXr1JDkvnhEUFKSEhASlpqbaHw86cuRIde3a9YrnBAAAAAAAN45KFyUkqW/fvurbt2+Z25KSkpza2rRpo2nTplV6TgAAAAAAcOPg6RsAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANzC4u4AbnZPP/20u0O4oZBP1yGXrkU+XYt8ug65dC3y6Vrk07XIp+uQS8B1uFICAAAAAAC4BUUJAAAAAADgFhQlAAAAAACAW1CUAAAAAAAAbkFRAgAAAAAAuAVFCQAAAAAA4BY8EvQSzPOWXbrDu7urJ5AbBPl0ncvmUiKfFcBn07XIp+twrrsWn03XIp+uw7nuWuQTuL5wpQQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt7BUZpBhGEpLS1NGRoZ+++03hYWFadSoUWrUqNElx23evFmLFy/W0aNHVb9+fT388MO6/fbb7du3bNmi9PR07d+/XydPnlRiYqLatm1bmRABAAAAAMA1rlJXSixdulSffvqpRo4cqalTp8rPz08vvviizpw5U+6YvXv3asaMGbrzzjv1yiuv6M4779Srr76qrKwse5+ioiK1bNlScXFxlQkLAAAAAABcRypclDAMQytXrtSgQYPUtWtXNW7cWOPHj9eZM2e0YcOGcsetWLFCbdu21f3336+QkBDdf//9atu2rVasWGHvc9ddd+nBBx/UrbfeWrmjAQAAAAAA140K376Rm5srm82m9u3b29u8vLzUunVr7dmzR7179y5z3N69e3XPPfc4tHXo0EGff/55RUOwS09PV3p6uiQpOTlZgYGBlZ7LXa7HmK9l5NN1yKVrkU/XIp+uQy5di3y6Fvl0LfLpOjdTLo+6YI7f58tisVQohzdSvquaz7Jycb3ns8JFCZvNJkmyWq0O7f7+/iooKLjkOH9/f6cxpfNVRkxMjGJiYuyvjx07Vum53OV6jPlaRj5dh1y6Fvl0LfLpOuTStcina5FP1yKfrkMuK+b3+QoMDKxQDsn3/ykrF9dDPhs2bFjutssWJb788ku9+eab9tcJCQmuiQoAAAAAANzULluU6Ny5s8LCwuyvz507J+nClQ8XX/Zx/PhxpyshLma1WnX8+HGHtuPHjztdcQEAAAAAAG4Ol13oskaNGmrQoIH9fyEhIbJarcrMzLT3OXv2rHbv3q3w8PBy52nZsqXDGEnKzMxUy5YtqxA+AAAAAAC4XlX46Rsmk0n9+vXT0qVLtWXLFv3888+aPXu2fHx8FBUVZe83ZcoUvffee/bX/fr1044dO/TJJ5/o0KFD+vjjj7Vz507de++99j6//fabsrOzdfDgQUnSL7/8ouzs7CqtOwEAAAAAAK5NFV7oUpLuu+8+nT17Vm+//bZOnTqlFi1a6LnnntP/Y+/uw60q67yBf4+8CKRwkCMgHMGUFxUFcxDPGJgapkEqD9VxUEkNtVQ0pqaufBzNCPOlUtIBk9RpnkpBbRQTHscANc2w0VQeUMSXDEWFUM9BEEVenj+82OPhnePGBfb5XBfXxVp73Wvf+7fvvfba332vfVq2bFnaZuHChWnXrl1puWfPnhk1alQmTpyYSZMmpWPHjhk1alSDS0MeffTRjB8/vrR8/fXXJ0m+9KUvpba2tjFdBQAAALZTjQolKioqUltbu8mgYNy4ceutq6mpSU1NzUbbHHHEETniiCMa0yUAAABgB7PVl28AAAAAlINQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAoRNOiOwAAAMDfr/PPP7/oLnys7Gj1NFMCAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAFS62QAAAIABJREFUKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKETTxjRas2ZNbrvttkyfPj1Lly5N9+7dM2LEiOy5556bbDdz5sxMmjQpCxcuTIcOHTJs2LD069cvSbJy5cpMnDgxTzzxRBYuXJiWLVumV69eOfnkk1NVVdWYbgIAAADbsUbNlJg8eXLuvvvunH766bnsssvSunXrjBkzJsuXL99om3nz5mXs2LEZMGBArrzyygwYMCBXXXVVnn322STJihUr8pe//CVDhw7NFVdcke985zt5/fXXc+mll2bVqlWNe3QAAADAdmurQ4k1a9Zk6tSpGTJkSGpqatKlS5eMHDkyy5cvz0MPPbTRdlOmTEmvXr0ydOjQVFdXZ+jQoenVq1emTJmSJGnVqlUuuuiiHHbYYenUqVO6deuWs846KwsWLMiCBQsa/wgBAACA7dJWX76xaNGi1NXVpXfv3qV1zZs3z3777ZdnnnkmRx999AbbzZs3L5///OcbrOvTp0/uueeejd7X22+/nST5xCc+scHbp02blmnTpiVJLr/88u3yMo/Tz+3WYLlp06ZZuXJlQb3Zsa1by0Q9Pwxjs7zUs3y81svL2Cwv9Swv9SwftSwv9fwfC8uwj3U/pzVt2nS7/Oz2Ufiw9dxQ3Xb0em51KFFXV5ckqaysbLC+TZs2efPNNzfZrk2bNuu1Wbu/da1cuTK//OUv8w//8A9p167dBrcZOHBgBg4cWFpevHjxFj2Gj9K6faqqqtou+7kj2FDd1LPxjM3yUs/y8VovL2OzvNSzvNSzfNSyvNSzvNSzfHbU86ROnTpt9LbNhhIPPvhgJkyYUFq+4IILytOrTVi1alWuueaaLFu2LN/5zne2+f0BAAAAH73NhhJ9+/ZN9+7dS8vvvfdekvdnPnxwikh9ff16MyE+qLKyMvX19Q3W1dfXrzfjYtWqVfnpT3+a+fPn55JLLsmuu+66ZY8EAAAA2KFs9ocuW7ZsmY4dO5b+VVdXp7KyMrNmzSpts2LFisydOzc9e/bc6H569OjRoE2SzJo1Kz169Cgtr1y5MldffXX++te/5nvf+956gQUAAADw8bHVf32joqIigwYNyuTJk/PII49k/vz5GT9+fFq0aJH+/fuXths9enRuvvnm0vKgQYMye/bs3HnnnVmwYEHuuOOOzJkzJ4MHD07y/gyJtX8i9Bvf+EYqKipSV1eXurq6rFixogwPFQAAANiebPUPXSbJCSeckBUrVuTGG2/MsmXL0q1bt1x44YVp2bJlaZuFCxc2+IHKnj17ZtSoUZk4cWImTZqUjh07ZtSoUaVLQ15//fU8+uijSZLvfve7De7vnHPOyRFHHNGYrgIAAADbqUaFEhUVFamtrU1tbe1Gtxk3btx662pqalJTU7PB7du3b59bb721Md0BAAAAdkBbffkGAAAAQDkIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCNG1MozVr1uS2227L9OnTs3Tp0nTv3j0jRozInnvuucl2M2fOzKRJk7Jw4cJ06NAhw4YNS79+/Uq3T5w4MTNnzszrr7+epk2b5pOf/GROPPHE9OzZszHdBAAAALZjjZopMXny5Nx99905/fTTc9lll6V169YZM2ZMli9fvtE28+bNy9ixYzNgwIBceeWVGTBgQK666qo8++yzpW06deqUESNG5Mc//nFGjx6d9u3b54c//GHq6uoa000AAABgO7bVocSaNWsyderUDBkyJDU1NenSpUtGjhyZ5cuX56GHHtpouylTpqRXr14ZOnRoqqurM3To0PTq1StTpkwpbXP44YfnwAMPTIcOHbLnnnvmK1/5SpYvX54XX3yxUQ8OAAAA2H5tdSixaNGi1NXVpXfv3qV1zZs3z3777Zdnnnlmo+3mzZuXPn36NFjXp0+fzJs3b4Pbr1y5MtOmTUvLli2z1157bW03AQAAgO3cVv+mxNpLKSorKxusb9OmTd58881NtmvTps16bda9NOOxxx7L2LFjs2LFilRWVuaiiy5a777WmjZtWqZNm5Ykufzyy1NVVbW1D2ebW7dPTZs23S77uSPYUN3Us/GMzfJSz/LxWi8vY7O81LO81LN81LK81PN/LCzDPtTzf3zYen4cz5M2G0o8+OCDmTBhQmn5ggsu2KYd6tWrV370ox9lyZIlmT59eq6++uqMGTMmbdu2XW/bgQMHZuDAgaXlxYsXb9O+Nca6faqqqtou+7kj2FDd1LPxjM3yUs/y8VovL2OzvNSzvNSzfNSyvNSzvNSzfHbU86ROnTpt9LbNhhJ9+/ZN9+7dS8vvvfdekvdnPnwwjamvr19vJsQHVVZWpr6+vsG6+vr69WZBtGjRIh07dkzHjh3To0ePnH/++Zk+fXq+9KUvba6rAAAAwA5ks78p0bJly1JI0LFjx1RXV6eysjKzZs0qbbNixYrMnTt3k3+6s0ePHg3aJMmsWbPSo0ePTd7/mjVrsnLlys11EwAAANjBbPUPXVZUVGTQoEGZPHlyHnnkkcyfPz/jx49PixYt0r9//9J2o0ePzs0331xaHjRoUGbPnp0777wzCxYsyB133JE5c+Zk8ODBSZK33347EydOzLPPPpvFixfnhRdeyPjx4/P666/nH//xH8vwUAEAAIDtyVb/0GWSnHDCCVmxYkVuvPHGLFu2LN26dcuFF16Yli1blrZZuHBh2rVrV1ru2bNnRo0alYkTJ2bSpEnp2LFjRo0aVbo0pEmTJnnppZdy33335a233squu+6affbZJ9///vfTtWvXD/kwAQAAgO1No0KJioqK1NbWpra2dqPbjBs3br11NTU1qamp2eD2O++8c7797W83pjsAAADADmirL98AAAAAKAehBAAAAFCIRl2+AQAAwMdfk5/ftekNfj33o+nIx4R6rs9MCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEE0b02jNmjW57bbbMn369CxdujTdu3fPiBEjsueee26y3cyZMzNp0qQsXLgwHTp0yLBhw9KvX78NbjthwoRMmzYtp5xySo4//vjGdBMAAADYjjVqpsTkyZNz99135/TTT89ll12W1q1bZ8yYMVm+fPlG28ybNy9jx47NgAEDcuWVV2bAgAG56qqr8uyzz6637cyZM/Pcc8+lbdu2jekeAAAAsAPY6lBizZo1mTp1aoYMGZKampp06dIlI0eOzPLly/PQQw9ttN2UKVPSq1evDB06NNXV1Rk6dGh69eqVKVOmNNjub3/7W/793/89559/fpo2bdREDgAAAGAHsNWhxKJFi1JXV5fevXuX1jVv3jz77bdfnnnmmY22mzdvXvr06dNgXZ8+fTJv3rzS8qpVq/LTn/40X/ziF1NdXb21XQMAAAB2IFs9FaGuri5JUllZ2WB9mzZt8uabb26yXZs2bdZrs3Z/SXLrrbdm1113zec+97kt6su0adMybdq0JMnll1+eqqqqLWr3UVq3T02bNt0u+7kj2FDd1LPxjM3yUs/y8VovL2OzvNSzvNSzfNSyvNSzvNSzfD6O50mbDSUefPDBTJgwobR8wQUXbJOOzJkzJ/fff39+9KMfbXGbgQMHZuDAgaXlxYsXb4uufSjr9qmqqmq77OeOYEN1U8/GMzbLSz3Lx2u9vIzN8lLP8lLP8lHL8lLP8lLP8tlRz5M6deq00ds2G0r07ds33bt3Ly2/9957Sd6f+fDBNKa+vn69mRAfVFlZmfr6+gbr6uvrSzMu5syZk7q6upx11lml21evXp1f//rXmTp1an72s59trqsAAADADmSzoUTLli3TsmXL0vKaNWtSWVmZWbNmpVu3bkmSFStWZO7cuTnllFM2up8ePXpk1qxZDf6856xZs9KjR48kyTHHHJOampoGbS699NJ8+tOfbjAbAgAAAPh42OofuqyoqMigQYMyefLkPPLII5k/f37Gjx+fFi1apH///qXtRo8enZtvvrm0PGjQoMyePTt33nlnFixYkDvuuCNz5szJ4MGDk7z/+xJdunRp8K9p06aprKzc5FQPAAAAYMfUqL+5ecIJJ2TFihW58cYbs2zZsnTr1i0XXnhhgxkVCxcuTLt27UrLPXv2zKhRozJx4sRMmjQpHTt2zKhRoxpcGgIAAAD8/WhUKFFRUZHa2trU1tZudJtx48att66mpma9SzQ2ZUP7AAAAAD4etvryDQAAAIByEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhWjamEZr1qzJbbfdlunTp2fp0qXp3r17RowYkT333HOT7WbOnJlJkyZl4cKF6dChQ4YNG5Z+/fqVbh83blweeOCBBm26d++eSy+9tDHdBAAAALZjjQolJk+enLvvvjvnnHNOOnXqlNtvvz1jxozJ2LFj07Jlyw22mTdvXsaOHZva2tr069cvf/rTn3LVVVflBz/4Qbp3717a7sADD8x55533Px1s2qguAgAAANu5rb58Y82aNZk6dWqGDBmSmpqadOnSJSNHjszy5cvz0EMPbbTdlClT0qtXrwwdOjTV1dUZOnRoevXqlSlTpjTYrlmzZqmsrCz922WXXbb+UQEAAADbva0OJRYtWpS6urr07t27tK558+bZb7/98swzz2y03bx589KnT58G6/r06ZN58+Y1WDd37tycccYZ+cY3vpGf/exnqa+v39ouAgAAADuArb42oq6uLklSWVnZYH2bNm3y5ptvbrJdmzZt1muzdn9JctBBB+XQQw9N+/bts2jRokyaNCmjR4/O5ZdfnmbNmq23z2nTpmXatGlJkssvvzxVVVVb+3C2uXX71LRp0+2ynzuCDdVNPRvP2Cwv9Swfr/XyMjbLSz3LSz3LRy3LSz3LSz3L5+N4nrTZUOLBBx/MhAkTSssXXHDBNuvMpz/96dL/u3Tpkr333jvnnntu/vznP+fQQw9db/uBAwdm4MCBpeXFixdvs7411rp9qqqq2i77uSPYUN3Us/GMzfJSz/LxWi8vY7O81LO81LN81LK81LO81LN8dtTzpE6dOm30ts2GEn379m3wQ5TvvfdekvdnPnwwjamvr19vJsQHVVZWrncpRn19/XozLj5ot912y2677ZZXX311c90EAAAAdjCb/U2Jli1bpmPHjqV/1dXVqayszKxZs0rbrFixInPnzk3Pnj03up8ePXo0aJMks2bNSo8ePTbaZsmSJXnjjTfStm3bLXksAAAAwA5kq3/osqKiIoMGDcrkyZPzyCOPZP78+Rk/fnxatGiR/v37l7YbPXp0br755tLyoEGDMnv27Nx5551ZsGBB7rjjjsyZMyeDBw9Okrzzzjv5P//n/2TevHlZtGhR5syZkyuuuCJt2rRJv379yvBQAQAAgO3JVv/QZZKccMIJWbFiRW688cYsW7Ys3bp1y4UXXpiWLVuWtlm4cGHatWtXWu7Zs2dGjRqViRMnZtKkSenYsWNGjRpVujRkp512yksvvZTf//73WbZsWdq2bZtevXrln//5nxvsFwAAAPh4aFQoUVFRkdra2tTW1m50m3Hjxq23rqamJjU1NRvcvnnz5rnwwgsb0x0AAABgB7TVl28AAAAAlINQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAAChE08Y0WrNmTW677bZMnz49S5cuTffu3TNixIjsueeem2w3c+bMTJo0KQsXLkyHDh0ybNiw9OvXr8E2r7zySm6++ebMnj07K1euTOfOnXPeeeelurq6MV0FAAAAtlONmikxefLk3H333Tn99NNz2WWXpXXr1hkzZkyWL1++0Tbz5s3L2LFjM2DAgFx55ZUZMGBArrrqqjz77LOlbRYtWpSLLroo7du3z8UXX5yf/OQnOfHEE9OiRYvGdBMAAADYjm11KLFmzZpMnTo1Q4YMSU1NTbp06ZKRI0dm+fLleeihhzbabsqUKenVq1eGDh2a6urqDB06NL169cqUKVNK29xyyy3p06dPvvKVr2TvvfdOhw4dcvDBB6eqqqpxjw4AAADYbm11KLFo0aLU1dWld+/epXXNmzfPfvvtl2eeeWaj7ebNm5c+ffo0WNenT5/MmzcvSbJ69eo89thjqa6uzqWXXpoRI0bkggsuyMMPP7y1XQQAAAB2AFv9mxJ1dXVJksrKygbr27RpkzfffHOT7dq0abNem7X7W7JkSd55553ccccdOfHEE3PyySdn9uzZueaaa9KiRYscfPDB6+1z2rRpmTZtWpLk8ssv3y5nVKzbp6ZNm26X/dwRbKhu6tl4xmZ5qWf5eK2Xl7FZXupZXupZPmpZXupZXupZPh/H86TNhhIPPvhgJkyYUFq+4IILtklHVq9enSTp27dvvvCFLyRJ9tprrzz//PO55557NhhKDBw4MAMHDiwtL168eJv07cNYt09VVVXbZT93BBuqm3o2nrFZXupZPl7r5WVslpd6lpd6lo9alpd6lpd6ls+Oep7UqVOnjd622VCib9++6d69e2n5vffeS/L+zIcPpjH19fXrzYT4oMrKytTX1zdYV19fX5px0bp16zRp0mS9v7LRuXNnl3AAAADAx9Bmf1OiZcuW6dixY+lfdXV1KisrM2vWrNI2K1asyNy5c9OzZ8+N7qdHjx4N2iTJrFmz0qNHjyTvTznZZ5998sorrzTY5tVXX83uu+++VQ8KAAAA2P5t9Q9dVlRUZNCgQZk8eXIeeeSRzJ8/P+PHj0+LFi3Sv3//0najR4/OzTffXFoeNGhQZs+enTvvvDMLFizIHXfckTlz5mTw4MGlbY4//vg8/PDDmTZtWl577bVMmzYtDz/8cI455pgP+TABAACA7c1W/9BlkpxwwglZsWJFbrzxxixbtizdunXLhRdemJYtW5a2WbhwYdq1a1da7tmzZ0aNGpWJEydm0qRJ6dixY0aNGtXg0pB+/frla1/7Wu644478+7//e/bYY4+ce+65G/w9CQAAAGDH1qhQoqKiIrW1tamtrd3oNuPGjVtvXU1NTWpqaja57yOOOCJHHHFEY7oFAAAA7EC2+vINAAAAgHIQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIVoWnQHAAAA2DFNPnnforvwsfL3WE8zJQAAAIBCCCUAAACAQlSsWbNmTdGdKJdXXnml6C5sVlVVVRYvXlx0Nz421LN81LK81LO81LN81LK81LO81LN81LK81LO81LO8doR6durUaaO3mSkBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABSiYs2aNWuK7gQAAADw98dMiY/Yd7/73aK78LGinuWjluWlnuWlnuWjluWlnuWlnuWjluWlnuWlnuW1o9dTKAEAAAAUQigBAAAAFKLJJZdccknRnfh7s/feexfdhY8V9SwftSwv9Swv9SwftSwv9Swv9SwftSwv9Swv9SyvHbmefugSAAAAKITLNwAAAIBCCCXgY27+/Pm58847s3LlyqK7AgAA0IBQokC1tbWZOXNm2fd7//33Z/jw4WXfb2MsWrQotbW1ef755z/UNn/vbr311nzrW9/a6nZvv/12fvKTn6RDhw5p2rTph+7HnDlzUltbmyVLlnyobT4uttVrmG3H8Qb+/owbNy6XX3550d0APkJ33XVXzj333KK7URZbcu5y7rnn5q677voIe1VeH/5TynbqiiuuyLvvvpuLL754vdtefvnlfPOb38yFF16YPn36FNC7902YMCGf+MQnyr7fww47LJ/61KfKvt911dbWbvL2z3zmM/nyl7+82f1UVVVlwoQJ2XXXXcvVtUYZN25c3nrrrY/s7/yOGzcuDzzwQI488sicffbZDW771a9+lbvuuisHH3xwvvvd7+b444/P5z//+a2+j/Hjx+eYY47JP/7jP5ar29uVtTVMkiZNmqRdu3bp169famtr06JFi4J7Vz6XXHJJ9txzz4wYMaLormyRrRnbjXX//ffnxhtvzC9/+csP293t1kdRx+3dx6UGt956ax555JH85Cc/abB+yZIlOeOMM/K9730vvXr1Ktv9zZkzJ9///vdzww03pHXr1lvV9oUXXsgFF1yQHj165Ac/+EHZ+rQlPky/N+X000/P9vITah/VmF6wYEFuu+22zJkzJ8uWLUvbtm1z6KGHZujQodlll10+1L63R9vD+cCG3qu31Zj+MOrq6nLHHXfkz3/+c15//fXsuuuu6dq1a4499tgcfPDBRXfv79qHOf5edtll2XnnnbdRz7a9j20ocdRRR+XHP/5xFi1alPbt2ze4bcaMGdl9991z4IEHFtS791VWVm7y9pUrVzbq2+3mzZunefPmje3WFpswYULp/4899liuv/76BuuaN2+epUuXbnY/O+2002Zr8XHVrl27/PGPf8zpp59eetO2lcVAAAAgAElEQVRctWpVfv/736eqqqq0XYsWLRr1pvov//IvW7RdY8fa9uDAAw/Meeedl5UrV2bu3Ln52c9+lnfffTdnnnlm0V37u7alY7sx/p4uRdqWddxRqMFHa8aMGTnmmGPywAMP5OWXX051dXXRXfrQWrVqVXQXGtjWY/q5557L6NGjs//+++fb3/52dtttt/z1r3/Nr371qzz++OMZM2bMNvlSrGjOBzZv0aJFueiii9KyZcsMGzYse+21V1avXp3Zs2fn5z//ea677rpG7XdHPo/cnnyY4+/mQq/t/Tnafnv2IR188MFp06ZN7r///gbf6K9cuTIPPvhgjjnmmCTJddddl9mzZ6euri7t2rXLZz/72Rx33HHZaaf3r2xZ++35vvvumylTpmTFihX53Oc+l2HDhuX222/Pvffem4qKigwePDhDhgwp3U9tbW2++tWv5vHHH8+cOXPSunXr/NM//VMOP/zwBtt885vfTE1NTRYtWpSRI0fm/PPPz/Tp0zNv3rwMHz48xx57bO67777cddddWbRoUaqqqnL00Udn0KBBpT6ua91vENd+QzN06NBMnDgx9fX1OeCAA/L1r3/9Q6W2HwwS1r65rRsurA0l/va3v+Xmm2/OM888k9133z2nn356evfunSSlx37ZZZdln332KaXKF110UW655ZbMnz8/1dXVOeussxr8qZsZM2bktttuy1tvvZUDDjggn/rUp3LjjTfm1ltvTZIsXrw4N910U55++um89957qaqqype//OV8+tOfXu+x3HrrraWEfe14WfvN1fz58/Mf//EfmTt3bpo3b56+ffvm9NNPL8tJTteuXfPmm2/mj3/8Y4488sgkyZ///Oc0a9Ys++23X6l+637LNn/+/PziF7/I888/n9WrV6djx4459dRTc8ABByR5fzbQL3/5yzz99NNp3rx5DjjggJx22mml5+eD4/qee+7JypUrc8MNN+T3v/99/u///b9ZsGBBmjdvnv333z+nnXZadttttwb9fvbZZzNx4sS88sorqa6uzte+9rVN/hmiZ555JjfffHOef/75fOITn0jfvn1z8sknl2r41FNP5de//nXmz5+fnXbaKZ06dcrZZ5+dLl26bLaGzZo1Kz2u/v37Z/bs2fnv//7vnHbaafn1r3+dP/zhD3n77bez1157Zfjw4dl3331LbRcsWJBf/epXeeqpp7J69ep06dIlX/va19KlS5c899xzmThxYv7yl79k5cqV6dKlS4YPH54ePXo0uP+lS5fmqquuyuOPP542bdqktra2wet8c+Nn7XPRu3fvTJ48OStWrMghhxySESNGZOedd864cePy1FNP5amnnsp//dd/JUn+7d/+LVVVVbn++us3efza3DjZlrZ0bK9evTr/+Z//menTp6e+vj577LFH/umf/imHHHJIkmz02HjTTTcl+Z/X65e+9KXU1tZu8RjeUZS7jt/85jfzu9/9boPH4mTTx46nnnoqP/jBD3Ldddc1ONbfcssteeyxx/LjH/+40BpsTR3Wvt+s9cH34yS5/fbbM2PGjNTV1eUTn/hE+vTpk5EjRyZJ1qxZk7vuuivTpk3LG2+8kY4dO+aEE05o8LpvrC3p3+aey0WLFuX73/9+kuSMM85I8v7MxS2ZxrxixYo89NBDGT16dN59993MmDEjX/nKVxr0bVNjaPXq1Y0+LrVv336j/X7iiSfyn//5n3nppZeSJN26dcupp57a4IR9U8/ZujMht2R/29KWjunGvObWrFmT6667LnvssUe+853vlOpeVVWVT37ykzn//PNzyy235Iwzzsi9996bqVOnZuzYsUmSWbNmZcyYMTnppJNK57TXXHNNmjdvnq9//eul88vvfOc7+cUvfpFFixalW7duOfvss9f7ArAIjT0f2NLzzk2dy2zsvXpjY3pbHkc25cYbb0ySXH755Q2+7Kqurs6AAQOSvD/LdsmSJQ1m66xevTrnnntuBg8enC984Qu55JJL0rlz5+y888554IEH0r59+1x22WW5++67c//992fhwoVp1apVPvWpT2X48OGlzwlbOoZ+97vf5a677srixYtTVVWVE044IQMHDtzkY5s8eXLuvvvuvPPOOzn00EM3OCa39jPVR2lTx991rV69OjfddFMef/zx/Ou//mv22GOPnHvuuTnmmGNy/PHHJ/mfz6KzZ8/Ok08+maOPPjqnnHLKZo/RRfnYhhJNmjTJZz7zmdx///350pe+VCr0Y489liVLluSII47I6tWrs9tuu+Wf//mf07p16zz33HOlywiOOuqo0r6efvrp7Lbbbrnkkkvyl7/8Jddee21efPHFfPKTn8zo0aMze/bs3HDDDendu3eDg9ett96aYcOG5dRTT83MmTMzbty4dO7cucGJxrpuueWWDB8+PGeffXaaNGmSadOm5dZbb81Xv/rV7L333pk/f36uv/76NG3aNMcee+wW12PRokV5+OGH8y//8i959913M3bs2EycODFnnXVWI6q79SZOnJhTTjklZ5xxRn7zm99k7NixGT9+/Ca//b/55ptz8sknp23btvnFL36Ra6+9NldddVUqKioyb968XH/99Rk2bFj69euXp556KrfcckuD9jfccEPee++9fO9730urVq3yyiuvbPS+jj/++CxYsCBLly7NeeedlyTZZZdd8s477+TSSy/NPvvsk8suuyxLly7N9ddfn/Hjx2/xLITNOfLII3PfffeVTkzW/n/hwoUbbfPTn/40Xbt2zQ9/+MM0adIk8+fPL82OefPNN/O9730vRx55ZIYPH55Vq1bllltuyZVXXpkxY8aUXgtPPfVUWrVqlf/9v/93ab8rV67Ml7/85XTu3DlvvfVWfv3rX+enP/1p6U11rV/+8pelD3q33357Lr/88lx77bUbnDY2f/78jBkzJrW1tfn617+epUuX5he/+EWuu+66fOtb38qqVavyox/9KEceeWTOO++8rFq1Kn/5y18afXBs3rx5Vq1alV/96lf54x//WHqju/vuu3PppZfmmmuuSdu2bfPGG2/k4osvTs+ePXPRRRelVatWee6557J69eokyTvvvJPDDz88p512WioqKnLPPffksssuyzXXXNPgUqPbb789J510Uk466aTMmDEj1113Xfbff/9UVVVt8fh5+umnU1lZmYsuuiivv/56rr766uyxxx75X//rf+X000/Pq6++mk6dOuWkk05K8n4aviXHr02Nk4/CloztqVOn5re//W3OPPPM7L333nnwwQfz4x//OFdccUX22muv0nYfPDbutNNOWb16dW655ZZce+21SVI6lmzpGN6RlLOOmzoWb+7Ysf/++6dDhw554IEHcsIJJyR5/8To97//fY477rjCa7A1ddiUmTNn5re//W2+8Y1vpEuXLqmvr8+zzz5bun3ixImZOXNmRowYkU6dOpXej3bZZZePdOrzxp7LqqqqfOtb38pPfvKTXHXVVdlll122+HU/c+bM7L777unSpUsOP/zwXH311TnppJMafLu2qTH0YY5Lm+r3O++8k0GDBqVr165ZsWJFfvOb3+SKK67I1VdfnaZNm272OVvX5vb3UdiSMd2Y19yLL76Yl156Keeff/5676O77bZb+vfvnz/84Q8ZMWJEevXqlRtuuCF1dXWl4HHXXXfNnDlzSqHE008/nWHDhpX2sXLlytx55505++yz06xZs4wbNy4///nPc+GFF5a1PuWwpecDa23qvHNz5zIbe6/e2Jgu4jiydOnSPPHEEznxxBM3eP69NjgYOHBgLr744rz55pul+syaNSt1dXUNQpMHH3wwAwcOzOjRo0uXR1VUVOS0005L+/btS18O3nTTTaVz62TzY+hPf/pTbrrpppx66qnp3bt3nnzyydx4442prKxM3759N/jYHn744UycODFf/epX06tXr8ycOTOTJ09ucKlSuT5TbStbcvxN3q/fv/3bv+Wll17KD37wg01+6XL77bdn2LBhGT58eCoqKrb4s28Rio+FtqGjjjoqixcvzv/7f/+vtG7GjBnp06dPqqqq0rRp05x44onp1q1b2rdvn8MOOyxHH310/vCHPzTYT6tWrXLGGWekc+fO6d+/fz75yU+mrq4uJ510Ujp16pTPfe5z2X333TN79uwG7fr165ejjz46nTp1ytChQ3PAAQdkypQpm+zzsccem5qamrRv3z7t2rXLb37zm5xyyimldX379s2QIUNKKeyWWptwdu3aNT169MjAgQMb1GVbGzx4cPr27Zs99tgjJ510UpYuXZoXX3xxk21OPPHEHHDAAencuXO++MUvZsGCBXnjjTeSvH/i2bt37wwZMiSdOnXKwIED069fvwbtFy9enH333Td77bVX2rdvn4MOOigHHXTQBu+rRYsWad68eSllr6ysTNOmTfPQQw/lnXfeyXnnnZcuXbpk//33z1lnnZU//elPee2118pSm/79++f555/Pq6++mrq6ujzxxBM54ogjNtlm8eLF6d27dzp37pyOHTumX79+pW/w77333nTt2jWnnHJKqqur07Vr14wcOTLPPfdcXnjhhdI+mjVrVpqNsHZGwlFHHZWDDz44HTp0SLdu3XLGGWfk6aefzuuvv97g/r/4xS/moIMOSpcuXXLOOeeU0t0Nueuuu3LYYYfluOOOyx577JHu3bvnzDPPzCOPPJL6+vosX748y5YtS9++fdOxY8fS66wx31g999xz+cMf/pBevXrl3nvvzcknn5yDDz649I1HZWVl6bXzX//1X9l5553zzW9+M926dUunTp1y+OGHlz64HHDAATn88MNTXV2dzp0756tf/WqaNWuWxx9/vMF9Hn744Tn88MPTsWPHnHjiiWnSpEmeeuqpJNni8dOqVaucddZZqa6uTp8+fVJTU1M6nrRq1SpNmzbNzjvvXBqbO+200xYdvzY1Tj4KWzK2f/vb3+a4445L//7906lTp5x44onZb7/91vuxpg8eG6uqqkozTdbWZO0J1paO4R1JOeu4qWPxlhw7jjrqqNx///2l/T355JOpr68vfcO2rWzpcXJL67ApixcvTmVlZXr37p2qqqrss88+pRPWd955J3fffXe+/vWv56CDDkr79u3Tv3//fPazn93s+/LLL7+c4cOHN/j3YX6EbWPP5U477VQ6EW/dunUqKyu3eGbfjBkzSs/l/vvvn5133jmPPvroFt1vkg91XNpUv2tqalJTU5M99tgjXbt2zTnnnJNFixblueeeK+1zY8/Zhmxufx+FLR3TW/uae/XVV5MknTt33uDt1dXVWbZsWZYsWZLOnTunsrKy9H4zZ86cHHfccZk7d25WrVqV1157La+//nqD3ztZtWpVRowYkW7duqVr16457rjjMmfOnO3mNzvW2przgbU2dd65uXOZjb1Xb2hMf5jjyIfx2muvZc2aNZs9v+rRo0c6d+5cmkGcvB+a9e3bt8EM6/bt2+crX/lKOnfuXNrn4MGDc8ABB6R9+/bZf//9c8opp+SPf/xj6QufZPNj6Le//W0GDBiQY489Np06dcrnP//59O/fP5MnT95on6dOnZrPfOYzDT53devWrcE25fpMta1syfH33XffzRVXXJG//e1v+f73v7/ZWaCHHXZYPvvZz6ZDhw5p3779Fn/2LcLHdqZEkuyxxx7Zf//9c99996VPnz5544038uSTT2bUqFGlbe69997MmDEjf/vb37JixYqsWrUqu+++e4P9VFdXN0ib27Rps961eG3atFnvrw2se/LfvXv39T7QrOuDsyiWLFmS119/PRMmTMjPf/7z0vrVq1dv9cH/gyfxSdK2bduP9K8jdO3atcF9J0l9ff0Wt1n7oquvr0+7du3yyiuv5B/+4R8abN+9e/dMnz69tDxo0KD8/Oc/zxNPPJEDDzww/fr12+QlBhuyYMGCdO3aNS1btiyt69mzZyoqKvLyyy+nY8eOW7W/Ddlll13Sr1+/3HfffWnVqlV69eq12WtKBw8enOuvvz4PPPBADjzwwBx66KGlE5AXXnghTz/99Ab/Astrr71WOkh36dIlzZo1a3D7Cy+8kNtvvz0vvvhili5dWhpnixcvTrt27UrbfXBst2jRIl26dMnLL7+8wb6+8MILee211/Lwww+vd9vChQvTo0ePHHHEEbn00ktzwAEH5MADD0xNTc0WX1f7xBNPZPjw4Vm9enVWrlyZQw45JMcee2xmzpyZnj17lrbbaaed0r1791I/X3zxxey7774b/Vasvr4+kyZNypw5c1JXV5fVq1dnxYoVWbx4cYPtPniJSZMmTdK6devSa2tLx8+6x5jddttti06ON3f82tQ4+Shsbmy//fbbefPNNxs8T0my7777rnes3NQMsw/a0jG8IylnHTd1LN6SY8cRRxyRiRMn5plnnknPnj1z33335ZBDDtnmP1S8JcfJranDptTU1GTq1KkZOXJk+vTpk4MOOih9+/ZNs2bN8vLLL+e9997LD3/4wwZtNnTusK6OHTvmggsuaLBu6dKlDWarbY3GvK9uymuvvZa5c+fm/PPPT/L+N579+/fPjBkzSpe1bMn9bovj0muvvZZJkyblueeey5IlS0rnQWuPx5t6zhqzv4/Clr73b+vX3P7775+nnnoqhxxySJ5//vl861vfyu9+97s8//zzeemll9KhQ4cGx85mzZqlU6dOpeW2bdtm5cqVWbZsWeE/oNnY84G1NnXeublzmTZt2mxxPz/MceTD2JrPDmsDkiFDhmTp0qV59NFH15shvKFz6tmzZ+eOO+7IggUL8vbbb5eei7q6ulJNNzeGXn755dIMorX23Xff9T6gf9CCBQvW+6a/e/fupS+AyvmZalvY0uPvtddem8rKynzve9/bot+a29BztCWffYvwsQ4lkvcT5uuvvz5Lly7N/fffn1122aU09efhhx/Of/zHf5SuE2/VqlXuueee/Pd//3eDfTRp0qTBckVFxQbXfTAFbKwPTn9fu78zzzxzvZOsrbXuB69y9XdLfbBeFRUVSTZ/cFy3xlvS5oOOOuqo9OnTJ48//nhmzZqVf/3Xf82QIUM2+1dDinDkkUdm3LhxadGiRU488cTNbl9bW5sBAwbk8ccfz5NPPpnbbrstZ555Zo466qisWbMmn/rUpzZ4HdoH3zTXvdRi7aUGBx54YEaOHJk2bdrkrbfeysUXX/yhflhwzZo1Oeqoo/KFL3xhvdvWvkGdc845GTRoUJ544ok8+uijueWWW/Ltb397ozNbPmi//fbL1772tTRp0iRt27ZN06ZN89e//rXR/V1r3Lhxqa+vz6mnnprdd989zZo1y/9n787jqqj+x4+/WAQF2ZFdQAI03FBJzVRKTM2iTcNyt1JzqyzN+vrxI6RpLtmiWFimuaJW5lKpHxEVsDTNnUWvimxeUBFFERC4vz94MD+vgIAC96rv5+Ph46Fzz5w5czwzZ+bMWT799NNyeVEb19a9lPXq3L/uVk7qS03LdmWqM6N0XZVhfVBb+Xi3e3F17h2WlpYEBAQQHR2Ni4sLBw8eZMqUKfecnpqojTwoa/y7/fq6s2zY29vz1VdfceLECY4dO8aKFSv4+eef+eyzz5T9pkyZUu4FsqLr+HbGxsblGrLv/DhQnfRVdLzq1qt3ExUVRUlJCWPHjlW23d6oV53j1tV9ac6cOdja2jJy5EhsbW0xMjLigw8+UPLmbv9nFT24VxVffalOma7pNefs7AyUvvg2a9as3O9paWmYm5srX7z9/Pz4/fffSUpKwsnJCWtra/z8/Dhx4gRpaWnlVoW5c0hIWRmoz2fKytzv88Dd6uLqPMtU1/3cR+6Hs7Oz8lHkzt7Fd+revTurV68mMTGRc+fOYWlpWW7Fwjvr5YsXLzJ79myCgoIYMGAAjRs35ty5c3z99dda19a9lqGycPeiNt+p6kJ177/t2rVj7969JCUlVWsFyTvvf9V999WFh75RonPnzvz444/s3buX6OhounfvrrxEJCYm4u3trdXF727j+Gvq9OnTWpXs6dOna/SV0traGhsbGzIzMwkMDKy1dD0MXFxcyq3VW9GXZTs7O3r27EnPnj357bff+PPPPyttlDA2Ni53Q3R1dSU6OpqbN28qX7uTkpKq1f2tJlq3bo2xsTG5ubnKhGxVcXZ2xtnZWekRsmvXLnr06EGzZs3466+/lCFK1ZWRkUFubi4DBw5UJgfav39/hWFPnz6No6MjUPoimJqaWunkTM2aNatWrxJPT088PT15+eWXmTVrFnv27KlWo4SpqWm5uB0dHTE2NlYesqC0Qjp9+rQy0amnpycxMTGVzkacmJjIiBEjlLGdOTk5XLlypcr03K62yk9FZbO696/Kykl9uVvZNjMzw8bGhqSkJK3VkBITE6vMn4rypCZl+EFTV/l4u+reO4KCgliwYAEODg5YW1vX20pWVd0nq5MPZS9iOTk5yu8VDSU0MTGhffv2tG/fnpdffplRo0aRlJSEr68vDRo04OLFi3UyYWx101eVsv+/6r4oFhcXs2fPHgYOHFhuPPuiRYvYvXt3tSbgu9/7UkXpzs3NJT09nbfeekvJ87Nnz1JcXKwVZ2X/Z3c+uFc3vvpQ3bq/Jtecp6cnrq6ubN26laeeekrrBTA7O5vY2Fiefvpp5QWvbF6J2NhY/Pz8lG0xMTFkZGRozSeh7+71eaA6qvMsU1G9VFGZdnNzq9P7SGUaN25M27Zt2b59O3379i33wnrjxg2lJ3hZT55du3aRnJxMYGBglXN9nTlzhqKiIoYPH66E/ffff2ucTjc3NxITE7WeVaqqz1xdXSt87yqjz+9UNbn/BgUF4eXlxbx58/joo4+0Jqqujrp+970fD/WcElBaSXXt2pUNGzaQmZmpVVidnZ05d+4chw8f5sKFC/z888/KWPDacODAAXbu3MmFCxfYuHEjJ06coG/fvjWKIyQkRJlNNiMjg5SUFPbs2cPGjRtrLZ0Por59+3L06FE2b97MhQsX2LVrFwcOHNAKs2zZMo4cOUJmZibJyckcPXr0rje0Jk2akJqaSkZGBteuXaOoqIhu3bphamrKokWLSElJIT4+niVLltCxY8daGbpRxsDAgPnz57No0aJKu5uWKSws5IcffuDkyZNkZWVx+vRprZt17969ycvL46uvvuL06dNkZmZy7NgxIiIiuHnzZqXx2tvb06BBA7Zt20ZmZib//vsv69atqzDsL7/8wrFjx0hNTeXbb7/F2NiYrl27Vhj2pZdeUibSOXfuHGq1mkOHDinLx2ZlZbF69WqSkpK4ePEiJ06c4Pz58/fV6NOwYUN69erF6tWr+ffff0lLS+P7778nJydHWXmnd+/e5Ofns2DBAlQqFWq1mtjYWOUFwNnZmZiYGNLS0lCpVHz99dc1ngCttspPkyZNUKlUZGVlKV2Nq7p/VVVO6ktVZfvFF19ky5YtxMbGkpGRwbp160hISKhy4sQmTZpw69Ytjh07xrVr1ygoKKhRGX7Q1FU+3q669442bdrQuHFjfv75Z55++ul6m7G7OvfJqvLBxMQEHx8fNm3aRGpqKklJScpKVWV2795NVFQUKSkpZGVlsXv3boyMjHB2dqZRo0YEBwezcuVKdu3ahVqtJjk5mR07drBz5877PsfqpK86mjRpgoGBAf/++y/Xrl0jPz//ruH//fdfcnNzCQoKUuYZKvvTpUsXoqOjq3Xc+70vVZRuc3NzLCwsiIqKQq1WEx8fz/fff6/1Rflu/2d3qk589aW6dX9NrjkDAwPGjBlDRkYGc+fO5dSpU1y6dIl///2XGTNm0KRJE15//XUlfNm8EjExMcoLctmQjjvnk3gQVed5oDqqepaBiuvqisp0Xd9H7uatt95Co9Hw8ccf89dff5GRkUF6ejo7duwoNzwjKCiI2NhYzp8/X244RUWcnZ3RaDT8/vvvZGVlERsbW+VcehUJDg4mJiaGbdu2ceHCBf78809iY2OVVSUq0rdvX/bs2aP13nXnx0p9faeq6f23Z8+eDBs2jHnz5nHs2LEaHauu333vx0PfUwJKu/Hv2LGD5s2baz2QP/vssyQnJ/PNN9+g0Wjo1KkTwcHB1a58q/Laa6+xf/9+li1bhqWlJWPGjCk36UpVgoKCMDU1ZcuWLaxduxYTExPc3Nz0YpZYXfL19WX06NFs2LCBdevW0bp1a1566SUiIyOVMBqNhh9//JHLly/TsGFDWrduXenSOlB6kcfHx/Pxxx+Tn5+vLAk6depUli9fzieffKK1pGNtu33egbsxNDTkxo0bLF68mCtXrmBhYUH79u2VceC2trbMmDGDNWvWMGvWLAoLC7G3t6dt27Z3feixtLRk3LhxrF27lu3bt+Pu7s7QoUPLjXkEGDRoECtWrCAjI4OmTZsyZcqUSse2eXh4EBYWRmRkJKGhoZSUlODg4KB0HTQxMeHChQssWLCA3NxcrKys6NatmzLT+L0aNGgQULrs740bN2jWrBlTp05VxkDb2toSFhbGqlWrCAsLw8DAAHd3d2VFmjFjxrBkyRKmTJmCra0tr732Wo3nYTE1Na2V8hMcHEx4eDgffPABhYWFLFq0qMr7V1XlpD7drWw/99xz3Lx5k9WrV5OTk4OLiwsffvhhlSslNG/enGeffZavv/6a3NxcZUnQ6pbhB1Fd5OPtqnvvMDAw4JlnnmHDhg1VTspb26q6T1YnH8aMGUNERASffPIJjo6OvP3220yfPl353czMjE2bNrFy5UqKi4txc3Nj0qRJSu+bAQMGYGVlxZYtW/jhhx9o1KgRnp6e933Pqm76qqPsnhUZGUlERATdu3e/66Sau3btomXLlhXOU/Dkk0+yZs2aaj383u99qbJ0T5w4kWXLlvHhhx/i5OTEkCFDlCWyoer/s9sZGhpWGV99qk7dX9NrztfXl9mzZ/Pzzz8zd+5cbty4gUfpTeMAACAASURBVK2tLR07dqRfv37l5n7w8/Pjr7/+UnpKODg4YGtri6Gh4QM7F8/tqnoeqI6qnmWg4rrawcGhwjJd1/eRyjg6OjJnzhw2btzI6tWryc7OxsLCAg8PD0aPHq0VtmXLltjZ2WFvb6/0jr0bDw8Phg8fzqZNm4iMjKR58+YMGTJEWXK2ujp27MiIESPYsmULP/30E/b29rz11luVrrwBpRM6ZmZmEhkZSUFBAQEBATz//PNak3Xq6zvVvdx/n332WTQaDfPmzWPy5MnV7jFR1+++98NAow+zezyE7lzzXNS95cuXc/z4cZ09WAghxKPi+++/R61WM23aNF0nRYhHglxzor4VFhYyevRo3nzzzTpfYUmIh374hnh4bd68meTkZNRqNTt27OB///tftbqXCSGEuDd5eXmcOnWKvXv38vzzz+s6OUI89OSaE/WtpKSEq1ev8ssvv2BiYsKTTz6p6ySJR8AjMXxDPJzOnDnDli1byMvLw8HBgYEDB9Z4zg4hhBDVN3fuXFQqFT169Cg3IZcQovbJNSfq26VLlxg/fjx2dnaMHTu2xvNpCXEvZPiGEEIIIYQQQgghdEKGbwghhBBCCCGEEEInpFFCB8LDw/n8888r/X337t06mSH/YRIaGsrSpUt1nQzxgLh+/TojR45ErVbrOinlLFiwgC1btug6GXrl5MmThISE1Hg1FFG5quqlh4U+X+srV67kxx9/1HUyakTyUzcelev1fuhz2ZR6XUj5LO+RHSQUHh6utUyMhYUFPj4+DBkyBFdXVx2m7NFwe/4bGRlhbm5O06ZN6dSpEz179rzv8WuTJk3SyZrjD4Lw8HByc3P5+OOPdZ0UvbFx40batWuHk5MTAMuWLSMpKYnU1FSsra0JDw8vt8++ffvYuHEjFy5cwNLSkj59+pRbQ3vbtm1s376drKws7O3tefXVVwkMDFR+3717N4sXLy4X96pVqzAxMQGgf//+TJ8+naCgIMzMzGrztGtFReXp0KFDfPnll7zwwgu8/vrrOkzd3YWGhtK0aVPeeustXSel2kJCQu76e2Bg4F2XfqzMiBEjeBRGc+rqWg8NDa1wLXg3NzcWLFgAwEsvvcSECRN4/vnnq7X8nj6Q/Lx/dz4P2dnZ0bFjR0JCQipdarsurtdx48bRu3fvcv8XDypdlU0onZw0MjKS/fv3k5ubi52dHW+88QZdunQB9K9el3ei+ldX5TM2NpZNmzZx4cIFGjVqROvWrRk6dCjW1tZKmD/++IMdO3Zw8eJFLCwsCAgIYPDgwcr9Rlfl85FtlABo3bo1EyZMACA7O5tVq1Yxf/58vvzyywrDFxUVyWQvtags/0tKSrh27RonTpxgw4YNxMTEMG3atEor4+q4cw1uISpTUFDArl27mDJlirJNo9EQGBhISkpKubWhAQ4fPsw333zDiBEj8Pf3Jz09nYiICExMTJT1rnfs2MHq1asZPXo0Pj4+qFQqIiIiMDc311pr29TUlIULF2rFX9YgAeDu7o6joyN79+7V+Vra1bF3716+++47Bg8eLBPP1oElS5Yofz906BARERFa224vO1D9eksfHozrmi6v9UmTJlFUVKTEe+vWLSZNmqQ1q72lpSVt2rRhx44dD0RvScnP2lP2PFRUVERiYiLfffcdBQUFjBw5UitccXExhoaGj8T1ej90WTaLioqYOXMmjRs3ZuLEidja2pKdna11H9bHer2m70RVKSurBgYGtZnMOo+7PtRV+UxMTGThwoUMGTKEjh07kpOTw9KlS/nmm2/473//C5Q2WqxatYp33nmHFi1akJWVxbfffsutW7cYM2YMoLvy+Ui/YTdo0EBpObK2tub5559nzpw5FBYWkpOTw/jx43n33XeJiori1KlTDBkyhF69evHrr78SFRXF1atXcXZ25vXXX+eJJ55Q4k1JSeGnn34iMTERExMTAgICGDFiRKWVSHJyMrNmzeKZZ57hjTfe0PotKyuLCRMmMGvWLB577DFl+86dO1m7di0REREkJSURFhbGtGnTWLt2LSkpKbi5uTFq1Ci8vLzqIOdqx+35b2tri6enJ23atGHKlCls3rxZ+SJ4/fp1li9fzqFDhygsLKRFixYMHz6cpk2bVhr3nV9Ax40bR48ePbh8+TJxcXE0atSIvn37PjRfBGpTWloaK1euJCEhARMTE1q1asXw4cOV/6uUlBSWL1/OmTNnKCkpwcnJiWHDhtGqVSsdp/zeHD58GIDmzZsr2958802gdNnZiiqHvXv30qFDB3r37g2Ao6MjL7/8Mps2baJ3794YGBiwd+9egoKC6Nq1qxLmzJkzbNq0SatRAtBqwa5IQEAAcXFxevPwUpnff/+d1atX884779C9e3cA9u/fz/r167lw4QJWVlY8++yzvPLKK8rDxLhx43jmmWfIzMxk//79mJubM2TIENq2bcv333/PoUOHsLGx4a233qJt27Zaxzt9+jSRkZFkZGTg5ubG6NGjlXtebm4uS5cuJTExkdzcXBwdHQkODlaWDQ4PDyc+Pp74+Hi2b98OwKJFi3BwcKiv7Lont5cVc3NzrW1ZWVmMGjWqXL311FNP3TUvoHyPl9DQUNzc3DAzMyMqKgoDAwO6d+/O4MGDMTR8MEd+6vJav7OhPCYmhoKCgnLLWAcEBLB27Vq9f4kGyc/adPvzUNeuXTlx4gT//PMPVlZW7N+/n+DgYH755ReysrL46aefWLp0qXK97ty5k3Xr1hEREaF1bX799dfk5+czZcoU1Go1K1as4PTp0+Tn5+Pi4kJISAgdOnQASq/3ixcvsmrVKlatWgXA+vXrAUhKSmLNmjWcOXNGefkeNGiQXjeM6LJs7t69m2vXrvHpp58qDREV1Sv6Vq/f7Z3IxMSE7OxsVqxYwdGjRwHw9fVl+PDhODs7A6XlpaKymp2dTUREBCqVCnt7e4YNG8aXX37JW2+9xdNPPw1wz3EnJiby66+/kpqaCoC3tzfDhg3Dzc1NOa+yBpYjR45QWFiIs7Ozzp9Z66p8njp1Cjs7O1544QWgtNz16dNHaxhbUlISPj4+yjOag4MDgYGB7N+/X+t4uiifD+aTRR24efMm+/btw93dXetL09q1a+nduzdffvklTzzxBH/88Qdbtmxh0KBBzJ8/n44dOzJ//nySk5MByM/P57PPPsPU1JTZs2czefJkTp06VWEXbYCEhATCwsJ48cUXyzVIQGlhadOmDdHR0Vrbo6Oj6datm1bL65o1axg4cCBz5szBwsKChQsXPnDdcd3d3fH399e6OBYvXoxKpWLy5MnMnj0bExMTZs2aRWFhYY3i/v3333F3d2fOnDm89NJLrFq1ilOnTtX2KTzQrly5wvTp02natCmzZs1i2rRp5OfnM3fuXEpKSoDSBx1ra2tmzZrFvHnzeO2118p9nX2QJCQk4OXlVaMW91u3btGgQQOtbSYmJly+fJmLFy8qYe7MFxMTE1QqldYXvsLCQsaOHcs777zD559/zrlz58odz9vbG5VKVeMyX58iIyNZu3YtkyZNUiq7s2fPsmDBAjp16sT8+fMZOHAgGzduZNu2bVr7/v7773h7ezNnzhyefPJJwsPD+eabb2jXrh3z5s3j8ccfZ+HCheXOf+XKlQwaNIjZs2fj6OjI559/TkFBAVCa/15eXnz88ccsWLCAvn37smTJEo4fPw6Udn/29fXl6aefZsmSJSxZsgR7e/t6yKm6d2e9VVVeVCYmJgYjIyNmzJjBm2++yR9//MG+ffvq6Sxqn66v9dtFRUXh7+9frsx5e3uTnZ2tl+OM7yT5WXdMTEwoLi4GShsbY2NjmThxIvPmzSuXf507dyYvL0/rRSY/P5+DBw/SrVs35d/+/v5MmzaNefPmKffk9PR0oLTniZ2dHf3791fuh1D6EWLmzJkEBAQwb948Jk2aRHJyMt9++219ZMM902XZ/Oeff2jevDk//vgjI0eOZOLEiaxfv75c2dXnev3Od6KCggLCwsJo0KABoaGhzJw5ExsbG2bMmKHUuVC+rBobGzN//nyMjIz47LPPGDduHD///LNWXtxr3A0aNCA/P5++ffsya9YsQkNDadSoEXPmzFHiz8/PVxrcJk+ezPz58+nfv3/9ZWQl6qp8tmjRgitXrnDw4EE0Gg3Xrl1j3759tGvXTtmnRYsWJCcnK+8/ly5d4uDBg1phQDfl85FulDhy5AhDhgxhyJAhDBs2jPj4eN59912tMH369KFz5844ODhgZ2fHli1bCA4OpmvXrri4uDBgwAAef/xxNm/eDJR2i8nPz2fChAm4u7vj5+fHqFGjOHDgQLlK8dChQ3z++ecMHz5cadWqSFBQEHFxcUrBSEtL4/Tp0/To0UMr3IABA2jVqhWurq7069eP9PR0srOzayOr6pWbmxuZmZkAXLhwgYMHDzJq1Cj8/Pxwd3dnwoQJ5OXlERMTU6N427RpQ58+fXBycuK5557DycmpyofyR82OHTvw8PBg8ODBuLm54eHhwfjx41GpVJw9exYovYG1adMGV1dXnJyc6NixI76+vjpO+b27ePEiNjY2NdrH39+fgwcPcvToUUpKSsjIyGDr1q0A5OTkANC2bVuio6NRqVRoNBrOnDlDVFQUxcXF5ObmAuDi4sKYMWP46KOPeO+992jQoAHTpk3jwoULWsezsbGhuLhYb6/nY8eO8euvv/LBBx/Qvn17ZfvWrVvx8/MjJCQEFxcXunXrRnBwMJs2bdLav23btvTu3RtnZ2dCQkK4desWjo6OBAYG4uTkRL9+/bh27ZryNaRMv3798Pf3x93dnbFjx1JYWEhsbCxQ2vvqxRdfxNPTE0dHR3r27EmnTp2Ii4sDSocrGBsbY2pqirW1NdbW1g9sD4A73VlvVZUXlXFzc2PAgAG4uLjQpUsXWrZsyYkTJ+rpLGqfLq/122VkZBAfH09QUFC538rSV/aQqc8kP+uGSqUiLi5O+ZJbVFTE+PHj8fLywt3dvdx8WY0bN6Zdu3Zaz0QHDhzA0NBQ+Xrv6elJr169cHd3x8nJiVdffRUvLy/+/vtvJQ5DQ0MaNmyo3A+h9Kttly5dCA4OxtnZGR8fH0aOHMn+/fu5evVqfWTHPdFl2czMzOTvv/+mqKiITz75hAEDBvC///2PNWvWaB1P3+r1u70TxcXFodFoGDt2LB4eHri6ujJq1Cjy8/M5dOiQEsedZfXEiRNkZGQwfvx4PD098fX1ZdiwYUqD2/3EbWRkROfOnencuTPOzs54eHgwduxYsrKyUKlUQOk7WU5ODpMnT+bxxx/HycmJTp066bxnb12VT19fX95//30WLlzIwIEDefvtt9FoNIwfP16J56mnnuKNN95g+vTpvPHGG4wdOxZ3d3cGDRqkdTxdlM9HevjG448/zujRo4HSIQI7duzgs88+47PPPlPC3D5kIi8vjytXrmh1t4HSVqeyrjjp6el4eHjQqFEj5ffmzZtjYGBAWlqaMqHJ2bNnmT9/Pu+++67WGMiKBAQEsHTpUg4cOEDXrl2Jjo7G29sbd3d3rXAeHh7K321tbQG4evUqdnZ21c4TfaDRaJTWw/T0dAwMDLRees3MzHB3dyctLa1G8d6eP1B6welzpaoLZ8+eJSEhocKurmq1Gm9vb55//nkiIiLYs2cPrVu3plOnTg/0REgVffmoSlBQEGq1mrlz51JcXKwMB9qwYYNSdvv3709OTg7Tpk1Do9FgZWVFYGAgmzdvVsL4+vpqle3mzZszefJk/vzzT6UrH/z/eQL08YsKQNOmTcnLy2PDhg00b95cGVaQnp5ervW9RYsW/Pzzz+Tl5Sndf2+/Nhs2bIipqanW/a3sAfnO6/X2vGvYsKHWfaGkpITffvuNffv2kZ2dza1btygqKqJly5a1eOb66fZ6C+49Lx62e6Yur/XbRUVFYWNjo9WAV0bfr/XbSX7WnrIXwpKSEoqKinjiiSd488032b59O7a2tlUO8evWrRvh4eEUFBRgampKbGwsnTp1Us4/Pz+fn3/+mUOHDpGTk0NRURG3bt0q9xx5p7Nnz6JWqyvsIZWZmYmVldW9n3Qd0mXZ1Gg0WFpa8s4772BoaIiXlxfXr1/np59+YsiQIUo4fSubd3snOnv2LFlZWQwdOlRrn8LCQuUjIlCurGZkZGBjY6O8k0Bp/XT7dXyvcUPpc+m6detQqVRcu3aNkpISNBoNly5dAkqHx3t4eGBpaXmv2VIn6qp8pqWl8eOPP9KvXz/atm3LlStXWLVqFUuWLFEaJuLj4/nll194++238fHxQa1Ws2zZMtavX8+AAQOU4+mifD7SjRKmpqZKIwGAl5cXw4YNY+fOnUovBFNT0zo5toODA1ZWVuzevZuAgIByXXJuZ2xsTPfu3YmOjubJJ59k7969WgWnTEWrTTxowzeg9KKqzrjumk5wc2f+GBgYPJD5U5c0Gg3t2rUrVzkAysNHSEgI3bp14/Dhwxw9epQNGzYwcuTIcj13HhQWFhZcv369RvsYGBgwePBgBg4cSE5ODpaWlkqvm7JZ3k1MTBg7diyjRo3i6tWr2NjYsHPnTho1alRpBWloaMhjjz1WrldVWfr0rWItY2Njw5QpUwgLC2PGjBn85z//qXKy2duv34ruXRVNzliT63Xz5s1s2bKFESNG4O7uTsOGDVmzZs0jsYzonfXWvebFw3bP1IdrvaioiD179hAUFFRhudf3a/12kp+1p+yF0MjICBsbG637X3Um/W7fvj1GRkb8888/tG7dmuPHjzN16lTl95UrVyoNH87OzpiamrJo0aJKh8OU0Wg09OjRo8LevLe/aOobXZZNa2trjI2NtXreubq6UlBQQG5urhJO38rm3d6JNBoNnp6evP/+++X2u72uv5cJ6u8n7jlz5mBra8vIkSOxtbXFyMiIDz74oMpyrWt1VT43btyIt7e3Ml+eh4cHDRs25L///S9vvPEGdnZ2REZG8tRTTyk9y9zd3cnPzyciIoL+/fsr91FdlM+Ho69qLTI0NKy0VcjMzAwbGxuSkpK0ticmJiqTqri6upKSksLNmzeV35OSktBoNFoTrzRu3Jhp06aRnZ3N/PnzuXXr1l3TFRQUxIkTJ9i+fTv5+fnKskIPm5SUFI4ePUrnzp2B0vzUaDRacz/k5eUpk3mK2tWsWTPS0tKwt7fHyclJ68/tvX+cnZ3p27cvn3zyCT169GDXrl06TPX98fT0VMbV1pShoSG2trYYGxsTFxeHr69vuRu4sbExdnZ2GBoaEhcXR/v27SsdJqDRaDh//ny5rwGpqanV+lqmS7a2toSGhlJQUMCMGTPIzc3F1dW1wvulnZ2dVnm6V6dPn1b+np+fT2pqqtJrJzExkQ4dOtC9e3dl2MKdw2KMjY2VuVIeZtXJi0eBPlzrBw4cIDc3t9JG3NTUVIyMjKr8gq0PJD9rT9kLYZMmTe5plbcGDRrQuXNnYmNj2bdvH9bW1vj5+Sm/JyYmEhgYSOfOnfHw8MDW1lbrKzRUfD8seya483nAyclJr+eS0mXZbN68OWq1WisvL1y4gKmpKRYWFsq2B6FeL3snatasGWq1GgsLi3Ll4G4fIFxcXLhy5YrWEICzZ89qNW7fa9y5ubmkp6fzyiuv0KZNG9zc3Lh586bW0BBPT0/Onz+vdx8j6qp8FhQUlLtHlv27LM8rC3PnBwddlM9HulHi1q1b5OTkkJOTo3R5yc/PV2YjrsiLL77Ili1biI2NJSMjg3Xr1pGQkEBwcDBQ2oWurAU6JSWF+Ph4lixZQseOHbVaIKG09WnatGlcvny5yoYJFxcXWrRowapVq+jUqZNez3pcXWX5n52dTXJyMlu3biUsLAwvLy8lP52dnQkICOD7778nISGBlJQUFi5ciJmZmTL7sai5mzdvkpycrPUnKyuL3r17k5eXx1dffcXp06fJzMzk2LFjREREcPPmTQoLC/nhhx84efIkWVlZnD59WqtR7kHk7+9PWlqa1lhltVpNcnIyV65coaioSMmjstb3a9eusWPHDtLS0khOTmbZsmX89ddfDB8+XIkjIyODvXv3cuHCBVQqFV999RWpqalaE9pu2LCBI0eOkJmZqUwelpKSQq9evbTSmJCQUG7lCX1kY2PD9OnTKSoq4tNPP6Vv377Ex8ezfv16MjIyiImJYevWrbW26s0vv/zCsWPHSE1N5dtvv8XY2Fi5L7i4uHDixAkSExNJT09n6dKlZGVlae3fpEkTVCoVWVlZStfPh1F18uJRoMtrvUxUVBStWrVSvmzdKSEhgccff7zOemnWJslP/dKtWzeOHj3K//73P5566imtFw9nZ2cOHDjA2bNnleeoOz/ANWnShMTERLKzs5WXuJdeegmVSsWSJUs4d+4carWaQ4cOaS1DrI90WTZ79eqlrBqXkZHBkSNHWL9+Pb169dLqIahv9frd3om6deuGlZUVc+fOJT4+nqysLOLj41mxYsVdG7jbtGmDi4sL4eHhyuSKP/30k1avpnuN29zcHAsLC6KiolCr1cTHx/P9999rxd21a1esrKyYN28eCQkJZGZmcvDgQZ3PjVRX5TMgIICDBw+yY8cOMjMzSUxMZNmyZTRr1kyZBLhDhw5ERUURFxdHVlYWx44dY926dUpvqzK6KJ+P9PCN48ePM2rUKAAaNWqEi4sLEydOpGXLlpU+sD333HPcvHmT1atXk5OTg4uLCx9++CGenp5AaWv31KlTWb58OZ988onWkqAVsbS05L///S+ffvopX3zxBR9++GGl6e3RowcJCQkPbDf5O5Xlv6GhIebm5jRt2pTXXnuNnj17an0pGDt2LMuXL2fu3LnKkqD/93//p9et9Lqye/duFi9eXOXShgkJCXz00Uda2zp16sSHH37IjBkzWLNmjbLCib29PW3btlWGGN24cYPFixdz5coVLCwsaN++vd4vt3Y37u7ueHt7ay199N133xEfH6+EKcur2/N1z549rFy5Eiid2yA0NBRvb29ln5KSErZu3UpGRgZGRka0bNmSmTNnav2/3LhxgyVLlpCTk4OZmRnNmjUjLCxMK57CwkIOHDig1RVXn1lbWzN9+nRmzJjBt99+y3vvvccvv/zCxo0bsba25uWXX661JaYGDRrEihUryMjIoGnTpkyZMkXp4vnqq6+SlZXFrFmzMDEx4emnn6Zbt25ac9EEBwcTHh7OBx98QGFh4QOxJOi9qE5ePAp0ea1D6Rj8EydO8N5771Waxri4OGU5bH0n+alfHn/8cWxtbUlLSyuXJ8OGDeO7775j+vTpmJub07dv33IfwkJCQvj++++ZMGECt27dYv369Xh4eBAWFkZkZCShoaGUlJTg4OBAx44d6/PUakyXZdPe3p6pU6eyYsUKJk+ejLW1Nc888wz9+vVTwuhjvX63dyKAsLAw1qxZw4IFC8jLy8PGxoaWLVsqc0hVxNDQkEmTJhEREcH//d//0aRJE4YOHcr8+fOVZ3hTU9N7jnvixIksW7aMDz/8ECcnJ4YMGcIXX3yhhGnYsCGhoaGsWLFCWZXDxcWFYcOG1UaW3bO6Kp9PP/00N2/eZNu2baxYsQIzMzNatWqlNYllv379MDAwYN26dVy+fBlLS0s6dOjA66+/roTRVfk00DzIA0QfMb/99hvR0dF8/fXXuk6K0FPr16/n77//Zt68eRWOrxUVO3LkCMuWLePLL7/UuxUYtm3bxsGDB/nPf/6j66QI8cDT52v933//ZeXKlcoSeg8CyU+hr/S5bD7K9XpycjIfffQRn3/+OV5eXrpOjs5I+SzPKDQ0NLRejyhqLD8/nwsXLvDjjz8SHBys1SomxO1WrVrF8OHDK+3KKirm5OSERqPBxsbmri3zupCcnEyvXr20xqIKIe6NPl/r586d4+mnn1a62T4IJD+FvtLnsvko1esHDhzg0qVLGBoacu7cOX744QesrKwICQmp8YT1DxMpn+VJT4kHQHh4OHFxcQQEBPDee+9Ji78QQgghhBBCr+3Zs4dff/2VS5cu0bhxY/z8/Bg2bJheT/ApdEMaJYQQQgghhBBCCKET+jWIRQghhBBCCCGEEI8MaZS4i+vXrzNy5EjUarWuk1LOggUL2LJli66TIXREyqbQZ1I+a4/kpRDiTkuXLqUmU8JlZWUREhLCmTNn6i5R4qGmz3XRJ598wt9//63rZIj79EgvCVqVjRs30q5dO5ycnABYtmwZSUlJpKamYm1tTXh4eLl99u3bx8aNG7lw4QKWlpb06dOHF198USvMtm3b2L59O1lZWdjb2/Pqq68SGBio/F62rOOdVq1apSyh079/f6ZPn05QUBBmZma1edq1Jjw8nD179gBgZGSkLPvZqVOncst+ipqRsin0mZTP2iN5KcSjJzw8nNzcXD7++GNdJ0UIQHd1EcDff//NunXryMzMxNHRkTfeeENrWdp+/fqxYsUKOnbsqHcrWYjqk7fCShQUFLBr1y6mTJmibNNoNAQGBpKSksKxY8fK7XP48GG++eYbRowYgb+/P+np6URERGBiYqKsQ7tjxw5Wr17N6NGj8fHxQaVSERERgbm5OQEBAUpcpqamLFy4UCv+sgdBKF3j1tHRkb179ypx66PWrVszYcIESkpKuHbtGidOnGDDhg3ExMQwbdo0GjZsqOskVqq4uBhDQ0O9mx1YyqbQZ1I+a4/kpRBCCF3TZV106tQpvvrqK0JCQujYsSMHDhxgwYIFzJgxAx8fHwDat29PREQER44coX379vWQI6IuSKNEJQ4fPgxA8+bNlW1vvvkmAJs3b67wAty7dy8dOnSgd+/eADg6OvLyyy+zadMmevfujYGBAXv37iUoKIiuXbsqYc6cOcOmTZu0HgaBKmemDQgIIC4uTq8fBhs0aKCch62tLZ6enrRp04YpU6awefNmQkJCKCoqIjIyktjYWK5fv07Tpk0ZMGAAoHpbUgAAHjpJREFU/v7+AJw8eZKwsDCmTZvG2rVrSUlJwc3NjVGjRuHl5UVeXh4jR45k4sSJWnl49OhRPv/8c7777jusrKzIzs5mxYoVHD16FABfX1+GDx+Os7MzAOvXr2f//v0EBwfzyy+/kJWVxU8//aR3DSdSNoU+k/JZeyQvhRAlJSWsWrWK6OhoAAIDAykpKdEKc+TIEX799VdSU1MB8Pb2ZtiwYbi5uWmFu3jxImvWrCEpKYkmTZowYsQI2rRpo/weHx/PqlWrOH/+PGZmZjz11FMMHjxYerY+4nRZF/3++++0bNmSV199FQA3NzdOnjzJ77//zvvvvw+AoaEh7dq1IzY2VholHmDSx6USCQkJeHl51egr+a1bt2jQoIHWNhMTEy5fvszFixeVMLd/aSoLo1KpKCoqUrYVFhYyduxY3nnnHT7//HPOnTtX7nje3t6oVCoKCwtrcmo65+7ujr+/P/v37wdg8eLFJCQk8O677/LFF18QGBjInDlzSE5O1tpvzZo1DBw4kDlz5mBhYcHChQvRaDSYmZnRoUMHYmNjtcLHxMTQpk0brKysKCgoICwsjAYNGhAaGsrMmTOxsbFhxowZFBQUKPtkZWURGxvLxIkTmTdvXrn/T30gZVPoMymftUfyUgixZcsWoqKiGDlyJDNnzqSkpKTc805+fj59+/Zl1qxZhIaG0qhRI+bMmaN1PQNERkby3HPPMW/ePB577DG++uor8vPzAcjOzmb27Nl4enoyZ84c3nnnHeLi4lizZk29navQT7qsi06dOkXbtm21wrRt25ZTp05pbfP29iYhIaHa6RP6RxolKnHx4kVsbGxqtI+/vz8HDx7k6NGjlJSUkJGRwdatWwHIyckBSi+k6OhoVCoVGo2GM2fOEBUVRXFxMbm5uQC4uLgwZswYPvroI9577z0aNGjAtGnTuHDhgtbxbGxsKC4uJjs7uxbOuH65ubmRmZmJWq0mLi6OiRMn4ufnh6OjI3369KFdu3bs3LlTa58BAwbQqlUrXF1d6devH+np6cq5d+/enYMHD3Lz5k2g9GH6n3/+oVu3bgDExcWh0WgYO3YsHh4euLq6MmrUKPLz8zl06JByjKKiIsaPH4+Xlxfu7u4YGRnVU45Un5RNoc+kfNYeyUshxB9//MFLL71Ely5dcHV1Zfjw4eV6MHXu3JnOnTvj7OyMh4cHY8eOJSsrC5VKpRXu+eefJyAgAGdnZwYOHMj169eVD0Dbt2/HxsaGt99+Gzc3Nzp06MCgQYPYtm2b1scb8ejRZV2Uk5ODlZWVVtxWVlZKHGVsbW3Jzs6muLj4Xk9T6Jj0x6pERa13VQkKCkKtVjN37lyKi4tp1KgRffv2ZcOGDUrrYv/+/cnJyWHatGloNBqsrKwIDAxk8+bNShhfX198fX2VeJs3b87kyZP5888/le5S8P/H9j6IX6g0Gg0GBgacO3cOjUbDxIkTtX4vKiqiVatWWts8PDyUv9va2gJw9epV7Ozs8Pf3x9TUlAMHDhAYGMjBgwfRaDQ88cQTAJw9e5asrCyGDh2qFWdhYSGZmZla8VbVXVnXpGwKfSbls/ZIXgrxaMvLy+PKlSta16KhoSHe3t5cvnxZ2aZWq1m3bh0qlYpr165RUlKCRqPh0qVLWvHd/hxV9pJ59epVANLT0/Hx8dGaKLBFixYUFRWhVqu19hWPFl3WRdVlYmKCRqPh1q1bevlBUVRNGiUqYWFhwfXr12u0j4GBAYMHD2bgwIHk5ORgaWnJ8ePHgdJxUlB60YwdO5ZRo0Zx9epVbGxs2LlzJ40aNcLS0rLCeA0NDXnsscfKLcNTlr7K9tNnaWlpODg4KI0Ts2fPLjdm8c4bYEU3GY1GA4CxsTFPPvkksbGxBAYGEhMTQ8eOHTE1NVXCeXp6KuPPbte4cWPl7/o2f0RFpGwKfSbls/ZIXgohqmPOnDnY2toycuRIbG1tMTIy4oMPPig3fOP256iyl76y56i70bcJv0X90mVdZG1trTSclbl69Wq5D4jXr1+nQYMGD8RzvKiYDN+ohKenJ+np6fe0r6GhIba2thgbGxMXF4evr2+5BzZjY2Ps7OwwNDQkLi6O9u3bV7qMjUaj4fz58+UuwNTU1Afiy/6dUlJSOHr0KJ07d8bT0xONRkNOTg5OTk5af8p6Q1RXt27dOH78OGlpaRw5ckQZugHQrFkz1Go1FhYW5Y5ze6PEg0DKptBnUj5rj+SlEI82MzMzbGxstMbPazQarWEZubm5pKen88orr9CmTRvc3Ny4efNmjbuxu7q6cvr0aa1JNBMTEzE2NlZeIsWjSZd1ka+vb7mJNI8dO6bVewhK3y28vLzuKY1CP0hPiUr4+/uzevVqcnNzsbCwAEq7x+Xn53PlyhWKioqUcXhubm4YGxtz7do1/v77b/z8/CgqKiI6Opq//vqLsLAwJd6MjAxUKhU+Pj7cuHGDrVu3kpqayrhx45QwGzZswMfHB2dnZ27evMkff/xBSkoKI0eO1EpjQkJCuclf9M2tW7fIycnRWhJ048aNeHl5ERwcTMOGDenatSuLFy9m6NChNGvWjOvXr3Py5EkcHR3p1KlTtY/VvHlzmjRpwtdff42lpSWtW7dWfuvWrRtbtmxh7ty5DBgwAHt7ey5dusTBgwd59tlnlRU4HgRSNoU+k/JZeyQvhRDPPfccv/32Gy4uLri7u7N9+3ZycnKU4Rfm5uZYWFgQFRWFvb092dnZrFy5ssZd2Hv37s0ff/zBDz/8QN++fcnKymL16tX06dNH6XUqHk26rIv69u3L9OnT+e2333jiiSc4cOAAJ0+e5NNPP9VKY2JiotRFDzhplKiEu7s73t7eWkudfffdd8THxythPvroIwAWLVqEg4MDAHv27GHlypVAaeteaGgo3t7eyj4lJSVs3bqVjIwMjIyMaNmyJTNnzlT2B7hx4wZLliwhJycHMzMzmjVrRlhYmFY8hYWFHDhwgKlTp9ZdJtSC48ePM2rUKAwNDTE3N6dp06a89tpr9OzZUxmuMXbsWH799VdWrVrF5cuXady4Md7e3uXmlKiOrl278ssvv/D8889rffEzNTUlLCyMNWvWsGDBAvLy8rCxsaFly5aYm5vX2vnWBymbQp9J+aw9kpdCiODgYHJycvjuu++A0om9u3btqny5NjQ0ZOLEiSxbtowPP/wQJycnhgwZwhdffFGj49ja2vLJJ5+watUqPvroI8zNzXnqqad44403av2cxINFl3VR8+bNef/994mMjGTdunU4OTnx/vvv4+Pjo4TJzs4mKSmJCRMm1F0miDpnoKnOYLJH1JEjR1i2bBlffvllpV1adWXbtm0cPHiQ//znP7pOitABKZtCn0n5rD2Sl0IIIXRNn+uilStXkpeXx+jRo3WdFHEfjEJDQ0N1nQh95eTkhEajwcbGRu++picnJ9OrVy+lG5V4tEjZFPpMymftkbwUQgiha/pcF50/f57nn39eJrl8wElPCSGEEEIIIYQQQuiEfvW/EUIIIYQQQgghxCNDGiWEEEIIIYQQQgihE9IoIfTayZMnCQkJ4dq1a7pOihBCCCGEEEKIWiaNEkIIIYQQQujQ9evXGTlyJGq1WtdJKWfBggVs2bJF18kQQjzEjHWdACGEEEIIIR5lGzdupF27djg5OQGwbNkykpKSSE1NxdramvDw8HL77Nu3j40bN3LhwgUsLS3p06cPL774olaYbdu2sX37drKysrC3t+fVV18lMDBQ+T01NZX169dz7tw5srKy6N+/PyEhIVpx9O/fn+nTpxMUFISZmVkdnL0Q4lEnjRKi3h05coQFCxawbNkyjIyMUKvVvPvuu/Ts2ZNRo0YBEBkZyenTp3n11VeB0uV+1q5dS0pKCm5ubowaNQovLy8lzqSkJNasWcOZM2cwNzcnICCAQYMGKZVnaGgobm5umJmZERUVhYGBAd27d2fw4MF6t96yEEIIIR4dBQUF7Nq1iylTpijbNBoNgYGBpKSkcOzYsXL7HD58mG+++YYRI0bg7+9Peno6ERERmJiY0KdPHwB27NjB6tWrGT16ND4+PqhUKiIiIpTnpLJjN2nShE6dOhEZGVlh+tzd3XF0dGTv3r1K3EIIUZvkbUzUuxYtWnDr1i3OnDkDlM4bYWFhQXx8vBLm5MmT+Pn5Kf9es2YNAwcOZM6cOVhYWLBw4ULKVrNNSUlh5syZBAQEMG/ePCZNmkRycjLffvut1nFjYmIwMjJixowZvPnmm/zxxx/s27evHs5YCCGEEKJihw8fBqB58+bKtjfffJPnnnsOZ2fnCvfZu3cvHTp0oHfv3jg6OtK+fXtefvllNm3apDwf7d27l6CgILp27YqjoyNPPfUUPXv2ZNOmTUo83t7eDB06lK5du2JqalppGgMCAoiLi6uN0xVCiHKkUULUu4YNG+Ll5cXJkyeB0gaIPn36cPHiRa5cuUJBQQFnzpyhZcuWyj4DBgygVatWuLq60q9fP9LT08nOzgZg8+bNdOnSheDgYJydnfHx8WHkyJHs37+fq1evKnG4ubkxYMAAXFxc6NKlCy1btuTEiRP1e/JCCCGEELdJSEjAy8sLAwODau9z69YtGjRooLXNxMSEy5cvc/HiRSWMiYlJuTAqlYqioqIapdHb2xuVSkVhYWGN9hNCiOqQ4RtCJ/z8/IiPj+eVV14hISGBvn37cvLkSU6ePImlpSVGRkZ4e3uTlJQEgIeHh7Kvra0tAFevXsXOzo6zZ8+iVqsr7PWQmZmJlZVVuTgAbGxstBothBBCCCHq28WLF7GxsanRPv7+/ixfvpyjR4/SunVr1Go1W7duBSAnJwcHBwfatm1LdHQ0HTt25LHHHuPs2bNERUVRXFxMbm5ujY5pY2NDcXEx2dnZyrwXQghRW6RRQuhEy5Yt2bZtG2lpaeTl5eHl5YWfnx8nT57EysoKX19fjI3/f/E0MjIqF0dZ90SNRkOPHj144YUXyoUpa8CoKA4DAwMlDiGEEEIIXaioR0NVgoKCUKvVzJ07l+LiYho1akTfvn3ZsGGD0uOif//+5OTkMG3aNDQaDVZWVgQGBrJ58+Ya9coAlPRJTwkhRF2QRgmhEy1atKCoqIjNmzfTokULDA0NadmyJREREVhZWeHv71/tuJo1a0ZaWpq03AshhBDigWNhYcH169drtI+BgQGDBw9m4MCB5OTkYGlpyfHjxwFwdHQEShsSxo4dy6hRo7h69So2Njbs3LmTRo0aYWlpWaPjlaWvpvsJIUR1yJwSQifK5pWIiYlR5o7w8fHh8uXLnD59Wms+iaq89NJLqFQqlixZwrlz51Cr1Rw6dIglS5bUVfKFEEIIIWqFp6cn6enp97SvoaEhtra2GBsbExcXh6+vb7mGA2NjY+zs7DA0NCQuLo727dvXeOWx1NRUbG1tsba2vqd0CiHE3UhPCaEzfn5+Wg0QJiYm+Pj4cObMGby9vasdj4eHB2FhYURGRhIaGkpJSQkODg507NixrpIuhBBCCFEr/P39Wb16Nbm5uVhYWACgVqvJz8/nypUrFBUVkZycDJRO2m1sbMy1a9f4+++/8fPzo6ioiOjoaP766y/CwsKUeDMyMlCpVPj4+HDjxg22bt1Kamoq48aNU8IUFRWRlpYGlA7NyMnJITk5mYYNG2r1QE1ISKBt27b1kBtCiEeRgUYG1QshhBBCCKEzU6dOpVu3bvTp0weA0NBQraXSyyxatAgHBweuXbvGnDlzSElJAcDX15fXX38dHx8fJWxaWhrffPMNGRkZGBkZ0bJlSwYPHoyLi4sSJisri/Hjx5c7jp+fH6GhoUBpY8XIkSOZOnUqvr6+tXnaQggBSKOEEEIIIYQQOnXkyBGWLVvGl19+WeOhFXVt27ZtHDx4kP/85z+6TooQ4iGlX3c9IYQQQgghHjH+/v707t2by5cv6zop5RgbG/Pmm2/qOhlCiIeY9JQQQgghhBBCCCGETkhPCSGEEEIIIYQQQuiENEoIIYQQQgghhBBCJ6RRQtSLb775hsmTJ1NUVKS1/fjx47zxxhskJSXdV/zr168nPDz8vuIQQgghhBBCCFG/pFFC1Iu33nqL69evs2HDBmVbXl4e3377LcHBwTRv3rxOj39nY4gQQgghhBBCCN0z1nUCxKPB3NycMWPGMHv2bJ544gm8vb356aefMDc3p0uXLsyYMYPExERMTEwICAhgxIgRmJmZARAeHk5ubi4ff/yxEt/69evZv38/X3zxRYXHCw0NxdXVFVNTU/bs2YODgwOzZ88mLS2NlStXkpCQgImJCa1atWL48OFYW1vXSz4IIYQQQgghhPj/pKeEqDdt2rTh2WefJTw8nL///pvY2FjGjRvH7NmzMTU1Zfbs2UyePJlTp06xePHi+z5eTEwMAJ9++injxo3jypUrTJ8+naZNmzJr1iymTZtGfn4+c+fOpaSk5L6PJ4QQQgghhBCiZqRRQtSrwYMHo9Fo+PLLLxkwYAAqlYr8/HwmTJiAu7s7fn5+jBo1igMHDqBWq6sdb0hICOPGjdPa5uDgwNChQ3F1dcXNzY0dO3bg4eHB4MGDcXNzw8PDg/Hjx6NSqTh79mxtn6oQQgghhBBCiCrI8A1Rr0xMTAgODmbZsmW88MILrFy5Eg8PDxo1aqSEad68OQYGBqSlpeHk5HTPx/Ly8tL699mzZ0lISGDIkCHlwqrVary9ve/5WEIIIYQQQgghak4aJUS9MzIywsDAAEPD6nXUMTAwQKPRaG0rLi6ucj9TU1Otf2s0Gtq1a8fQoUPLhbWysqpWWoQQQgghhBBC1B5plBA65erqSnR0NDdv3lR6SyQlJaHRaHBzcwPA0tKS8+fPa+2XnJxc42M1a9aMv/76C3t7e4yNpegLIYQQQgghhK7JnBJCp7p164apqSmLFi0iJSWF+Ph4lixZQseOHZWhG61ateLcuXPs2rULtVrNpk2bSEpKqvGxevfuTV5eHl999RWnT58mMzOTY8eOERERwc2bN2v71IQQQgghhBBCVEE+FwudMjU1ZerUqSxfvpxPPvlEa0nQMv7+/vTv35/IyEgKCgro1q0bvXr14tChQzU6lq2tLTNmzGDNmjXMmjWLwsJC7O3tadu2LQ0aNKjtUxNCCCGEEEIIUQUDzZ2D9YUQQgghhBBCCCHqgQzfEEIIIYQQQgghhE5Io4QQQgghhBBCCCF0QholhBBCCCGEEEIIoRPSKCGEEEIIIYQQQgidkEYJIYQQQgghhBBC6IQ0SohH0tKlSwkNDdV1MoQQQgghhBDikWas6wSIR1N4eDh79uwBwMjICHNzc5o2bUqnTp3o2bMnxsZSNIUQQgghhBDiYSdvfkJnWrduzYQJEygpKeHatWucOHGCDRs2EBMTw7Rp02jYsKGukyiEEEIIIYQQog5Jo4TQmQYNGmBtbQ2Ara0tnp6etGnThilTprB582ZCQkIoKioiMjKS2NhYrl+/TtOmTRkwYAD+/v4AlJSUEBERwYkTJ8jJycHOzo6goCCCg4MxNDRUwqxatYro6GgAAgMDKSkp0c1JCyGEEEIIIYRQSKOE0Cvu7u74+/uzf/9+QkJCWLx4MZmZmbz77rvY2dlx+PBh5syZw+zZs/H09KSkpARbW1smTpyIpaUlKpWKJUuWYGFhQY8ePQDYsmULUVFRjB49Gg8PD7Zv305sbCzNmjXT8dkKIYQQQgghxKNNJroUesfNzY3MzEzUajVxcXFMnDgRPz8/HB0d6dOnD+3atWPnzp0AGBsbM2DAALy9vXFwcKBLly48++yzxMXFKfH98ccfvPTSS3Tp0gVXV1eGDx+u9NAQQgghhBBCCKE70lNC6B2NRoOBgQHnzp1Do9EwceJErd+Liopo1aqV8u8dO3awa9cuLl68SGFhIcXFxTRp0gSAvLw8rly5gq+vrxLe0NAQb29vLl++XD8nJIQQQgghhBCiQtIoIfROWloaDg4OSuPE7Nmzy63GYWJiAsC+ffv46aefGDJkCL6+vpiZmbFt2zb++ecfXSRdCCGEEEIIIUQNSKOE0CspKSkcPXqUV199FU9PTzQaDTk5OVo9I26XmJiIt7c3ffr0UbZlZmYqfzczM8PGxoZTp04pcWg0GlQqFTY2NnV7MkIIIYQQQggh7koaJYTO3Lp1i5ycHK0lQTdu3IiXlxfBwcE0bNiQrl27snjxYoYOHUqzZs24fv06J0+exNHRkU6dOuHs7Mzu3bs5fPgwTk5OxMXFER8fT+PGjZXjPPfcc/z222+4uLjg7u7O9u3bycnJkUYJIYQQQgghhNAxA41Go9F1IsSjJzw8nD179gClczyYm5vTtGlTOnfuTM+ePZXhGkVFRfz666/s3buXy5cv07hxY7y9vXnttdfw8vKiqKiI77//ngMHDqDRaOjUqRNNmjQhOjqa8PBwAIqLi1m5ciW7d+8GoHv37hQXF5Oenk5oaKguTl8IIYT4f+3dX0hTfRzH8Y9rLjRczsJlhGhtE+rCZd4V7GKBq4uIEILoJsEiu+gquygpoasuCioiu/GivJIQa5RFZQmrCMH+QBIsKIdzJNZKpZrbznMRHZ49M+p5Htt5eHq/YDdnv+9333M4N/vyPecHAABEUwIAAAAAAFiELUEBAAAAAIAlaEoAAAAAAABL0JQAAAAAAACWoCkBAAAAAAAsQVMCBTMzM6PW1lYlEgmrS8lz6tQpXbt2zeoyAAAAAOC3Yre6APw++vr6tH79eq1YsUKS1N3drZcvXyoWi6m8vNzcwvPPHjx4oL6+Pk1MTMjpdCoUCmnbtm05awYGBnTz5k29fftWy5cv144dOxQIBMzv7927p/Pnz+flvnz5shwOhySpublZx44dUzAYVGlp6UKeNgAAAADgO2hKoCC+fPmiu3fv6vDhw+YxwzAUCAQ0NjamZ8+e5cWMjIzozJkz2rNnj/x+v8bHx9XV1SWHw6FQKCRJunXrlnp6erRv3z55vV5Fo1F1dXVpyZIlamxsNHMtXrxYZ8+ezcn/rSEhSdXV1XK73RoaGjJzAwAAAAB+LR7fQEGMjIxIkurq6sxjLS0t2rJli6qqquaNGRoa0oYNG9TU1CS3262GhgZt375d/f39MgzDXBMMBrVp0ya53W5t3LhRmzdvVn9/f16+8vLynM9fNTY2KhKJLMTpAgAAAAB+ApMSKIjR0VGtXr1aRUVFPx0zNzen4uLinGMOh0NTU1OanJxUZWWl5ubmciYevq2JRqNKp9Oy27/e4qlUSm1tbcpms6qpqdHOnTtVW1ubE+fxeHTlyhWlUqm8nAAAAACAhcekBApicnJSLpfrb8X4/X4NDw/r6dOnymazisfjCofDkqRkMilJqq+v1+DgoKLRqAzD0KtXr3Tnzh1lMhlNT09LklauXKn9+/ervb1dBw8eVHFxsTo6OjQxMZHzey6XS5lMRu/evVuAMwYAAAAA/AiTEiiI+SYafiQYDCqRSOjkyZPKZDIqKSnR1q1b1dvba05cNDc3K5lMqqOjQ4ZhaOnSpQoEArp69aq5xufzyefzmXnr6up06NAh3bhxQy0tLebxb/WlUql/e7oAAAAAgJ9AUwIFUVZWppmZmb8VU1RUpN27d2vXrl1KJpNyOp16/vy5JMntdkv62khoa2vT3r179eHDB7lcLt2+fVslJSVyOp3z5rXZbFqzZk3e1qTf6vteHAAAAABgYfH4BgqipqZG4+Pj/yjWZrOpoqJCdrtdkUhEPp8vr3Fgt9u1bNky2Ww2RSIRNTQ0yGab//Y2DENv3rzJe9llLBZTRUXFvC/BBAAAAAAsPCYlUBB+v189PT2anp5WWVmZJCmRSOjz5896//690um0Xr9+LUlatWqV7Ha7Pn78qEePHmnt2rVKp9MaHBzUw4cP1dnZaeaNx+OKRqPyer2anZ1VOBxWLBbTgQMHzDW9vb3yer2qqqrSp0+fdP36dY2Njam1tTWnxtHRUdXX1//6iwEAAAAAkERTAgVSXV0tj8ejSCSiUCgkSbpw4YJevHhhrmlvb5cknTt3TpWVlZKk+/fv69KlS5K+vhvi+PHj8ng8Zkw2m1U4HFY8HteiRYu0bt06nThxwoyXpNnZWV28eFHJZFKlpaWqra1VZ2dnTp5UKqXHjx/ryJEjv+4iAAAAAAByFBmGYVhdBH4PT548UXd3t06fPv3dRyusMjAwoOHhYR09etTqUgAAAADgt/Hf+meI/zW/36+mpiZNTU1ZXUoeu92esxMHAAAAAODXY1ICAAAAAABYgkkJAAAAAABgCZoSAAAAAADAEjQlAAAAAACAJWhKAAAAAAAAS9CUAAAAAAAAlqApAQAAAAAALPEHH8VjFbSZjLsAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"6zcqiJUEwwdQ"},"source":["Note that cosine annealing scheduler is a bit different from other schedules as soon as it starts with `base_lr` and gradually decreases it to the minimal value while triangle schedulers increase the original rate."]},{"cell_type":"markdown","metadata":{"id":"27Y_3GcPgTfS"},"source":["## MF with PyTorch on ML-100k"]},{"cell_type":"markdown","metadata":{"id":"2jqJNaQhK8ji"},"source":["### Utils"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vsby4vwWGlWJ","executionInfo":{"status":"ok","timestamp":1633680741267,"user_tz":-330,"elapsed":521,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"52308560-91b9-4db7-daea-3dc502d34bb5"},"source":["%%writefile utils.py\n","\n","import os\n","import requests\n","import zipfile\n","\n","import numpy as np\n","import pandas as pd\n","import scipy.sparse as sp\n","\n","\"\"\"\n","Shamelessly stolen from\n","https://github.com/maciejkula/triplet_recommendations_keras\n","\"\"\"\n","\n","\n","def train_test_split(interactions, n=10):\n"," \"\"\"\n"," Split an interactions matrix into training and test sets.\n"," Parameters\n"," ----------\n"," interactions : np.ndarray\n"," n : int (default=10)\n"," Number of items to select / row to place into test.\n","\n"," Returns\n"," -------\n"," train : np.ndarray\n"," test : np.ndarray\n"," \"\"\"\n"," test = np.zeros(interactions.shape)\n"," train = interactions.copy()\n"," for user in range(interactions.shape[0]):\n"," if interactions[user, :].nonzero()[0].shape[0] > n:\n"," test_interactions = np.random.choice(interactions[user, :].nonzero()[0],\n"," size=n,\n"," replace=False)\n"," train[user, test_interactions] = 0.\n"," test[user, test_interactions] = interactions[user, test_interactions]\n","\n"," # Test and training are truly disjoint\n"," assert(np.all((train * test) == 0))\n"," return train, test\n","\n","\n","def _get_data_path():\n"," \"\"\"\n"," Get path to the movielens dataset file.\n"," \"\"\"\n"," data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),\n"," 'data')\n"," if not os.path.exists(data_path):\n"," print('Making data path')\n"," os.mkdir(data_path)\n"," return data_path\n","\n","\n","def _download_movielens(dest_path):\n"," \"\"\"\n"," Download the dataset.\n"," \"\"\"\n","\n"," url = 'http://files.grouplens.org/datasets/movielens/ml-100k.zip'\n"," req = requests.get(url, stream=True)\n","\n"," print('Downloading MovieLens data')\n","\n"," with open(os.path.join(dest_path, 'ml-100k.zip'), 'wb') as fd:\n"," for chunk in req.iter_content(chunk_size=None):\n"," fd.write(chunk)\n","\n"," with zipfile.ZipFile(os.path.join(dest_path, 'ml-100k.zip'), 'r') as z:\n"," z.extractall(dest_path)\n","\n","\n","def read_movielens_df():\n"," path = _get_data_path()\n"," zipfile = os.path.join(path, 'ml-100k.zip')\n"," if not os.path.isfile(zipfile):\n"," _download_movielens(path)\n"," fname = os.path.join(path, 'ml-100k', 'u.data')\n"," names = ['user_id', 'item_id', 'rating', 'timestamp']\n"," df = pd.read_csv(fname, sep='\\t', names=names)\n"," return df\n","\n","\n","def get_movielens_interactions():\n"," df = read_movielens_df()\n","\n"," n_users = df.user_id.unique().shape[0]\n"," n_items = df.item_id.unique().shape[0]\n","\n"," interactions = np.zeros((n_users, n_items))\n"," for row in df.itertuples():\n"," interactions[row[1] - 1, row[2] - 1] = row[3]\n"," return interactions\n","\n","\n","def get_movielens_train_test_split(implicit=False):\n"," interactions = get_movielens_interactions()\n"," if implicit:\n"," interactions = (interactions >= 4).astype(np.float32)\n"," train, test = train_test_split(interactions)\n"," train = sp.coo_matrix(train)\n"," test = sp.coo_matrix(test)\n"," return train, test"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Writing utils.py\n"]}]},{"cell_type":"markdown","metadata":{"id":"d9wghzkqLR2i"},"source":["### Metrics"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EWmOo0AqKiEN","executionInfo":{"status":"ok","timestamp":1633680748266,"user_tz":-330,"elapsed":577,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"5b4b9d36-218e-42ed-da99-f5af0b4b93f9"},"source":["%%writefile metrics.py\n","\n","import numpy as np\n","from sklearn.metrics import roc_auc_score\n","from torch import multiprocessing as mp\n","import torch\n","\n","\n","def get_row_indices(row, interactions):\n"," start = interactions.indptr[row]\n"," end = interactions.indptr[row + 1]\n"," return interactions.indices[start:end]\n","\n","\n","def auc(model, interactions, num_workers=1):\n"," aucs = []\n"," processes = []\n"," n_users = interactions.shape[0]\n"," mp_batch = int(np.ceil(n_users / num_workers))\n","\n"," queue = mp.Queue()\n"," rows = np.arange(n_users)\n"," np.random.shuffle(rows)\n"," for rank in range(num_workers):\n"," start = rank * mp_batch\n"," end = np.min((start + mp_batch, n_users))\n"," p = mp.Process(target=batch_auc,\n"," args=(queue, rows[start:end], interactions, model))\n"," p.start()\n"," processes.append(p)\n","\n"," while True:\n"," is_alive = False\n"," for p in processes:\n"," if p.is_alive():\n"," is_alive = True\n"," break\n"," if not is_alive and queue.empty():\n"," break\n","\n"," while not queue.empty():\n"," aucs.append(queue.get())\n","\n"," queue.close()\n"," for p in processes:\n"," p.join()\n"," return np.mean(aucs)\n","\n","\n","def batch_auc(queue, rows, interactions, model):\n"," n_items = interactions.shape[1]\n"," items = torch.arange(0, n_items).long()\n"," users_init = torch.ones(n_items).long()\n"," for row in rows:\n"," row = int(row)\n"," users = users_init.fill_(row)\n","\n"," preds = model.predict(users, items)\n"," actuals = get_row_indices(row, interactions)\n","\n"," if len(actuals) == 0:\n"," continue\n"," y_test = np.zeros(n_items)\n"," y_test[actuals] = 1\n"," queue.put(roc_auc_score(y_test, preds.data.numpy()))\n","\n","\n","def patk(model, interactions, num_workers=1, k=5):\n"," patks = []\n"," processes = []\n"," n_users = interactions.shape[0]\n"," mp_batch = int(np.ceil(n_users / num_workers))\n","\n"," queue = mp.Queue()\n"," rows = np.arange(n_users)\n"," np.random.shuffle(rows)\n"," for rank in range(num_workers):\n"," start = rank * mp_batch\n"," end = np.min((start + mp_batch, n_users))\n"," p = mp.Process(target=batch_patk,\n"," args=(queue, rows[start:end], interactions, model),\n"," kwargs={'k': k})\n"," p.start()\n"," processes.append(p)\n","\n"," while True:\n"," is_alive = False\n"," for p in processes:\n"," if p.is_alive():\n"," is_alive = True\n"," break\n"," if not is_alive and queue.empty():\n"," break\n","\n"," while not queue.empty():\n"," patks.append(queue.get())\n","\n"," queue.close()\n"," for p in processes:\n"," p.join()\n"," return np.mean(patks)\n","\n","\n","def batch_patk(queue, rows, interactions, model, k=5):\n"," n_items = interactions.shape[1]\n","\n"," items = torch.arange(0, n_items).long()\n"," users_init = torch.ones(n_items).long()\n"," for row in rows:\n"," row = int(row)\n"," users = users_init.fill_(row)\n","\n"," preds = model.predict(users, items)\n"," actuals = get_row_indices(row, interactions)\n","\n"," if len(actuals) == 0:\n"," continue\n","\n"," top_k = np.argpartition(-np.squeeze(preds.data.numpy()), k)\n"," top_k = set(top_k[:k])\n"," true_pids = set(actuals)\n"," if true_pids:\n"," queue.put(len(top_k & true_pids) / float(k))"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Writing metrics.py\n"]}]},{"cell_type":"markdown","metadata":{"id":"7N1Rl15-LJAj"},"source":["### Model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NUlsa6LeLKqu","executionInfo":{"status":"ok","timestamp":1633680758913,"user_tz":-330,"elapsed":511,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"6326b89c-3659-4008-8801-f8d3b991b6fc"},"source":["%%writefile torchmf.py\n","\n","import collections\n","import os\n","\n","import numpy as np\n","from sklearn.metrics import roc_auc_score\n","import torch\n","from torch import nn\n","import torch.multiprocessing as mp\n","import torch.utils.data as data\n","from tqdm import tqdm\n","\n","import metrics\n","\n","\n","# Models\n","# Interactions Dataset => Singular Iter => Singular Loss\n","# Pairwise Datasets => Pairwise Iter => Pairwise Loss\n","# Pairwise Iters\n","# Loss Functions\n","# Optimizers\n","# Metric callbacks\n","\n","# Serve up users, items (and items could be pos_items, neg_items)\n","# In this case, the iteration remains the same. Pass both items into a model\n","# which is a concat of the base model. it handles the pos and neg_items\n","# accordingly. define the loss after.\n","\n","\n","class Interactions(data.Dataset):\n"," \"\"\"\n"," Hold data in the form of an interactions matrix.\n"," Typical use-case is like a ratings matrix:\n"," - Users are the rows\n"," - Items are the columns\n"," - Elements of the matrix are the ratings given by a user for an item.\n"," \"\"\"\n","\n"," def __init__(self, mat):\n"," self.mat = mat.astype(np.float32).tocoo()\n"," self.n_users = self.mat.shape[0]\n"," self.n_items = self.mat.shape[1]\n","\n"," def __getitem__(self, index):\n"," row = self.mat.row[index]\n"," col = self.mat.col[index]\n"," val = self.mat.data[index]\n"," return (row, col), val\n","\n"," def __len__(self):\n"," return self.mat.nnz\n","\n","\n","class PairwiseInteractions(data.Dataset):\n"," \"\"\"\n"," Sample data from an interactions matrix in a pairwise fashion. The row is\n"," treated as the main dimension, and the columns are sampled pairwise.\n"," \"\"\"\n","\n"," def __init__(self, mat):\n"," self.mat = mat.astype(np.float32).tocoo()\n","\n"," self.n_users = self.mat.shape[0]\n"," self.n_items = self.mat.shape[1]\n","\n"," self.mat_csr = self.mat.tocsr()\n"," if not self.mat_csr.has_sorted_indices:\n"," self.mat_csr.sort_indices()\n","\n"," def __getitem__(self, index):\n"," row = self.mat.row[index]\n"," found = False\n","\n"," while not found:\n"," neg_col = np.random.randint(self.n_items)\n"," if self.not_rated(row, neg_col, self.mat_csr.indptr,\n"," self.mat_csr.indices):\n"," found = True\n","\n"," pos_col = self.mat.col[index]\n"," val = self.mat.data[index]\n","\n"," return (row, (pos_col, neg_col)), val\n","\n"," def __len__(self):\n"," return self.mat.nnz\n","\n"," @staticmethod\n"," def not_rated(row, col, indptr, indices):\n"," # similar to use of bsearch in lightfm\n"," start = indptr[row]\n"," end = indptr[row + 1]\n"," searched = np.searchsorted(indices[start:end], col, 'right')\n"," if searched >= (end - start):\n"," # After the array\n"," return False\n"," return col != indices[searched] # Not found\n","\n"," def get_row_indices(self, row):\n"," start = self.mat_csr.indptr[row]\n"," end = self.mat_csr.indptr[row + 1]\n"," return self.mat_csr.indices[start:end]\n","\n","\n","class BaseModule(nn.Module):\n"," \"\"\"\n"," Base module for explicit matrix factorization.\n"," \"\"\"\n"," \n"," def __init__(self,\n"," n_users,\n"," n_items,\n"," n_factors=40,\n"," dropout_p=0,\n"," sparse=False):\n"," \"\"\"\n","\n"," Parameters\n"," ----------\n"," n_users : int\n"," Number of users\n"," n_items : int\n"," Number of items\n"," n_factors : int\n"," Number of latent factors (or embeddings or whatever you want to\n"," call it).\n"," dropout_p : float\n"," p in nn.Dropout module. Probability of dropout.\n"," sparse : bool\n"," Whether or not to treat embeddings as sparse. NOTE: cannot use\n"," weight decay on the optimizer if sparse=True. Also, can only use\n"," Adagrad.\n"," \"\"\"\n"," super(BaseModule, self).__init__()\n"," self.n_users = n_users\n"," self.n_items = n_items\n"," self.n_factors = n_factors\n"," self.user_biases = nn.Embedding(n_users, 1, sparse=sparse)\n"," self.item_biases = nn.Embedding(n_items, 1, sparse=sparse)\n"," self.user_embeddings = nn.Embedding(n_users, n_factors, sparse=sparse)\n"," self.item_embeddings = nn.Embedding(n_items, n_factors, sparse=sparse)\n"," \n"," self.dropout_p = dropout_p\n"," self.dropout = nn.Dropout(p=self.dropout_p)\n","\n"," self.sparse = sparse\n"," \n"," def forward(self, users, items):\n"," \"\"\"\n"," Forward pass through the model. For a single user and item, this\n"," looks like:\n","\n"," user_bias + item_bias + user_embeddings.dot(item_embeddings)\n","\n"," Parameters\n"," ----------\n"," users : np.ndarray\n"," Array of user indices\n"," items : np.ndarray\n"," Array of item indices\n","\n"," Returns\n"," -------\n"," preds : np.ndarray\n"," Predicted ratings.\n","\n"," \"\"\"\n"," ues = self.user_embeddings(users)\n"," uis = self.item_embeddings(items)\n","\n"," preds = self.user_biases(users)\n"," preds += self.item_biases(items)\n"," preds += (self.dropout(ues) * self.dropout(uis)).sum(dim=1, keepdim=True)\n","\n"," return preds.squeeze()\n"," \n"," def __call__(self, *args):\n"," return self.forward(*args)\n","\n"," def predict(self, users, items):\n"," return self.forward(users, items)\n","\n","\n","def bpr_loss(preds, vals):\n"," sig = nn.Sigmoid()\n"," return (1.0 - sig(preds)).pow(2).sum()\n","\n","\n","class BPRModule(nn.Module):\n"," \n"," def __init__(self,\n"," n_users,\n"," n_items,\n"," n_factors=40,\n"," dropout_p=0,\n"," sparse=False,\n"," model=BaseModule):\n"," super(BPRModule, self).__init__()\n","\n"," self.n_users = n_users\n"," self.n_items = n_items\n"," self.n_factors = n_factors\n"," self.dropout_p = dropout_p\n"," self.sparse = sparse\n"," self.pred_model = model(\n"," self.n_users,\n"," self.n_items,\n"," n_factors=n_factors,\n"," dropout_p=dropout_p,\n"," sparse=sparse\n"," )\n","\n"," def forward(self, users, items):\n"," assert isinstance(items, tuple), \\\n"," 'Must pass in items as (pos_items, neg_items)'\n"," # Unpack\n"," (pos_items, neg_items) = items\n"," pos_preds = self.pred_model(users, pos_items)\n"," neg_preds = self.pred_model(users, neg_items)\n"," return pos_preds - neg_preds\n","\n"," def predict(self, users, items):\n"," return self.pred_model(users, items)\n","\n","\n","class BasePipeline:\n"," \"\"\"\n"," Class defining a training pipeline. Instantiates data loaders, model,\n"," and optimizer. Handles training for multiple epochs and keeping track of\n"," train and test loss.\n"," \"\"\"\n","\n"," def __init__(self,\n"," train,\n"," test=None,\n"," model=BaseModule,\n"," n_factors=40,\n"," batch_size=32,\n"," dropout_p=0.02,\n"," sparse=False,\n"," lr=0.01,\n"," weight_decay=0.,\n"," optimizer=torch.optim.Adam,\n"," loss_function=nn.MSELoss(reduction='sum'),\n"," n_epochs=10,\n"," verbose=False,\n"," random_seed=None,\n"," interaction_class=Interactions,\n"," hogwild=False,\n"," num_workers=0,\n"," eval_metrics=None,\n"," k=5):\n"," self.train = train\n"," self.test = test\n","\n"," if hogwild:\n"," num_loader_workers = 0\n"," else:\n"," num_loader_workers = num_workers\n"," self.train_loader = data.DataLoader(\n"," interaction_class(train), batch_size=batch_size, shuffle=True,\n"," num_workers=num_loader_workers)\n"," if self.test is not None:\n"," self.test_loader = data.DataLoader(\n"," interaction_class(test), batch_size=batch_size, shuffle=True,\n"," num_workers=num_loader_workers)\n"," self.num_workers = num_workers\n"," self.n_users = self.train.shape[0]\n"," self.n_items = self.train.shape[1]\n"," self.n_factors = n_factors\n"," self.batch_size = batch_size\n"," self.dropout_p = dropout_p\n"," self.lr = lr\n"," self.weight_decay = weight_decay\n"," self.loss_function = loss_function\n"," self.n_epochs = n_epochs\n"," if sparse:\n"," assert weight_decay == 0.0\n"," self.model = model(self.n_users,\n"," self.n_items,\n"," n_factors=self.n_factors,\n"," dropout_p=self.dropout_p,\n"," sparse=sparse)\n"," self.optimizer = optimizer(self.model.parameters(),\n"," lr=self.lr,\n"," weight_decay=self.weight_decay)\n"," self.warm_start = False\n"," self.losses = collections.defaultdict(list)\n"," self.verbose = verbose\n"," self.hogwild = hogwild\n"," if random_seed is not None:\n"," if self.hogwild:\n"," random_seed += os.getpid()\n"," torch.manual_seed(random_seed)\n"," np.random.seed(random_seed)\n","\n"," if eval_metrics is None:\n"," eval_metrics = []\n"," self.eval_metrics = eval_metrics\n"," self.k = k\n","\n"," def break_grads(self):\n"," for param in self.model.parameters():\n"," # Break gradient sharing\n"," if param.grad is not None:\n"," param.grad.data = param.grad.data.clone()\n","\n"," def fit(self):\n"," for epoch in range(1, self.n_epochs + 1):\n","\n"," if self.hogwild:\n"," self.model.share_memory()\n"," processes = []\n"," train_losses = []\n"," queue = mp.Queue()\n"," for rank in range(self.num_workers):\n"," p = mp.Process(target=self._fit_epoch,\n"," kwargs={'epoch': epoch,\n"," 'queue': queue})\n"," p.start()\n"," processes.append(p)\n"," for p in processes:\n"," p.join()\n","\n"," while True:\n"," is_alive = False\n"," for p in processes:\n"," if p.is_alive():\n"," is_alive = True\n"," break\n"," if not is_alive and queue.empty():\n"," break\n","\n"," while not queue.empty():\n"," train_losses.append(queue.get())\n"," queue.close()\n"," train_loss = np.mean(train_losses)\n"," else:\n"," train_loss = self._fit_epoch(epoch)\n","\n"," self.losses['train'].append(train_loss)\n"," row = 'Epoch: {0:^3} train: {1:^10.5f}'.format(epoch, self.losses['train'][-1])\n"," if self.test is not None:\n"," self.losses['test'].append(self._validation_loss())\n"," row += 'val: {0:^10.5f}'.format(self.losses['test'][-1])\n"," for metric in self.eval_metrics:\n"," func = getattr(metrics, metric)\n"," res = func(self.model, self.test_loader.dataset.mat_csr,\n"," num_workers=self.num_workers)\n"," self.losses['eval-{}'.format(metric)].append(res)\n"," row += 'eval-{0}: {1:^10.5f}'.format(metric, res)\n"," self.losses['epoch'].append(epoch)\n"," if self.verbose:\n"," print(row)\n","\n"," def _fit_epoch(self, epoch=1, queue=None):\n"," if self.hogwild:\n"," self.break_grads()\n","\n"," self.model.train()\n"," total_loss = torch.Tensor([0])\n"," pbar = tqdm(enumerate(self.train_loader),\n"," total=len(self.train_loader),\n"," desc='({0:^3})'.format(epoch))\n"," for batch_idx, ((row, col), val) in pbar:\n"," self.optimizer.zero_grad()\n","\n"," row = row.long()\n"," # TODO: turn this into a collate_fn like the data_loader\n"," if isinstance(col, list):\n"," col = tuple(c.long() for c in col)\n"," else:\n"," col = col.long()\n"," val = val.float()\n","\n"," preds = self.model(row, col)\n"," loss = self.loss_function(preds, val)\n"," loss.backward()\n","\n"," self.optimizer.step()\n","\n"," total_loss += loss.item()\n"," batch_loss = loss.item() / row.size()[0]\n"," pbar.set_postfix(train_loss=batch_loss)\n"," total_loss /= self.train.nnz\n"," if queue is not None:\n"," queue.put(total_loss[0])\n"," else:\n"," return total_loss[0]\n","\n"," def _validation_loss(self):\n"," self.model.eval()\n"," total_loss = torch.Tensor([0])\n"," for batch_idx, ((row, col), val) in enumerate(self.test_loader):\n"," row = row.long()\n"," if isinstance(col, list):\n"," col = tuple(c.long() for c in col)\n"," else:\n"," col = col.long()\n"," val = val.float()\n","\n"," preds = self.model(row, col)\n"," loss = self.loss_function(preds, val)\n"," total_loss += loss.item()\n","\n"," total_loss /= self.test.nnz\n"," return total_loss[0]"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Writing torchmf.py\n"]}]},{"cell_type":"markdown","metadata":{"id":"5KpaDgMwLNI5"},"source":["### Trainer"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HxKI9a2dLDDy","executionInfo":{"status":"ok","timestamp":1633680767265,"user_tz":-330,"elapsed":417,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"538238ee-aa22-49e9-aa44-68eb30d5e92a"},"source":["%%writefile run.py\n","\n","import argparse\n","import pickle\n","\n","import torch\n","\n","from torchmf import (BaseModule, BPRModule, BasePipeline,\n"," bpr_loss, PairwiseInteractions)\n","import utils\n","\n","\n","def explicit():\n"," train, test = utils.get_movielens_train_test_split()\n"," pipeline = BasePipeline(train, test=test, model=BaseModule,\n"," n_factors=10, batch_size=1024, dropout_p=0.02,\n"," lr=0.02, weight_decay=0.1,\n"," optimizer=torch.optim.Adam, n_epochs=40,\n"," verbose=True, random_seed=2017)\n"," pipeline.fit()\n","\n","\n","def implicit():\n"," train, test = utils.get_movielens_train_test_split(implicit=True)\n","\n"," pipeline = BasePipeline(train, test=test, verbose=True,\n"," batch_size=1024, num_workers=4,\n"," n_factors=20, weight_decay=0,\n"," dropout_p=0., lr=.2, sparse=True,\n"," optimizer=torch.optim.SGD, n_epochs=40,\n"," random_seed=2017, loss_function=bpr_loss,\n"," model=BPRModule,\n"," interaction_class=PairwiseInteractions,\n"," eval_metrics=('auc', 'patk'))\n"," pipeline.fit()\n","\n","\n","def hogwild():\n"," train, test = utils.get_movielens_train_test_split(implicit=True)\n","\n"," pipeline = BasePipeline(train, test=test, verbose=True,\n"," batch_size=1024, num_workers=4,\n"," n_factors=20, weight_decay=0,\n"," dropout_p=0., lr=.2, sparse=True,\n"," optimizer=torch.optim.SGD, n_epochs=40,\n"," random_seed=2017, loss_function=bpr_loss,\n"," model=BPRModule, hogwild=True,\n"," interaction_class=PairwiseInteractions,\n"," eval_metrics=('auc', 'patk'))\n"," pipeline.fit()\n","\n","\n","if __name__ == '__main__':\n"," parser = argparse.ArgumentParser(description='torchmf')\n"," parser.add_argument('--example',\n"," help='explicit, implicit, or hogwild')\n"," args = parser.parse_args()\n"," if args.example == 'explicit':\n"," explicit()\n"," elif args.example == 'implicit':\n"," implicit()\n"," elif args.example == 'hogwild':\n"," hogwild()\n"," else:\n"," print('example must be explicit, implicit, or hogwild')"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Writing run.py\n"]}]},{"cell_type":"markdown","metadata":{"id":"40lNybzWLtRP"},"source":["### Explicit Model Training"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0i4BoW9HLHSb","executionInfo":{"status":"ok","timestamp":1633680846688,"user_tz":-330,"elapsed":37600,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b2a2a2bc-c5c8-4c6e-f576-6a68326d5a30"},"source":["!python run.py --example explicit"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Making data path\n","Downloading MovieLens data\n","( 1 ): 100% 89/89 [00:01<00:00, 74.87it/s, train_loss=7.64] \n","Epoch: 1 train: 14.61587 val: 8.83048 \n","( 2 ): 100% 89/89 [00:00<00:00, 112.92it/s, train_loss=2.5]\n","Epoch: 2 train: 4.20514 val: 4.05539 \n","( 3 ): 100% 89/89 [00:00<00:00, 112.48it/s, train_loss=1.57]\n","Epoch: 3 train: 1.86044 val: 2.45105 \n","( 4 ): 100% 89/89 [00:00<00:00, 113.76it/s, train_loss=1.15]\n","Epoch: 4 train: 1.20612 val: 1.82121 \n","( 5 ): 100% 89/89 [00:00<00:00, 110.97it/s, train_loss=0.966]\n","Epoch: 5 train: 0.98724 val: 1.51758 \n","( 6 ): 100% 89/89 [00:00<00:00, 112.02it/s, train_loss=0.89]\n","Epoch: 6 train: 0.89150 val: 1.35180 \n","( 7 ): 100% 89/89 [00:00<00:00, 112.91it/s, train_loss=0.906]\n","Epoch: 7 train: 0.83810 val: 1.25295 \n","( 8 ): 100% 89/89 [00:00<00:00, 108.44it/s, train_loss=0.873]\n","Epoch: 8 train: 0.80769 val: 1.18821 \n","( 9 ): 100% 89/89 [00:00<00:00, 113.67it/s, train_loss=0.83]\n","Epoch: 9 train: 0.78222 val: 1.15017 \n","(10 ): 100% 89/89 [00:00<00:00, 113.89it/s, train_loss=0.777]\n","Epoch: 10 train: 0.76105 val: 1.11414 \n","(11 ): 100% 89/89 [00:00<00:00, 113.23it/s, train_loss=0.73]\n","Epoch: 11 train: 0.74182 val: 1.08541 \n","(12 ): 100% 89/89 [00:00<00:00, 112.17it/s, train_loss=0.64]\n","Epoch: 12 train: 0.72437 val: 1.06774 \n","(13 ): 100% 89/89 [00:00<00:00, 107.08it/s, train_loss=0.733]\n","Epoch: 13 train: 0.70896 val: 1.05505 \n","(14 ): 100% 89/89 [00:00<00:00, 111.47it/s, train_loss=0.702]\n","Epoch: 14 train: 0.69648 val: 1.03989 \n","(15 ): 100% 89/89 [00:00<00:00, 114.82it/s, train_loss=0.69]\n","Epoch: 15 train: 0.68401 val: 1.03105 \n","(16 ): 100% 89/89 [00:00<00:00, 113.08it/s, train_loss=0.772]\n","Epoch: 16 train: 0.67320 val: 1.02541 \n","(17 ): 100% 89/89 [00:00<00:00, 112.42it/s, train_loss=0.624]\n","Epoch: 17 train: 0.66667 val: 1.01918 \n","(18 ): 100% 89/89 [00:00<00:00, 113.76it/s, train_loss=0.671]\n","Epoch: 18 train: 0.65996 val: 1.01878 \n","(19 ): 100% 89/89 [00:00<00:00, 113.38it/s, train_loss=0.667]\n","Epoch: 19 train: 0.65364 val: 1.01307 \n","(20 ): 100% 89/89 [00:00<00:00, 110.37it/s, train_loss=0.745]\n","Epoch: 20 train: 0.64888 val: 1.01569 \n","(21 ): 100% 89/89 [00:00<00:00, 113.61it/s, train_loss=0.671]\n","Epoch: 21 train: 0.64512 val: 1.01603 \n","(22 ): 100% 89/89 [00:00<00:00, 108.82it/s, train_loss=0.623]\n","Epoch: 22 train: 0.64155 val: 1.01564 \n","(23 ): 100% 89/89 [00:00<00:00, 112.19it/s, train_loss=0.677]\n","Epoch: 23 train: 0.63771 val: 1.01452 \n","(24 ): 100% 89/89 [00:00<00:00, 114.23it/s, train_loss=0.739]\n","Epoch: 24 train: 0.63746 val: 1.00893 \n","(25 ): 100% 89/89 [00:00<00:00, 114.42it/s, train_loss=0.766]\n","Epoch: 25 train: 0.63591 val: 1.01990 \n","(26 ): 100% 89/89 [00:00<00:00, 112.95it/s, train_loss=0.586]\n","Epoch: 26 train: 0.63194 val: 1.01370 \n","(27 ): 100% 89/89 [00:00<00:00, 111.70it/s, train_loss=0.734]\n","Epoch: 27 train: 0.63205 val: 1.01533 \n","(28 ): 100% 89/89 [00:00<00:00, 112.88it/s, train_loss=0.733]\n","Epoch: 28 train: 0.63321 val: 1.01158 \n","(29 ): 100% 89/89 [00:00<00:00, 107.37it/s, train_loss=0.645]\n","Epoch: 29 train: 0.63266 val: 1.01819 \n","(30 ): 100% 89/89 [00:00<00:00, 112.43it/s, train_loss=0.683]\n","Epoch: 30 train: 0.63357 val: 1.01789 \n","(31 ): 100% 89/89 [00:00<00:00, 109.35it/s, train_loss=0.7]\n","Epoch: 31 train: 0.63155 val: 1.01247 \n","(32 ): 100% 89/89 [00:00<00:00, 113.49it/s, train_loss=0.68]\n","Epoch: 32 train: 0.63328 val: 1.01842 \n","(33 ): 100% 89/89 [00:00<00:00, 112.89it/s, train_loss=0.68]\n","Epoch: 33 train: 0.63136 val: 1.01667 \n","(34 ): 100% 89/89 [00:00<00:00, 113.90it/s, train_loss=0.752]\n","Epoch: 34 train: 0.63255 val: 1.01864 \n","(35 ): 100% 89/89 [00:00<00:00, 113.94it/s, train_loss=0.716]\n","Epoch: 35 train: 0.63282 val: 1.01362 \n","(36 ): 100% 89/89 [00:00<00:00, 112.76it/s, train_loss=0.618]\n","Epoch: 36 train: 0.63292 val: 1.01480 \n","(37 ): 100% 89/89 [00:00<00:00, 113.63it/s, train_loss=0.666]\n","Epoch: 37 train: 0.63206 val: 1.02341 \n","(38 ): 100% 89/89 [00:00<00:00, 107.43it/s, train_loss=0.652]\n","Epoch: 38 train: 0.63254 val: 1.02066 \n","(39 ): 100% 89/89 [00:00<00:00, 112.98it/s, train_loss=0.65]\n","Epoch: 39 train: 0.63397 val: 1.01905 \n","(40 ): 100% 89/89 [00:00<00:00, 109.15it/s, train_loss=0.732]\n","Epoch: 40 train: 0.63401 val: 1.01783 \n"]}]},{"cell_type":"markdown","metadata":{"id":"JxqWyDE4LxPb"},"source":["### Implicit Model Training"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"IBXdHOUXLdPE","outputId":"62bd667d-56d6-4969-e390-8cab4842353e"},"source":["!python run.py --example implicit"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," cpuset_checked))\n","( 1 ): 100% 46/46 [00:01<00:00, 28.55it/s, train_loss=0.382]\n","Epoch: 1 train: 0.41578 val: 0.39289 eval-auc: 0.55840 eval-patk: 0.00913 \n","( 2 ): 100% 46/46 [00:01<00:00, 28.86it/s, train_loss=0.323]\n","Epoch: 2 train: 0.34652 val: 0.34228 eval-auc: 0.61282 eval-patk: 0.01507 \n","( 3 ): 100% 46/46 [00:01<00:00, 30.01it/s, train_loss=0.273]\n","Epoch: 3 train: 0.27728 val: 0.31357 eval-auc: 0.65768 eval-patk: 0.02215 \n","( 4 ): 100% 46/46 [00:01<00:00, 29.36it/s, train_loss=0.226]\n","Epoch: 4 train: 0.23051 val: 0.29723 eval-auc: 0.69258 eval-patk: 0.02991 \n","( 5 ): 100% 46/46 [00:01<00:00, 29.57it/s, train_loss=0.198]\n","Epoch: 5 train: 0.20115 val: 0.28018 eval-auc: 0.71729 eval-patk: 0.03539 \n","( 6 ): 100% 46/46 [00:01<00:00, 28.66it/s, train_loss=0.152]\n","Epoch: 6 train: 0.17812 val: 0.26524 eval-auc: 0.73440 eval-patk: 0.03607 \n","( 7 ): 100% 46/46 [00:01<00:00, 30.65it/s, train_loss=0.15]\n","Epoch: 7 train: 0.16726 val: 0.25652 eval-auc: 0.74640 eval-patk: 0.03813 \n","( 8 ): 100% 46/46 [00:01<00:00, 29.89it/s, train_loss=0.172]\n","Epoch: 8 train: 0.15538 val: 0.24975 eval-auc: 0.75780 eval-patk: 0.03950 \n","( 9 ): 100% 46/46 [00:01<00:00, 29.88it/s, train_loss=0.133]\n","Epoch: 9 train: 0.14574 val: 0.24520 eval-auc: 0.76651 eval-patk: 0.04498 \n","(10 ): 100% 46/46 [00:01<00:00, 30.02it/s, train_loss=0.14]\n","Epoch: 10 train: 0.13953 val: 0.22739 eval-auc: 0.77529 eval-patk: 0.04749 \n","(11 ): 100% 46/46 [00:01<00:00, 29.70it/s, train_loss=0.151]\n","Epoch: 11 train: 0.13218 val: 0.22872 eval-auc: 0.78306 eval-patk: 0.04749 \n","(12 ): 100% 46/46 [00:01<00:00, 29.81it/s, train_loss=0.13]\n","Epoch: 12 train: 0.12857 val: 0.22756 eval-auc: 0.78880 eval-patk: 0.04840 \n","(13 ): 100% 46/46 [00:01<00:00, 30.26it/s, train_loss=0.13]\n","Epoch: 13 train: 0.12364 val: 0.21565 eval-auc: 0.79382 eval-patk: 0.05114 \n","(14 ): 100% 46/46 [00:01<00:00, 30.80it/s, train_loss=0.0979]\n","Epoch: 14 train: 0.11943 val: 0.21567 eval-auc: 0.79833 eval-patk: 0.05479 \n","(15 ): 100% 46/46 [00:01<00:00, 30.27it/s, train_loss=0.109]\n","Epoch: 15 train: 0.11619 val: 0.21074 eval-auc: 0.80249 eval-patk: 0.05548 \n","(16 ): 100% 46/46 [00:01<00:00, 29.81it/s, train_loss=0.129]\n","Epoch: 16 train: 0.11254 val: 0.21105 eval-auc: 0.80617 eval-patk: 0.05890 \n","(17 ): 100% 46/46 [00:01<00:00, 30.27it/s, train_loss=0.111]\n","Epoch: 17 train: 0.10796 val: 0.20284 eval-auc: 0.80958 eval-patk: 0.05890 \n","(18 ): 100% 46/46 [00:01<00:00, 30.48it/s, train_loss=0.1]\n","Epoch: 18 train: 0.10627 val: 0.19820 eval-auc: 0.81167 eval-patk: 0.06119 \n","(19 ): 100% 46/46 [00:01<00:00, 29.63it/s, train_loss=0.132]\n","Epoch: 19 train: 0.10392 val: 0.20573 eval-auc: 0.81511 eval-patk: 0.06370 \n","(20 ): 100% 46/46 [00:01<00:00, 29.22it/s, train_loss=0.106]\n","Epoch: 20 train: 0.10310 val: 0.20031 eval-auc: 0.81784 eval-patk: 0.06393 \n","(21 ): 100% 46/46 [00:01<00:00, 29.44it/s, train_loss=0.084]\n","Epoch: 21 train: 0.10323 val: 0.19672 eval-auc: 0.82062 eval-patk: 0.06530 \n","(22 ): 100% 46/46 [00:01<00:00, 28.61it/s, train_loss=0.123]\n","Epoch: 22 train: 0.10163 val: 0.19164 eval-auc: 0.82266 eval-patk: 0.06986 \n","(23 ): 100% 46/46 [00:01<00:00, 29.98it/s, train_loss=0.109]\n","Epoch: 23 train: 0.09932 val: 0.18622 eval-auc: 0.82489 eval-patk: 0.06849 \n","(24 ): 100% 46/46 [00:01<00:00, 30.33it/s, train_loss=0.125]\n","Epoch: 24 train: 0.09856 val: 0.18985 eval-auc: 0.82689 eval-patk: 0.06941 \n","(25 ): 100% 46/46 [00:01<00:00, 30.46it/s, train_loss=0.0867]\n","Epoch: 25 train: 0.09591 val: 0.18680 eval-auc: 0.82851 eval-patk: 0.07100 \n","(26 ): 100% 46/46 [00:01<00:00, 29.23it/s, train_loss=0.0945]\n","Epoch: 26 train: 0.09670 val: 0.18181 eval-auc: 0.83038 eval-patk: 0.07009 \n","(27 ): 100% 46/46 [00:01<00:00, 29.79it/s, train_loss=0.0699]\n","Epoch: 27 train: 0.09253 val: 0.18122 eval-auc: 0.83169 eval-patk: 0.06667 \n","(28 ): 100% 46/46 [00:01<00:00, 30.00it/s, train_loss=0.0759]\n","Epoch: 28 train: 0.09226 val: 0.18196 eval-auc: 0.83282 eval-patk: 0.06826 \n","(29 ): 100% 46/46 [00:01<00:00, 29.22it/s, train_loss=0.0822]\n","Epoch: 29 train: 0.09307 val: 0.18249 eval-auc: 0.83441 eval-patk: 0.07648 \n","(30 ): 100% 46/46 [00:01<00:00, 30.18it/s, train_loss=0.114]\n","Epoch: 30 train: 0.09162 val: 0.18411 eval-auc: 0.83504 eval-patk: 0.07648 \n","(31 ): 100% 46/46 [00:01<00:00, 29.39it/s, train_loss=0.086]\n","Epoch: 31 train: 0.08987 val: 0.17815 eval-auc: 0.83631 eval-patk: 0.07374 \n","(32 ): 100% 46/46 [00:01<00:00, 29.27it/s, train_loss=0.0911]\n","Epoch: 32 train: 0.08841 val: 0.18399 eval-auc: 0.83683 eval-patk: 0.07306 \n","(33 ): 100% 46/46 [00:01<00:00, 29.72it/s, train_loss=0.0876]\n","Epoch: 33 train: 0.09061 val: 0.17719 eval-auc: 0.83845 eval-patk: 0.07489 \n","(34 ): 100% 46/46 [00:01<00:00, 29.93it/s, train_loss=0.0647]\n","Epoch: 34 train: 0.08688 val: 0.18095 eval-auc: 0.83955 eval-patk: 0.06918 \n","(35 ): 100% 46/46 [00:01<00:00, 30.05it/s, train_loss=0.0928]\n","Epoch: 35 train: 0.08915 val: 0.17626 eval-auc: 0.84050 eval-patk: 0.07215 \n","(36 ): 100% 46/46 [00:01<00:00, 29.89it/s, train_loss=0.111]\n","Epoch: 36 train: 0.08683 val: 0.17530 eval-auc: 0.84146 eval-patk: 0.07420 \n","(37 ): 100% 46/46 [00:01<00:00, 29.70it/s, train_loss=0.0915]\n","Epoch: 37 train: 0.08663 val: 0.16717 eval-auc: 0.84286 eval-patk: 0.07215 \n","(38 ): 100% 46/46 [00:01<00:00, 29.93it/s, train_loss=0.0765]\n","Epoch: 38 train: 0.08452 val: 0.16749 eval-auc: 0.84417 eval-patk: 0.07763 \n","(39 ): 100% 46/46 [00:01<00:00, 30.19it/s, train_loss=0.0737]\n","Epoch: 39 train: 0.08514 val: 0.16763 eval-auc: 0.84427 eval-patk: 0.07443 \n","(40 ): 100% 46/46 [00:01<00:00, 29.84it/s, train_loss=0.0934]\n","Epoch: 40 train: 0.08454 val: 0.16994 eval-auc: 0.84553 eval-patk: 0.07283 \n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Y2yIPrmu98bL"},"source":["## Hybrid Model with PyTorch on ML-100k"]},{"cell_type":"markdown","metadata":{"id":"kImzuHXJ9_kV"},"source":["Testing out the features of Collie Recs library on MovieLens-100K. Training Factorization and Hybrid models with Pytorch Lightning."]},{"cell_type":"code","metadata":{"id":"P43fS4H27gCt"},"source":["!pip install -q collie_recs\n","!pip install -q git+https://github.com/sparsh-ai/recochef.git"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PD0n8kefAb67"},"source":["import os\n","import joblib\n","import numpy as np\n","import pandas as pd\n","\n","from collie_recs.interactions import Interactions\n","from collie_recs.interactions import ApproximateNegativeSamplingInteractionsDataLoader\n","from collie_recs.cross_validation import stratified_split\n","from collie_recs.metrics import auc, evaluate_in_batches, mapk, mrr\n","from collie_recs.model import CollieTrainer, MatrixFactorizationModel, HybridPretrainedModel\n","from collie_recs.movielens import get_recommendation_visualizations\n","\n","import torch\n","from pytorch_lightning.utilities.seed import seed_everything\n","\n","from recochef.datasets.movielens import MovieLens\n","from recochef.preprocessing.encode import label_encode as le\n","\n","from IPython.display import HTML"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ClWobbREN4VU","executionInfo":{"status":"ok","timestamp":1633680987476,"user_tz":-330,"elapsed":493,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"3f4c1125-a726-444e-d6f5-bc231df2a167"},"source":["# this handy PyTorch Lightning function fixes random seeds across all the libraries used here\n","seed_everything(22)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["Global seed set to 22\n"]},{"output_type":"execute_result","data":{"text/plain":["22"]},"metadata":{},"execution_count":10}]},{"cell_type":"markdown","metadata":{"id":"T0N-kmIrcDZr"},"source":["### Data Loading"]},{"cell_type":"code","metadata":{"id":"LEuHGsYgGv-e"},"source":["data_object = MovieLens()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"g3lLxQE1G120","executionInfo":{"status":"ok","timestamp":1633680994186,"user_tz":-330,"elapsed":578,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"2c6b310f-b237-4f1f-a69e-bee2ac458172"},"source":["df = data_object.load_interactions()\n","df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","0 196 242 3.0 881250949\n","1 186 302 3.0 891717742\n","2 22 377 1.0 878887116\n","3 244 51 2.0 880606923\n","4 166 346 1.0 886397596"]},"metadata":{},"execution_count":12}]},{"cell_type":"markdown","metadata":{"id":"3X8-bE9YcGBg"},"source":["### Preprocessing"]},{"cell_type":"code","metadata":{"id":"tJvifZdrLsGK"},"source":["# drop duplicate user-item pair records, keeping recent ratings only\n","df.drop_duplicates(subset=['USERID','ITEMID'], keep='last', inplace=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"eq7GY0lEINgA"},"source":["# convert the explicit data to implicit by only keeping interactions with a rating ``>= 4``\n","df = df[df.RATING>=4].reset_index(drop=True)\n","df['RATING'] = 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Jo4FnRzSHPzs"},"source":["# label encode\n","df, umap = le(df, col='USERID')\n","df, imap = le(df, col='ITEMID')\n","\n","df = df.astype('int64')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"LA9BtiDrJ_wf","executionInfo":{"status":"ok","timestamp":1633680998415,"user_tz":-330,"elapsed":18,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9044fa8b-53c4-421f-cbfa-8ce6b374d666"},"source":["df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
0001884182806
1111891628467
2221879781125
3331876042340
4441879270459
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","0 0 0 1 884182806\n","1 1 1 1 891628467\n","2 2 2 1 879781125\n","3 3 3 1 876042340\n","4 4 4 1 879270459"]},"metadata":{},"execution_count":16}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"W3hUWib-MyjB","executionInfo":{"status":"ok","timestamp":1633681000375,"user_tz":-330,"elapsed":19,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"148d3ca2-17fb-4622-db52-122f045d1563"},"source":["user_counts = df.groupby(by='USERID')['ITEMID'].count()\n","user_list = user_counts[user_counts>=3].index.tolist()\n","df = df[df.USERID.isin(user_list)]\n","\n","df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
0001884182806
1111891628467
2221879781125
3331876042340
4441879270459
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","0 0 0 1 884182806\n","1 1 1 1 891628467\n","2 2 2 1 879781125\n","3 3 3 1 876042340\n","4 4 4 1 879270459"]},"metadata":{},"execution_count":17}]},{"cell_type":"markdown","metadata":{"id":"37Y7qEEJNiE4"},"source":["### Interactions\n","While we have chosen to represent the data as a ``pandas.DataFrame`` for easy viewing now, Collie uses a custom ``torch.utils.data.Dataset`` called ``Interactions``. This class stores a sparse representation of the data and offers some handy benefits, including: \n","\n","* The ability to index the data with a ``__getitem__`` method \n","* The ability to sample many negative items (we will get to this later!) \n","* Nice quality checks to ensure data is free of errors before model training \n","\n","Instantiating the object is simple! "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dLcqt57TI-6l","executionInfo":{"status":"ok","timestamp":1633681005765,"user_tz":-330,"elapsed":17,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b399e7e8-fec0-460a-8048-e7a1e9395106"},"source":["interactions = Interactions(\n"," users=df['USERID'],\n"," items=df['ITEMID'],\n"," ratings=df['RATING'],\n"," allow_missing_ids=True,\n",")\n","\n","interactions"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Checking for and removing duplicate user, item ID pairs...\n","Checking ``num_negative_samples`` is valid...\n","Maximum number of items a user has interacted with: 378\n","Generating positive items set...\n"]},{"output_type":"execute_result","data":{"text/plain":["Interactions object with 55375 interactions between 942 users and 1447 items, returning 10 negative samples per interaction."]},"metadata":{},"execution_count":18}]},{"cell_type":"markdown","metadata":{"id":"4TaWM_OFNZzn"},"source":["### Data Splits \n","With an ``Interactions`` dataset, Collie supports two types of data splits. \n","\n","1. **Random split**: This code randomly assigns an interaction to a ``train``, ``validation``, or ``test`` dataset. While this is significantly faster to perform than a stratified split, it does not guarantee any balance, meaning a scenario where a user will have no interactions in the ``train`` dataset and all in the ``test`` dataset is possible. \n","2. **Stratified split**: While this code runs slower than a random split, this guarantees that each user will be represented in the ``train``, ``validation``, and ``test`` dataset. This is by far the most fair way to train and evaluate a recommendation model. \n","\n","Since this is a small dataset and we have time, we will go ahead and use ``stratified_split``. If you're short on time, a ``random_split`` can easily be swapped in, since both functions share the same API! "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"U_4IPy2aNLVE","executionInfo":{"status":"ok","timestamp":1633681012293,"user_tz":-330,"elapsed":4939,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"5db1d12d-5d17-46f5-c4d1-05cfe4d458e4"},"source":["train_interactions, val_interactions = stratified_split(interactions, test_p=0.1, seed=42)\n","train_interactions, val_interactions"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Generating positive items set...\n","Generating positive items set...\n"]},{"output_type":"execute_result","data":{"text/plain":["(Interactions object with 49426 interactions between 942 users and 1447 items, returning 10 negative samples per interaction.,\n"," Interactions object with 5949 interactions between 942 users and 1447 items, returning 10 negative samples per interaction.)"]},"metadata":{},"execution_count":19}]},{"cell_type":"markdown","metadata":{"id":"ZO5rLYzH-imu"},"source":["### Model Architecture \n","With our data ready-to-go, we can now start training a recommendation model. While Collie has several model architectures built-in, the simplest by far is the ``MatrixFactorizationModel``, which use ``torch.nn.Embedding`` layers and a dot-product operation to perform matrix factorization via collaborative filtering."]},{"cell_type":"markdown","metadata":{"id":"ZWi4VUjeghUA"},"source":["Digging through the code of [``collie_recs.model.MatrixFactorizationModel``](../collie_recs/model.py) shows the architecture is as simple as we might think. For simplicity, we will include relevant portions below so we know exactly what we are building: \n","\n","````python\n","def _setup_model(self, **kwargs) -> None:\n"," self.user_biases = ZeroEmbedding(num_embeddings=self.hparams.num_users,\n"," embedding_dim=1,\n"," sparse=self.hparams.sparse)\n"," self.item_biases = ZeroEmbedding(num_embeddings=self.hparams.num_items,\n"," embedding_dim=1,\n"," sparse=self.hparams.sparse)\n"," self.user_embeddings = ScaledEmbedding(num_embeddings=self.hparams.num_users,\n"," embedding_dim=self.hparams.embedding_dim,\n"," sparse=self.hparams.sparse)\n"," self.item_embeddings = ScaledEmbedding(num_embeddings=self.hparams.num_items,\n"," embedding_dim=self.hparams.embedding_dim,\n"," sparse=self.hparams.sparse)\n","\n"," \n","def forward(self, users: torch.tensor, items: torch.tensor) -> torch.tensor:\n"," user_embeddings = self.user_embeddings(users)\n"," item_embeddings = self.item_embeddings(items)\n","\n"," preds = (\n"," torch.mul(user_embeddings, item_embeddings).sum(axis=1)\n"," + self.user_biases(users).squeeze(1)\n"," + self.item_biases(items).squeeze(1)\n"," )\n","\n"," if self.hparams.y_range is not None:\n"," preds = (\n"," torch.sigmoid(preds)\n"," * (self.hparams.y_range[1] - self.hparams.y_range[0])\n"," + self.hparams.y_range[0]\n"," )\n","\n"," return preds\n","````\n","\n","Let's go ahead and instantiate the model and start training! Note that even if you are running this model on a CPU instead of a GPU, this will still be relatively quick to fully train. "]},{"cell_type":"markdown","metadata":{"id":"L7o3t9vNN-Lt"},"source":["Collie is built with PyTorch Lightning, so all the model classes and the ``CollieTrainer`` class accept all the training options available in PyTorch Lightning. Here, we're going to set the embedding dimension and learning rate differently, and go with the defaults for everything else"]},{"cell_type":"code","metadata":{"id":"LNfxzlruN1xx"},"source":["model = MatrixFactorizationModel(\n"," train=train_interactions,\n"," val=val_interactions,\n"," embedding_dim=10,\n"," lr=1e-2,\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TKxeyMsMN1vg"},"source":["trainer = CollieTrainer(model, max_epochs=10, deterministic=True)\n","\n","trainer.fit(model)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"dx8EoEhC_Cjh"},"source":["```text\n","GPU available: False, used: False\n","TPU available: False, using: 0 TPU cores\n","IPU available: False, using: 0 IPUs\n","\n"," | Name | Type | Params\n","----------------------------------------------------\n","0 | user_biases | ZeroEmbedding | 942 \n","1 | item_biases | ZeroEmbedding | 1.4 K \n","2 | user_embeddings | ScaledEmbedding | 9.4 K \n","3 | item_embeddings | ScaledEmbedding | 14.5 K\n","4 | dropout | Dropout | 0 \n","----------------------------------------------------\n","26.3 K Trainable params\n","0 Non-trainable params\n","26.3 K Total params\n","0.105 Total estimated model params size (MB)\n","Validation sanity check: 0%\n","0/2 [00:00User 895:\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Willy Wonka and the Chocolate Factory (1971)Mighty Aphrodite (1995)Conspiracy Theory (1997)Sense and Sensibility (1995)Liar Liar (1997)In & Out (1997)Return of the Jedi (1983)Ransom (1996)Emma (1996)Toy Story (1995)
Some loved films:
\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Princess Bride, The (1987)Graduate, The (1967)Cold Comfort Farm (1995)Apartment, The (1960)Jerry Maguire (1996)Sleeper (1973)Independence Day (ID4) (1996)Desperado (1995)Three Colors: Red (1994)Lawnmower Man, The (1992)
Recommended films:
-----

User 895 has rated 12 films with a 4 or 5

User 895 has rated 8 films with a 1, 2, or 3

% of these films rated 5 or 4 appearing in the first 10 recommendations:0.0%

% of these films rated 1, 2, or 3 appearing in the first 10 recommendations: 0.0%

"],"text/plain":[""]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"DbZ9ufGvghUE"},"source":["### Save and Load a Standard Model "]},{"cell_type":"code","metadata":{"id":"WqJbXHXgghUG"},"source":["# we can save the model with...\n","os.makedirs('models', exist_ok=True)\n","model.save_model('models/matrix_factorization_model.pth')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Dz_8miLPghUG","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1633681118093,"user_tz":-330,"elapsed":19,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"1f2f8c07-be29-4fb9-ca8c-168ac1515f16"},"source":["# ... and if we wanted to load that model back in, we can do that easily...\n","model_loaded_in = MatrixFactorizationModel(load_model_path='models/matrix_factorization_model.pth')\n","\n","model_loaded_in"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["MatrixFactorizationModel(\n"," (user_biases): ZeroEmbedding(942, 1)\n"," (item_biases): ZeroEmbedding(1447, 1)\n"," (user_embeddings): ScaledEmbedding(942, 10)\n"," (item_embeddings): ScaledEmbedding(1447, 10)\n"," (dropout): Dropout(p=0.0, inplace=False)\n",")"]},"metadata":{},"execution_count":25}]},{"cell_type":"markdown","metadata":{"id":"cQpWlCuughUH"},"source":["Now that we've built our first model and gotten some baseline metrics, we now will be looking at some more advanced features in Collie's ``MatrixFactorizationModel``. "]},{"cell_type":"markdown","metadata":{"id":"Q3Ne8ETsgzLe"},"source":["### Faster Data Loading Through Approximate Negative Sampling "]},{"cell_type":"markdown","metadata":{"id":"8nBu6PZhgzLe"},"source":["With sufficiently large enough data, verifying that each negative sample is one a user has *not* interacted with becomes expensive. With many items, this can soon become a bottleneck in the training process. \n","\n","Yet, when we have many items, the chances a user has interacted with most is increasingly rare. Say we have ``1,000,000`` items and we want to sample ``10`` negative items for a user that has positively interacted with ``200`` items. The chance that we accidentally select a positive item in a random sample of ``10`` items is just ``0.2%``. At that point, it might be worth it to forgo the expensive check to assert our negative sample is true, and instead just randomly sample negative items with the hope that most of the time, they will happen to be negative. \n","\n","This is the theory behind the ``ApproximateNegativeSamplingInteractionsDataLoader``, an alternate DataLoader built into Collie. Let's train a model with this below, noting how similar this procedure looks to that in the previous tutorial. "]},{"cell_type":"code","metadata":{"id":"MgSijz04gzLf"},"source":["train_loader = ApproximateNegativeSamplingInteractionsDataLoader(train_interactions, batch_size=1024, shuffle=True)\n","val_loader = ApproximateNegativeSamplingInteractionsDataLoader(val_interactions, batch_size=1024, shuffle=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"n9wQLd9-gzLf"},"source":["model = MatrixFactorizationModel(\n"," train=train_loader,\n"," val=val_loader,\n"," embedding_dim=10,\n"," lr=1e-2,\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AbYirCSNgzLg"},"source":["trainer = CollieTrainer(model, max_epochs=10, deterministic=True)\n","\n","trainer.fit(model)\n","model.eval()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"uvykQiW2_Oof"},"source":["```text\n","GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","----------------------------------------------------\n","0 | user_biases | ZeroEmbedding | 941 \n","1 | item_biases | ZeroEmbedding | 1.4 K \n","2 | user_embeddings | ScaledEmbedding | 9.4 K \n","3 | item_embeddings | ScaledEmbedding | 14.5 K\n","4 | dropout | Dropout | 0 \n","----------------------------------------------------\n","26.3 K Trainable params\n","0 Non-trainable params\n","26.3 K Total params\n","0.105 Total estimated model params size (MB)\n","Detected GPU. Setting ``gpus`` to 1.\n","Global seed set to 22\n","\n","MatrixFactorizationModel(\n"," (user_biases): ZeroEmbedding(941, 1)\n"," (item_biases): ZeroEmbedding(1447, 1)\n"," (user_embeddings): ScaledEmbedding(941, 10)\n"," (item_embeddings): ScaledEmbedding(1447, 10)\n"," (dropout): Dropout(p=0.0, inplace=False)\n",")\n","```"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":117,"referenced_widgets":["d7c34c7be74246acb1a7da702b490029","8ca15bf2e3be4ba88b7dbbf4ed26820d","3e63ae601740455e81c35d4fbab2db6e","b289ad3cdd3f4a25a9e92ed22c37aeed","3d7d55e46c7d4e2caa6680229fe73b4e","e841d95562314124b242c2e4225afbdb","7b9d13b53df04e2082d4e236f50b807d","9a4137c369cc4f26817ed3ab568a5ef7"]},"id":"xLKDSq-hgzLg","outputId":"44183d74-a567-418d-a85e-946bf1443713"},"source":["mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n","\n","print(f'MAP@10 Score: {mapk_score}')\n","print(f'MRR Score: {mrr_score}')\n","print(f'AUC Score: {auc_score}')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d7c34c7be74246acb1a7da702b490029","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","MAP@10 Score: 0.027979833367276323\n","MRR Score: 0.1703751336709069\n","AUC Score: 0.8517987786322347\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"qipOCQzxgzLh"},"source":["We're seeing a small hit on performance and only a marginal improvement in training time compared to the standard ``MatrixFactorizationModel`` model because MovieLens 100K has so few items. ``ApproximateNegativeSamplingInteractionsDataLoader`` is especially recommended for when we have more items in our data and training times need to be optimized. \n","\n","For more details on this and other DataLoaders in Collie (including those for out-of-memory datasets), check out the [docs](https://collie.readthedocs.io/en/latest/index.html)! "]},{"cell_type":"markdown","metadata":{"id":"iGgM-FaegzLi"},"source":["### Multiple Optimizers "]},{"cell_type":"markdown","metadata":{"id":"4xcFspsbgzLi"},"source":["Training recommendation models at ShopRunner, we have encountered something we call \"the curse of popularity.\" \n","\n","This is best thought of in the viewpoint of a model optimizer - say we have a user, a positive item, and several negative items that we hope have recommendation scores that score lower than the positive item. As an optimizer, you can either optimize every single embedding dimension (hundreds of parameters) to achieve this, or instead choose to score a quick win by optimizing the bias terms for the items (just add a positive constant to the positive item and a negative constant to each negative item). \n","\n","While we clearly want to have varied embedding layers that reflect each user and item's taste profiles, some models learn to settle for popularity as a recommendation score proxy by over-optimizing the bias terms, essentially just returning the same set of recommendations for every user. Worst of all, since popular items are... well, popular, **the loss of this model will actually be decent, solidifying the model getting stuck in a local loss minima**. \n","\n","To counteract this, Collie supports multiple optimizers in a ``MatrixFactorizationModel``. With this, we can have a faster optimizer work to optimize the embedding layers for users and items, and a slower optimizer work to optimize the bias terms. With this, we impel the model to do the work actually coming up with varied, personalized recommendations for users while still taking into account the necessity of the bias (popularity) terms on recommendations. \n","\n","At ShopRunner, we have seen significantly better metrics and results from this type of model. With Collie, this is simple to do, as shown below. "]},{"cell_type":"code","metadata":{"id":"GxUMsr61gzLj"},"source":["model = MatrixFactorizationModel(\n"," train=train_interactions,\n"," val=val_interactions,\n"," embedding_dim=10,\n"," lr=1e-2,\n"," bias_lr=1e-1,\n"," optimizer='adam',\n"," bias_optimizer='sgd',\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":557,"referenced_widgets":["49f97e2e5f55454bab313b1570b4b9c4","b3c7b293fe8b44e0920b060f00382401","80b4510b6a7e4d77879516c59662180c","f1e818c6ef934704ae0ef1f80eaa7b5b","695c6402e0a1475eb2965bee589c2bf4","06f180111dfe4f57a12723bccb253f98","316eaa17f9cd4558b8c7f6951b98d6f0","75fc6c2f339b4e47bb22790d37e7ebab","8d714e45f0d64e85b637961f4a25f3d0","b5a4c2dd499c4d5c8d3ab2e890b44642","d1dd8920ddfc4c97beffe89ece6b982e","fc7d1f4bea4145af93d9059177a477fd","ef66bb3e20eb4d25a6dc3af0fccf2e2a","9275275798974477a5f5858880d1b429","2bcffbf7f05b41688936d51d6c3ffcac","28198fbab991465583818c47d8ddbb6d","83658275a6c346d7822fd1368345e09a","3389371180f64e088e7c6549a25b3ee1","54d47349fff54cd0bca2e4903b06e5d8","ec37483292aa4715a97c6f0de133cb24","78845118227b40adac2503e57e1f38a7","ce20fc80baf0494f976d7ebb54303833","b0606438031a4d0fbe1b3b1bc4d49e9e","38cb1183eb294363af2d2762d8a49a13","b9a2d6afed7641d995fba91c5a94f8ec","6f42c45ff32f46deb83569801ab976c8","58d6593d8810421bb8bd39099e82feb9","7cdc87e9c27f48e2b081bdbf2d6228e8","45b86b0dc3394da4b2f87191d23dfed7","585898b88c044e2e8d6fd0c46c3b575d","a0259f8bb2264993a6d27acd097b6c05","5b0ed212d376472987768ea06e314f9d","a59ea6641fca44bbaa553a1624deab51","671b80df47dc444d817c51101c7adf49","d56ba50fc09c4502a1055ff9a3199062","1bd1fa4ccb564778be34c5fe9036e731","1b3ff709422c4864acff0ca42ec89e07","5b6f8c0d04be4ec19c55259caf622c02","b8903c665b7c4a3695101aa7d6e8f929","518b4b044b4d411fb43dac5fc993c02c","36d600f8e9be46a6b05ec580025aac20","f14194bdbf724d20bdd7d5b572374608","b62d3801dd3d4f45b53de6d6c25e26bd","066ce24dcd2e46acb54f1fd437963b85","bd80f8f37cb247c69f969d96b146ea8c","4100968fe609430f8cd1f73c48a356f2","9e6c6c48df824c548aa0425888adac9b","27866404913f4820bb46d1641f63dc1f","d75b9faa61b242279706a77458e55276","ba792412f171494d937051219e01607d","f43bba18ba134b269006e100f8de55d1","63c6fb5bfa3147a89962641d1a7d215b","6d7a19d43c4846d99d3d470fd5d7d66e","90af7f79f905435f8c81ca21e59cab59","fad0632c95f74aca92b6e72751da0632","7eb49fb403564170b47e0c6f867f81e1","57322a343ddd4ad2bc408a840eca0e98","18b3142ac5c24401941cd4f79bac783b","6d730313b939447bbd396526cdee3fad","73cccc23b004406e8521b0cc26401c9c","7520241c3b2c4591bc0532d20dafccda","5dd8ebbab9ce418cae4eaf9f568ab5c9","0afd2fe607b04590813dc297c3bfc4da","c14405b6297a435eaab8032dbbb9966f","0351e40f8c0f46708aeea7233565800a","d7e152362cd54b4380a6cd3e506e7b86","73dff796a59d444a96016b2578f277d7","89a5fbcb3a4e4732a72c8cee79a346b6","bcde36e5444444568f7a6fa34d85ccaa","53e3249bbef446208f54a6216420062f","235ef9871d5443a9ab3b31f1c231f53d","b329742fdb3e44e2974d3a31700072a3","93fbdf1f9df1464888cdecc6285ef575","7ef0973e658740528843af596067fe8e","dad5c7f76e20437ca651d4d39ac68660","3a61c8ddd8b74af78c6ceffb7001da02","fec250de7d6f45688eb48b51d837b53a","54d415a20e204d53bbe9baddd4598e2f","136520e97ebe4b04b9b46a44e46297fc","c4d1f1810116465b9dcb9282eed0f113","2c8df37c614447d5b5064dbbaa837081","dc9b62bd3eb94dc8afd9c21491faff0b","2416bc3ad43149ff9e8d10572693f115","93054dc57a344b9daae9e6e36e14b594","4d9cf1e48cb74afb91410b46b2d601b5","897727bf519648e0bc3f204771ded957","795429863893411cab25dfe5771a2e4a","93a527dd2045487393fb8280ff3945b1","3d3ca53ac0cb41d1bbb58aa754a79619","600e766c4f4c4136875db902954f9ac7","2cf5bd88c0c74729a89a70f6ba9dd02e","a247748059184bbaac041b8fffd99e3f","010c2bf6e2af470d950542394f8ceef5","5bb0a358fcd64106acdac950f82d12a7","16202361258644f6ab7b168990f73136","c2fffba8678b48f9957b9a581aae9e2e"]},"id":"dQ_tTRfOgzLj","outputId":"4d3af437-c62d-4b9b-ddb5-eaf3731556da"},"source":["trainer = CollieTrainer(model, max_epochs=10, deterministic=True)\n","\n","trainer.fit(model)\n","model.eval()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","----------------------------------------------------\n","0 | user_biases | ZeroEmbedding | 941 \n","1 | item_biases | ZeroEmbedding | 1.4 K \n","2 | user_embeddings | ScaledEmbedding | 9.4 K \n","3 | item_embeddings | ScaledEmbedding | 14.5 K\n","4 | dropout | Dropout | 0 \n","----------------------------------------------------\n","26.3 K Trainable params\n","0 Non-trainable params\n","26.3 K Total params\n","0.105 Total estimated model params size (MB)\n"],"name":"stderr"},{"output_type":"stream","text":["Detected GPU. Setting ``gpus`` to 1.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"49f97e2e5f55454bab313b1570b4b9c4","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Global seed set to 22\n"],"name":"stderr"},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8d714e45f0d64e85b637961f4a25f3d0","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"83658275a6c346d7822fd1368345e09a","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b9a2d6afed7641d995fba91c5a94f8ec","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a59ea6641fca44bbaa553a1624deab51","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n","Epoch 3: reducing learning rate of group 0 to 1.0000e-02.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"36d600f8e9be46a6b05ec580025aac20","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d75b9faa61b242279706a77458e55276","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"57322a343ddd4ad2bc408a840eca0e98","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0351e40f8c0f46708aeea7233565800a","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"93fbdf1f9df1464888cdecc6285ef575","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2c8df37c614447d5b5064dbbaa837081","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3d3ca53ac0cb41d1bbb58aa754a79619","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["MatrixFactorizationModel(\n"," (user_biases): ZeroEmbedding(941, 1)\n"," (item_biases): ZeroEmbedding(1447, 1)\n"," (user_embeddings): ScaledEmbedding(941, 10)\n"," (item_embeddings): ScaledEmbedding(1447, 10)\n"," (dropout): Dropout(p=0.0, inplace=False)\n",")"]},"metadata":{"tags":[]},"execution_count":55}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":117,"referenced_widgets":["5c2218b13d6345ff91c4d8980b2b2ca0","309d24f05b1b4bc5b7d0148795fd5cf4","ccfb5dc3eef14f1f8324bc48b3dc0e7f","a84e5372c5c24417a08487dd3efcebe8","c28c07424b7f45088e73ac25f2bb712a","5d01bda6f6354f25bb6dcb15ef4596a0","758cba83e9414f60bf87799a71b3cd7f","48d71d4847be4a25a237568371ae4493"]},"id":"ENZSOd1DgzLk","outputId":"7b1844fb-b81a-43cb-8a98-5206b87b4506"},"source":["mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n","\n","print(f'MAP@10 Score: {mapk_score}')\n","print(f'MRR Score: {mrr_score}')\n","print(f'AUC Score: {auc_score}')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"5c2218b13d6345ff91c4d8980b2b2ca0","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","MAP@10 Score: 0.03243186201880122\n","MRR Score: 0.19819369246580287\n","AUC Score: 0.8617710409716284\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"blXcVpg3gzLk"},"source":["Again, we're not seeing as much performance increase here compared to the standard model because MovieLens 100K has so few items. For a more dramatic difference, try training this model on a larger dataset, such as MovieLens 10M, adjusting the architecture-specific hyperparameters, or train longer. "]},{"cell_type":"markdown","metadata":{"id":"1Dt0IsWJgzLk"},"source":["### Item-Item Similarity "]},{"cell_type":"markdown","metadata":{"id":"sqQJpyKVgzLl"},"source":["While we've trained every model thus far to work for member-item recommendations (given a *member*, recommend *items* - think of this best as \"Personalized recommendations for you\"), we also have access to item-item recommendations for free (given a seed *item*, recommend similar *items* - think of this more like \"People who interacted with this item also interacted with...\"). \n","\n","With Collie, accessing this is simple! "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":343},"id":"RRMouFweUOhw","outputId":"02e6281a-f64e-43c4-a151-84601d91ab50"},"source":["df_item = data_object.load_items()\n","df_item.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
01Toy Story (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Toy%20Story%2...0001110000000000000
12GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
23Four Rooms (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Four%20Rooms%...0000000000000000100
34Get Shorty (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Get%20Shorty%...0100010010000000000
45Copycat (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Copycat%20(1995)0000001010000000100
\n","
"],"text/plain":[" ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n","0 1 Toy Story (1995) 01-Jan-1995 ... 0 0 0\n","1 2 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n","2 3 Four Rooms (1995) 01-Jan-1995 ... 1 0 0\n","3 4 Get Shorty (1995) 01-Jan-1995 ... 0 0 0\n","4 5 Copycat (1995) 01-Jan-1995 ... 1 0 0\n","\n","[5 rows x 24 columns]"]},"metadata":{"tags":[]},"execution_count":68}]},{"cell_type":"code","metadata":{"id":"Ycl8PcLIgzLl","colab":{"base_uri":"https://localhost:8080/","height":343},"outputId":"27f2b6aa-9b6f-4fe4-9d8f-3637cadd4bbe"},"source":["df_item = le(df_item, col='ITEMID', maps=imap)\n","df_item.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
09.0Toy Story (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Toy%20Story%2...0001110000000000000
1160.0GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
2579.0Four Rooms (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Four%20Rooms%...0000000000000000100
325.0Get Shorty (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Get%20Shorty%...0100010010000000000
4436.0Copycat (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Copycat%20(1995)0000001010000000100
\n","
"],"text/plain":[" ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n","0 9.0 Toy Story (1995) 01-Jan-1995 ... 0 0 0\n","1 160.0 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n","2 579.0 Four Rooms (1995) 01-Jan-1995 ... 1 0 0\n","3 25.0 Get Shorty (1995) 01-Jan-1995 ... 0 0 0\n","4 436.0 Copycat (1995) 01-Jan-1995 ... 1 0 0\n","\n","[5 rows x 24 columns]"]},"metadata":{"tags":[]},"execution_count":69}]},{"cell_type":"code","metadata":{"id":"TvtbOrbtgzLl","colab":{"base_uri":"https://localhost:8080/","height":117},"outputId":"67ef757a-91e9-4b32-c2ac-5f43c4742634"},"source":["df_item.loc[df_item['TITLE'] == 'GoldenEye (1995)']"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
1160.0GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
\n","
"],"text/plain":[" ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n","1 160.0 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n","\n","[1 rows x 24 columns]"]},"metadata":{"tags":[]},"execution_count":70}]},{"cell_type":"code","metadata":{"id":"Uom6EVG8gzLm","colab":{"base_uri":"https://localhost:8080/"},"outputId":"a4f6c8a4-c472-48c2-f240-14763b5d1d8e"},"source":["# let's start by finding movies similar to GoldenEye (1995)\n","item_similarities = model.item_item_similarity(item_id=160)\n","\n","item_similarities"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["160 1.000000\n","123 0.842003\n","948 0.828162\n","398 0.827030\n","197 0.826931\n"," ... \n","26 -0.654127\n","88 -0.680429\n","165 -0.697536\n","499 -0.729313\n","312 -0.780792\n","Length: 1447, dtype: float64"]},"metadata":{"tags":[]},"execution_count":71}]},{"cell_type":"code","metadata":{"id":"MW741iOcgzLm","colab":{"base_uri":"https://localhost:8080/","height":428},"outputId":"ad4b1617-bc85-4698-ebf6-65b70161e7ce"},"source":["df_item.iloc[item_similarities.index][:5]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
162182.0Return of the Pink Panther, The (1974)01-Jan-1974NaNhttp://us.imdb.com/M/title-exact?Return%20of%2...0000010000000000000
125229.0Spitfire Grill, The (1996)06-Sep-1996NaNhttp://us.imdb.com/M/title-exact?Spitfire%20Gr...0000000010000000000
975933.0Solo (1996)23-Aug-1996NaNhttp://us.imdb.com/M/title-exact?Solo%20(1996)0100000000000001100
401175.0Ghost (1990)01-Jan-1990NaNhttp://us.imdb.com/M/title-exact?Ghost%20(1990)0000010000000010100
19972.0Shining, The (1980)01-Jan-1980NaNhttp://us.imdb.com/M/title-exact?Shining,%20Th...0000000000010000000
\n","
"],"text/plain":[" ITEMID TITLE ... WAR WESTERN\n","162 182.0 Return of the Pink Panther, The (1974) ... 0 0\n","125 229.0 Spitfire Grill, The (1996) ... 0 0\n","975 933.0 Solo (1996) ... 0 0\n","401 175.0 Ghost (1990) ... 0 0\n","199 72.0 Shining, The (1980) ... 0 0\n","\n","[5 rows x 24 columns]"]},"metadata":{"tags":[]},"execution_count":72}]},{"cell_type":"markdown","metadata":{"id":"_py39oQ8gzLm"},"source":["Unfortunately, not seen these movies. Can't say if these are relevant.\n","\n","``item_item_similarity`` method is available in all Collie models, not just ``MatrixFactorizationModel``! \n","\n","Next, we will incorporate item metadata into recommendations for even better results."]},{"cell_type":"markdown","metadata":{"id":"8hkoWyfVg9AK"},"source":["### Partial Credit Loss\n","Most of the time, we don't *only* have user-item interactions, but also side-data about our items that we are recommending. These next two notebooks will focus on incorporating this into the model training process. \n","\n","In this notebook, we're going to add a new component to our loss function - \"partial credit\". Specifically, we're going to use the genre information to give our model \"partial credit\" for predicting that a user would like a movie that they haven't interacted with, but is in the same genre as one that they liked. The goal is to help our model learn faster from these similarities. "]},{"cell_type":"markdown","metadata":{"id":"4iFhjr7eg9AK"},"source":["### Read in Data"]},{"cell_type":"markdown","metadata":{"id":"bK4bGSUEWe9F"},"source":["To do the partial credit calculation, we need this data in a slightly different form. Instead of the one-hot-encoded version above, we're going to make a ``1 x n_items`` tensor with a number representing the first genre associated with the film, for simplicity. Note that with Collie, we could instead make a metadata tensor for each genre"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mi7IZRHbWwsc","outputId":"20906809-3682-4bee-eee3-1d98fe9b03ca"},"source":["df_item.columns[5:]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Index(['UNKNOWN', 'ACTION', 'ADVENTURE', 'ANIMATION', 'CHILDREN', 'COMEDY',\n"," 'CRIME', 'DOCUMENTARY', 'DRAMA', 'FANTASY', 'FILMNOIR', 'HORROR',\n"," 'MUSICAL', 'MYSTERY', 'ROMANCE', 'SCIFI', 'THRILLER', 'WAR', 'WESTERN'],\n"," dtype='object')"]},"metadata":{"tags":[]},"execution_count":76}]},{"cell_type":"code","metadata":{"id":"bWBxAUUXYvZ3"},"source":["metadata_df = df_item[df_item.columns[5:]]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"LSy_-Jsxg9AL","colab":{"base_uri":"https://localhost:8080/"},"outputId":"a1d9aa32-d10c-454a-dc7b-9cfcda410071"},"source":["genres = (\n"," torch.tensor(metadata_df.values)\n"," .topk(1)\n"," .indices\n"," .view(-1)\n",")\n","\n","genres"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([ 5, 1, 16, ..., 14, 5, 8])"]},"metadata":{"tags":[]},"execution_count":77}]},{"cell_type":"markdown","metadata":{"id":"LhaQLQQig9AM"},"source":["### Train a model with our new loss"]},{"cell_type":"markdown","metadata":{"id":"5NrYwTFVXCco"},"source":["now, we will pass in ``metadata_for_loss`` and ``metadata_for_loss_weights`` into the model ``metadata_for_loss`` should have a tensor containing the integer representations for metadata we created above for every item ID in our dataset ``metadata_for_loss_weights`` should have the weights for each of the keys in ``metadata_for_loss``"]},{"cell_type":"code","metadata":{"id":"Sysr04kSg9AN"},"source":["model = MatrixFactorizationModel(\n"," train=train_interactions,\n"," val=val_interactions,\n"," embedding_dim=10,\n"," lr=1e-2,\n"," metadata_for_loss={'genre': genres},\n"," metadata_for_loss_weights={'genre': 0.4},\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":472,"referenced_widgets":["7f9d29d6c1f04d4b9adcda6292a698fb","4d95aa15fb8e45168224c12f033db89e","567edd5b84854cc3ba9d8b34702716ce","3ccc6ebee63d473aba28adc868b5b5c0","5db3013f6fca47428b7f05d89d611441","650154b034e843648fdd31198e6b5c8b","b85c146244634db5a8ec5bb8c37a6a3f","71e7704f60a44170988aed05dc8a4788","71fb17475171409ab7b30f0915d3c3f9","ec82867f02de401faf33bdeb69a1bf2e","4ffc6e6cdbbb47239b7da5a07f16b5fb","cce5557f779742e3932bb8ef2016a17c","668011e6db324325974aa6a6b1fdc782","ebcdd1df171847a38f7655716f2d6061","9d11e361b37a4849a83531e249455615","30fefc2060004b2bb028ed54ea9383b6","85b96bc221984d08815cf3c09fde7c9f","1abf2c736a91486eb8621d1cd679f9b2","89b238d17aad4ef389c20de16ed1e55c","70d5a3c3cf41478b83813ec40298cd22","bdf376ea2f8e4b9e9d495a1fe44b2b59","c6213cfdba3d4228aad387361e062fc7","a28aa408fe444c179080e2ef9433a877","6e3313408a184dcca85da1f7514d8bfe","b6f3004c83574b379bc3520a410b5df5","57e0d47b70b744898e4e548c28d27095","d5f9ad23ba314382a2ffa29ecc311b5c","75bcb22fb4514a288fdd43cdb2084b11","d7bf1b026d10490fa70221af127986e5","bef2ed39c203462a80a1f217adfc301b","1038bcbdb10a4da08c7b2ffbc08c7e81","2a435432b9914e3faf2d49a0645a9fd5","127ad96af11c41519ded336bbc246c42","b5fe20560cc64837af9e13936fd702d3","2260ec6d57724ecca381bf65a151e97f","add9b2d1bfeb4b3baaa97d39ca4eb8e3","2baf39d33c2a4f28bfd557ba7b9f8c78","bdf0ea6c2c8042d68c50cd5e19b1c15b","143ecbd32aa14efeb1d2eae6873609ce","b4eee80b60f748b4886ede6073b37a7c","1997f9f7a4d04e5ebec4b8907e32b797","21380c3a63df4a7592b0e580535603df","0bd610aa3a55452dae1e3fbc721cb87a","07e2308d6aff45c5bd0ad77e246e9981","2354778e0ce6427385e2e107ea30332b","717568c66bc444ddb60b415c5169588c","13a5b3039ac04a2dbd6049992f822c89","96773fc1d61d4b9faf8f8e7f0d0c3786","2bb338780a1a40318d91fc689692c9ae","869cd72009244cd997fc6b8bca19cd53","8f341ddecce34a2e9967972e901123b6","4f53235cc013444c8a71141df07ccc4b","d76f878252a54a79bd98b1e093285964","69b2d3fe2f5d4f73ba34142ccea3dbf8","818427165e3c40d6b9c60f5fc26ea0db","a62b8b75b5bb4b1a9a35a98b9f87edc0","b066cece2abe4a74b1aed6c238cd82b3","5a5b267987ed4ad0a143744fe30f27b1","2666a159d7f1406887a6bff3efdd1dd6","7b3b88973bc74728a3ea7eb04a122371","7e684d1ba34a49e1b108909220e0a487","a96a161717124b54ba9473f5d4476177","2137633428de4deb8056902144e7551d","2454cc03bb5342eca1ab02501d3ac39e","2da2765da24841709884a7cca16f1107","9d9f37ef8c9844a9ad5f4022ce904af3","b7febc3da8714283b798aafaab3a8d8f","f44ea695c0b2485faff83a6bbe12407e","1e1888b6dc3e4c47aadf843a62f03233","d75a669ab0a147ddb4185d6950c8267e","a1eab9ad9a6147c2a3a13ed1d837f94b","dd40e90d90a34827bb07d9d86c4234ce","2391e70d0de74d71ab5372f39a4fa49b","05fa6eea049e4b41a40604eed4c798d0","e271494364c24f4e901c42b64e464056","69b72f3ef4ae40a0a7b060be0849a354","3e756777c21b4c56a06b537ca232ffc3","0d4e9b5e9a3241dd920146b8be70d1ce","d6e7bf43dc2346c0a2278d6b19078c90","852593a171a048bc81b0cd3466366bd0","13c2754aca2a42c2ac06d3640062b6d4","1ac7aa00d4b24eccaf9b0cb622a65b05","50691a81b511451aa9f78cbd0803a652","43a743d7e9f14787a606cea5d0176931","8561cb62512f47d9929c3abd557ef739","52ac26d1722c40b68b18e0d92c6ac994","cbf2e61a7dea4b91a8f86774bcfc8ff7","8e58b5a80dca4cca8b03635c8d529072","9adf92080dd2458a942dbd76a285ce48","9a6a243411e84f049dfaeb99cccbc562","5458638e910744509182e8c4ace883ec","e692df9e6fb949c6a345dbaf9235d195","7409891326ad4a259842211e306e980a","0f57dd28b6954df693a160125075b506","ae151ed064b547b6a8bca5181668f491","5e8575da45094823bce0d16b9fe01f09"]},"id":"ZAk1C815g9AN","outputId":"0c5600e3-d81d-4f6e-d621-c7882a767fb1"},"source":["trainer = CollieTrainer(model=model, max_epochs=10, deterministic=True)\n","\n","trainer.fit(model)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","----------------------------------------------------\n","0 | user_biases | ZeroEmbedding | 941 \n","1 | item_biases | ZeroEmbedding | 1.4 K \n","2 | user_embeddings | ScaledEmbedding | 9.4 K \n","3 | item_embeddings | ScaledEmbedding | 14.5 K\n","4 | dropout | Dropout | 0 \n","----------------------------------------------------\n","26.3 K Trainable params\n","0 Non-trainable params\n","26.3 K Total params\n","0.105 Total estimated model params size (MB)\n"],"name":"stderr"},{"output_type":"stream","text":["Detected GPU. Setting ``gpus`` to 1.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"7f9d29d6c1f04d4b9adcda6292a698fb","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Global seed set to 22\n"],"name":"stderr"},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"71fb17475171409ab7b30f0915d3c3f9","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"85b96bc221984d08815cf3c09fde7c9f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b6f3004c83574b379bc3520a410b5df5","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"127ad96af11c41519ded336bbc246c42","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n","Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1997f9f7a4d04e5ebec4b8907e32b797","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2bb338780a1a40318d91fc689692c9ae","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b066cece2abe4a74b1aed6c238cd82b3","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 6: reducing learning rate of group 0 to 1.0000e-04.\n","Epoch 6: reducing learning rate of group 0 to 1.0000e-04.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2da2765da24841709884a7cca16f1107","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2391e70d0de74d71ab5372f39a4fa49b","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"13c2754aca2a42c2ac06d3640062b6d4","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9adf92080dd2458a942dbd76a285ce48","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"NQdGTCPvg9AN"},"source":["### Evaluate the Model "]},{"cell_type":"markdown","metadata":{"id":"ISQzTUnVg9AO"},"source":["Again, we'll evaluate the model and look at some particular users' recommendations to get a sense of what these recommendations look like using a partial credit loss function during model training. "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":117,"referenced_widgets":["b1c126acba99422c9aad4bc9b1b0293f","3e7789b77e3342acabd132d924906d5e","a9db4a7b06d34e2eaee4933580e8e28e","2e28af584886415c869f079215546e5e","e078e83f8216456997e974acfc7f4816","12b2182ba29547a8845685618988224e","6d1261287d6042b486877c29f1056305","6715a50cf1074e98b4e8e8f9aca4932e"]},"id":"6STWH4Ozg9AO","outputId":"861755ec-75be-40fe-975b-17accf21477f"},"source":["mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n","\n","print(f'MAP@10 Score: {mapk_score}')\n","print(f'MRR Score: {mrr_score}')\n","print(f'AUC Score: {auc_score}')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b1c126acba99422c9aad4bc9b1b0293f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","MAP@10 Score: 0.02882727154889818\n","MRR Score: 0.1829242957435939\n","AUC Score: 0.8585049499223719\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Bk75mVQWg9AP"},"source":["Broken record alert: we're not seeing as much performance increase here compared to the standard model because MovieLens 100K has so few items. For a more dramatic difference, try training this model on a larger dataset, such as MovieLens 10M, adjusting the architecture-specific hyperparameters, or train longer. "]},{"cell_type":"markdown","metadata":{"id":"9X25yfucbbKx"},"source":["### Inference"]},{"cell_type":"code","metadata":{"id":"dB6eeXWfg9AP","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"805ffd4f-38fd-4f4b-d9d1-b9ddfbdda60c"},"source":["user_id = np.random.randint(10, train_interactions.num_users)\n","\n","display(\n"," HTML(\n"," get_recommendation_visualizations(\n"," model=model,\n"," user_id=user_id,\n"," filter_films=True,\n"," shuffle=True,\n"," detailed=True,\n"," )\n"," )\n",")"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/html":["

User 895:

\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Willy Wonka and the Chocolate Factory (1971)Mighty Aphrodite (1995)Conspiracy Theory (1997)Sense and Sensibility (1995)Liar Liar (1997)In & Out (1997)Return of the Jedi (1983)Ransom (1996)Emma (1996)Toy Story (1995)
Some loved films:
\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Cold Comfort Farm (1995)Apartment, The (1960)Blown Away (1994)Star Wars (1977)Star Trek: First Contact (1996)Sex, Lies, and Videotape (1989)Big Squeeze, The (1996)Client, The (1994)Jerry Maguire (1996)Ghost and the Darkness, The (1996)
Recommended films:
-----

User 895 has rated 12 films with a 4 or 5

User 895 has rated 8 films with a 1, 2, or 3

% of these films rated 5 or 4 appearing in the first 10 recommendations:10.0%

% of these films rated 1, 2, or 3 appearing in the first 10 recommendations: 10.0%

"],"text/plain":[""]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"ZoIh0oHfg9AQ"},"source":["Partial credit loss is useful when we want an easy way to boost performance of any implicit model architecture, hybrid or not. When tuned properly, partial credit loss more fairly penalizes the model for more egregious mistakes and relaxes the loss applied when items are more similar. \n","\n","Of course, the loss function isn't the only place we can incorporate this metadata - we can also directly use this in the model (and even use a hybrid model combined with partial credit loss). Next, we will train a hybrid Collie model! "]},{"cell_type":"markdown","metadata":{"id":"Laxa0vh1hE3o"},"source":["### Train a ``MatrixFactorizationModel`` "]},{"cell_type":"markdown","metadata":{"id":"Fj3tJg-1hE3o"},"source":["The first step towards training a Collie Hybrid model is to train a regular ``MatrixFactorizationModel`` to generate rich user and item embeddings. We'll use these embeddings in a ``HybridPretrainedModel`` a bit later. "]},{"cell_type":"code","metadata":{"id":"m75xWkQLhE3o"},"source":["model = MatrixFactorizationModel(\n"," train=train_interactions,\n"," val=val_interactions,\n"," embedding_dim=30,\n"," lr=1e-2,\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":438,"referenced_widgets":["62986af6e0394e969bb8942274ba6784","4e51bc72f9fe41ca82096d01e26a5c23","a3b4a6f432f7406b97c59bdd31eec848","d6b7dc0b58874a93927b9182fe92ae63","2528199682404110adb35e51bf1c39ff","16dc9e31d8bb483faa575fd6db3915d7","d9b41c7acb1741d98f5edf4e942e3b77","7aabb759b56a4fc295687e0a24b6cb62","b1ccac573c5c4c79b4357c14f99a08b6","1c45f04d9cdc4876969a8a85d9087f18","84065182993c4cdcbc3e5a37c769447b","13ef2e644cae4a48ad8ab142e8b6435e","191a266c1f034c9fa6e7b4138805d60c","b8f64ba5faab4995a8b6f676b5a8e5d5","22cb3e6197d34688914565f07d26ecef","cc2768fdc75942c4a894bcf6006ad11f","952b7b171ddb4f4eb96cc91f1257ca9a","660daa560af241f2911156e2b7187c7e","847fd69ae9924b03929b1a0e59e80f87","e1cfc5099d1b4cc7bedcc7a53da34453","9b62c6c326784d799f790398e7fa22e9","65115b3c64504cc09ed3459b778995bf","069b286ddde94f77a425f288e6e14d09","f051d817821744d78a30577f69178e1e","c0632ad487cf400e906da9a52efa239a","12183c2f615f435a9cd920100f2820d4","839eb1025574493ca1cf19f5a2f2010e","ca5087b11f79495e8d32cfe8cca64f7b","949ae8d78ed644cebf9b8fd86a6be7fc","38f2d3b8e5ad41d1af054ef4c2ea47be","195cc632420d4e85aadf55741dafe166","e6ee962e992e44cca46b92e93b9f2c37","b06e9ed8cacb428ca888b8c7e0558e54","77d14fa2a785407181e4a9da9d40a3d0","fd4cec7630304a159031e309fdbd715c","cffb8b1a826b429488fc7d6af390df80","c58a9370d7484bf8bf50c8e1fef3da12","8d8c48b368c943aca716058d8a6c7485","4ff6be2f8fe8422fa6ed67aa4e4bb2cc","741722e693c344cca086aacee15eb874","d227738e76a3420ba0a79a6684de0dcf","0057a0667a434d1e8ac207b121049b2c","c6316fd570b741628ea906f553c52679","845c79c672984d0bab35df090dc632e5","52d654208b6b4001a56c287a822354d3","c47d7500fff14be39a2184e2d30b3795","f6273966027146e9b345c02f935e65be","72d633d88bd6413cad4cd3d2717dc7ca","b423c474bb5d4ba98c14502c6b02d95d","75241cec097b4fba877099eede78e3e7","2cef863fdeb0413a8a3ce9bc68d1f985","be5f89ff1dbc41b2a8263817d4a59f1f","9459414718e44c6a9b88760f1fb1b46a","1c8e2a3fde7942e697e111b07aefe26b","637168904e464ac9a6782ada14784eb1","a069be3b5a6d46e083524d14053bcb9f","71907183c5584961bfd232d68e685d3d","9e35548cd1714a0c9bf3ccc52d268fb1","5a131f872f8d4fcb96b713b60e56656b","7e72190f3fd44d1489badb8d28b6bb29","82dc472697914b1aaf5866e1573c662c","dd1af81809254587aa2a3d0deacff0a0","b61d4544c6a54060a2dc120eee5362f3","3b096679a00843148f3b874b9c70e901","2e2eb1edad7748ee92249d555840d612","8b2752894a8b4a69af0ad6be380c8c23","09a7ea71de884bcc94aa330697ab728b","29c1f761f5634be2bf0ff25736fd7d61","be4983b296534b4ba8991262f81d6c0a","058111735ae7416fab7f7adb8c981d14","1e7ed32472134c508325266efe272c40","d46475c0add24876b714a409bc9933b2","51757d5291844c6981c0258a245ad5be","36090e9a0cc94f56a377ff368eca2b64","cb77a4c364704cdca658427246c1a3fb","921e0906b18841d4b68b0b72dc911121","86439582332e43cea3d1eea6062510f2","edb0b8167858489c8f94c95337647d9f","92b45c3467f44cdfb4b12fc7d9f25f5a","1b13814df36f42969c8795c40b351ccf","d1a11fbd7c3a4dadbdee14b744937c30","248753435c404995812b93422ca8c062","2b3ae44914934e31ab2229c3b7d1b9a7","08c6b30a70024da28e546ff11dce4f94","ce736cb44cec467e8d6db28528e30780","d0ea236e247c46c78d6ee489433b76cf","33e822baba3b4f0bbec237b48deba7f4","fb8e91ab0228428487907e96827a91b9","c7fec0dc1e8648ca869c65caefa05751","acaa2bdd5eef41a49a99c02f384a8173","2a62b154d02740988eaf6df9ce39cd93","369e4fbc8a1b4c6d91f502a7fe5b18f4","f2ff83d1d99d40c7bb9d563ad623e164","e359213b91d841e3a28cdc293668a104","86a3bee8f0604c9a922c94e4161fc24b","f260359ad48e4f4c99e7aec1fc5cd09f"]},"id":"bjh0jE0yhE3p","outputId":"34e09142-f66f-4d32-9052-5e41d3e7d166"},"source":["trainer = CollieTrainer(model=model, max_epochs=10, deterministic=True)\n","\n","trainer.fit(model)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","----------------------------------------------------\n","0 | user_biases | ZeroEmbedding | 941 \n","1 | item_biases | ZeroEmbedding | 1.4 K \n","2 | user_embeddings | ScaledEmbedding | 28.2 K\n","3 | item_embeddings | ScaledEmbedding | 43.4 K\n","4 | dropout | Dropout | 0 \n","----------------------------------------------------\n","74.0 K Trainable params\n","0 Non-trainable params\n","74.0 K Total params\n","0.296 Total estimated model params size (MB)\n"],"name":"stderr"},{"output_type":"stream","text":["Detected GPU. Setting ``gpus`` to 1.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"62986af6e0394e969bb8942274ba6784","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Global seed set to 22\n"],"name":"stderr"},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b1ccac573c5c4c79b4357c14f99a08b6","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"952b7b171ddb4f4eb96cc91f1257ca9a","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c0632ad487cf400e906da9a52efa239a","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b06e9ed8cacb428ca888b8c7e0558e54","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n","Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d227738e76a3420ba0a79a6684de0dcf","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b423c474bb5d4ba98c14502c6b02d95d","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"71907183c5584961bfd232d68e685d3d","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2e2eb1edad7748ee92249d555840d612","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"51757d5291844c6981c0258a245ad5be","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d1a11fbd7c3a4dadbdee14b744937c30","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c7fec0dc1e8648ca869c65caefa05751","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":117,"referenced_widgets":["9b69ca9af02e4dcead166064746dfcb2","1748fb86e5824e739a39350e4777a4eb","27157458047e4a769ae205d23b8e6221","2c7ac2735ed649cfafc52ca48eb38d1a","ed61762550634ce6a906b6da4550f4c7","b1ab812f3fae427db2b8cef8fe572f83","08103e370a6047068d6e7d54653c1f3a","1d603bf9ea684001b97a63be7f446e3b"]},"id":"6tvE66cfhE3p","outputId":"1424a753-c468-48c2-dfc9-3b2118195955"},"source":["mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n","\n","print(f'Standard MAP@10 Score: {mapk_score}')\n","print(f'Standard MRR Score: {mrr_score}')\n","print(f'Standard AUC Score: {auc_score}')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9b69ca9af02e4dcead166064746dfcb2","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","Standard MAP@10 Score: 0.024415062120220127\n","Standard MRR Score: 0.1551878337645617\n","Standard AUC Score: 0.8575152364604943\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"CRq29RVfhE3q"},"source":["### Train a ``HybridPretrainedModel`` "]},{"cell_type":"markdown","metadata":{"id":"lFh1LEcChE3q"},"source":["With our trained ``model`` above, we can now use these embeddings and additional side data directly in a hybrid model. The architecture essentially takes our user embedding, item embedding, and item metadata for each user-item interaction, concatenates them, and sends it through a simple feedforward network to output a recommendation score. \n","\n","We can initially freeze the user and item embeddings from our previously-trained ``model``, train for a few epochs only optimizing our newly-added linear layers, and then train a model with everything unfrozen at a lower learning rate. We will show this process below. "]},{"cell_type":"code","metadata":{"id":"RPgUTdR1hE3r"},"source":["# we will apply a linear layer to the metadata with ``metadata_layers_dims`` and\n","# a linear layer to the combined embeddings and metadata data with ``combined_layers_dims``\n","hybrid_model = HybridPretrainedModel(\n"," train=train_interactions,\n"," val=val_interactions,\n"," item_metadata=metadata_df,\n"," trained_model=model,\n"," metadata_layers_dims=[8],\n"," combined_layers_dims=[16],\n"," lr=1e-2,\n"," freeze_embeddings=True,\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":438,"referenced_widgets":["1b0f046d63ec46909920000d416a956b","2010b6b4500e4b139e93077008383a3c","3a31cc9116d7465bbeea6471c6e93968","560c2c1f4ca94b73a47aeb344ee81ab1","cf1581e1b6614caa92b7845465c9de39","346d17d614ee426fbe2be794a34737f3","52271d9aa0d64d0dacb8d2d0d827adc0","eb62d6f5333a4cad90fe00ecab4f4de3","73d516b143a94e0a9ff2115a975dfe0b","cabcae1934154c03a2ddbfa47b2b4501","726b226d57474815ac786d93f4039902","1a2825490a2043aca84c13d316d11ed2","293790e04a5a44bb9f14edf7ac0ab8cc","5ccacb85b1214ee1a0e5665364c235b7","fb3d481dfea04ea3b6d1f8d57682826c","044c05d2a4824890a595a2f196d64459","ad5390af70b14da0aa0d3a0b5d20a90a","0f93d0dc68984a279b6382fbedda6630","e20d6099034e4567a726f6a7b038395c","76f0fc24f91d40e4869be6e1b5b4b0d3","a0666c429914487faf1fac59402e7364","d6c7474eb9114cde9ff9adde036f3bc2","83e1559c03c543c88418b270c16c88f6","01c61537bfe842649a60c70fde9a51a3","32b3c6d930f94d56b9f6e652303cc1ca","27b7e22b72b94b9da9d22190c338ac84","e39a8caba8854f5696d3ff03277bfaa0","ad367116739e4f4c8717c5b5b741b2ee","916dfdca43f74c058da5b45dff1eb621","564e25c0cc9d4f8bb9a00e8f480eaf7b","0cfb4e7dc348480d9cae85167df7a73e","24f32e83468d4a4eb03b91130a526df5","cf3dcff653a3443184d0942defe61230","7f75f89935e84173bf5d914bc8f50150","8c86d4c10827436b90b7f62885c57ef6","3971fc6570fc4f3ca85df75030b1a984","b5be05a8a7c045c1bce4eda3852a9256","5802083d9af4473684eadd22aa17d5c0","00a30c0b9eac4a38b0bd3be2cc05476b","dc0817dca939450caa82a751377451d4","aa23f91202f7432ab937a60fd0cc69c2","fa12cd0edffc4459aadb255b2f85309c","cfa25ee4254c479498e117189651d8c4","7e6c87609e254706b7e6ee6ac7bf78b4","a50062a20b2a475fb760c5ddf6266197","82c055fff9914a1d8a6c838062046b99","c16130a0a9624f848af1b5553264b78d","1f2cb0a8875946439822315ad3a7e1e5","22fbff68d90c43d6960fea7ddd420810","6f02c441db1a45488e41b29c656de74e","1408cb730c964bd39fd32f56916b9701","095d3de7d1e1464eb81d927e6edc3b0c","e5379796148347bc8f6ab506e1f5b1cd","bb09a545620c4dd1af632339d6fdd04a","08ee77ef97744d6d92a86674b9247b25","ef2fb16bb26f4a489fe456a33a9aaabf","c4588fb79e384aeabc04a175fe8d6548","d519f414b012498bafdd6ebd8503bbf0","a571a202b33d4a0dae7428d244d64cf6","f8841a4feab24ea1ace015833bbf0891","e0eb2bb0242e44aba710350121762fad","2cbdcb710a39477494596ee8e941152a","7ef288048daf401699f47fcf24ae2b2a","7927e1af2f644102b87c7c13ad22546a","c560a29789ca48129d338f88feddfeee","a2dc9ff9d3b6413489e37645e5a81969","e23fffe8236f4f9ab8b76858b371ed8c","7a1cdaf584e44e3392d963cd39f65cc0","0303b557637e45078eb791c8e6091f61","7025cdb08e7646169e13713fc442b5e3","c3216b05839d47d781220cc4dc762e46","41b780dbe1394396a24bb180a7945c3d","c58dc7499eef43f39a406d873aa9bf2f","3a7fe9e14ff94e14a3ea9be6a3685299","65d8dc2103ba4794a94ddbf4e1ef9825","16a6b8dc6f36458da47ce830fac15d26","5be8f16d753c47a6a68248c561a9605e","c85c795ddd734662b1277c3b4dd4bb6f","37aacb2f901c46c4bc7b9c580890a3c6","f837d58642d247cf9ffb657002d83ae6","109fb55f40334538966bbdb2961cfdf9","1a1fa94f37524d4cb93f5a2bfde53b12","7978b82dcd874c2782f4821d859ef918","e4f3bb71ec4645649a3e81bfd8b3e310","0374e23a55c249bb955a605926214583","14acab5f63754fd0970ca0a6e204d47e","720887ecd42045539a4f03a57b8a2d5e","94f0ec5f8e5e43de8397102e899dde43","fcbf9f9aecf54c1f97c3e2e3915c36c2","2b71910b6bf74406a468ef78e914db9f","7bfe5a6f9f2f4787acb15eedd6e23341","365f4e59afcb443f8c3cad99137dcd40","d9068f85980247bc89628635dd221f3c","c98453288d7e416c8ae6067db3220b5f","6657854d04e446339437228998a63325","27a1d1d028474aab8ee3cdc1c00166ad"]},"id":"vyyUg5ilhE3r","outputId":"77ffdf06-0964-4a7d-f491-a83d53d3e210"},"source":["hybrid_trainer = CollieTrainer(model=hybrid_model, max_epochs=10, deterministic=True)\n","\n","hybrid_trainer.fit(hybrid_model)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","--------------------------------------------------------------\n","0 | _trained_model | MatrixFactorizationModel | 74.0 K\n","1 | embeddings | Sequential | 71.6 K\n","2 | dropout | Dropout | 0 \n","3 | metadata_layer_0 | Linear | 160 \n","4 | combined_layer_0 | Linear | 1.1 K \n","5 | combined_layer_1 | Linear | 17 \n","--------------------------------------------------------------\n","75.3 K Trainable params\n","71.6 K Non-trainable params\n","146 K Total params\n","0.588 Total estimated model params size (MB)\n"],"name":"stderr"},{"output_type":"stream","text":["Detected GPU. Setting ``gpus`` to 1.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1b0f046d63ec46909920000d416a956b","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Global seed set to 22\n"],"name":"stderr"},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"73d516b143a94e0a9ff2115a975dfe0b","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ad5390af70b14da0aa0d3a0b5d20a90a","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"32b3c6d930f94d56b9f6e652303cc1ca","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"cf3dcff653a3443184d0942defe61230","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"aa23f91202f7432ab937a60fd0cc69c2","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"22fbff68d90c43d6960fea7ddd420810","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c4588fb79e384aeabc04a175fe8d6548","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c560a29789ca48129d338f88feddfeee","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c58dc7499eef43f39a406d873aa9bf2f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"109fb55f40334538966bbdb2961cfdf9","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"fcbf9f9aecf54c1f97c3e2e3915c36c2","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 10: reducing learning rate of group 0 to 1.0000e-03.\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":117,"referenced_widgets":["f3842767c3b9491ca0e39c7d83c3bba7","42e789d0121c43df87d03eeb4e58d3fe","bdebaa528fe345f58ed0e85d0015187b","08455bd7b312416a8828902e10a0a6de","4a1d091431614ff2a9a94baa79f4ebdd","78f224d19e054aa3a6787b9c3aa536eb","b6cff5f6b6004d6999a277ef1ed64fe1","9c1fd029256144a68b8c996e201c3e08"]},"id":"I8eEYwcfhE3s","outputId":"9babb089-060c-4636-f3dd-ebbf91d939e6"},"source":["mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, hybrid_model)\n","\n","print(f'Hybrid MAP@10 Score: {mapk_score}')\n","print(f'Hybrid MRR Score: {mrr_score}')\n","print(f'Hybrid AUC Score: {auc_score}')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f3842767c3b9491ca0e39c7d83c3bba7","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","Hybrid MAP@10 Score: 0.02650305521043056\n","Hybrid MRR Score: 0.15837650977843062\n","Hybrid AUC Score: 0.780685132170672\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EEw83cTUhE3s"},"source":["hybrid_model_unfrozen = HybridPretrainedModel(\n"," train=train_interactions,\n"," val=val_interactions,\n"," item_metadata=metadata_df,\n"," trained_model=model,\n"," metadata_layers_dims=[8],\n"," combined_layers_dims=[16],\n"," lr=1e-4,\n"," freeze_embeddings=False,\n",")\n","\n","hybrid_model.unfreeze_embeddings()\n","hybrid_model_unfrozen.load_from_hybrid_model(hybrid_model)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":472,"referenced_widgets":["b776627648254a919b1a18b4025fbcb3","2b4a275a432241ec91b5fe2a67fff267","b9fed3cbac434c5e8e8eccc37fdc7909","518844d0b3cc4deabe8b0ebb657eac8f","20a1726ffbf9408b8a00ccd5c44b81a8","f9e3ccff7f41406eb5ec4f9658b24718","d354c4d6b73d412f87a41f8cbb7bb9eb","01f11efae5cc4bc48726f6554c346655","bede57139cc14b599167088250548b43","512af7cae0c84552b736431802ed4402","2a5f13e1cce34787ba032800b8a59ef6","2dd137e524324334b5f7c56e7d4d8877","e7f0ad3f8d204c1a9c611a897655abbe","dbaaabf67d1640d5b988ca6e095577c2","9814e7c9d49245e7802828d385800a6d","7efc774340274141a6a6d4c3e5bfcf12","fe7f069fe1f94fe3a98c44c3cf5c3ed1","82de05cc4d4f4b32bf7127007ba17bef","3e5ca461bd1f4d6ba6e3f5d2a5c708b4","4d565f9c62d84ba487c79a1e2053770d","7beb2d7bb4b8421fa6a3fdc5a0a578b1","b56b52a8d05c4c2496e18f32df63ee5b","55a7ecf02b1f492e82b95e5460e04f22","7d93dfd6d22843108d76c8a2416bee21","3b0d7093adde4f64bfddffdb4444e1f2","aa9c6996aa3c40d18f0b3f0b8cd1705d","7dc6a6399f604fef8dda1b5a1d7b2920","c1ab4cda390d4a99b92162421e86718a","c894ed9774704d88bff3e9d1ad542900","4968d6ba2303488c9256042f3a7f8206","27da441d4d55492ebc526ca00dd7d01e","ad048e4075c847f1b911080e51548cdf","84cd0ec866694431a1bc3f6eb7686107","8b4924ef271d4097abd6e57303794327","200eca6c62424921ab682d2ab8a0785d","80117a933a694fd9914f88917466a00f","47c26c18cccf44f4b6a5caf3ccdd4e83","4fdf8e0a36244ff1bdf34e9060e3f035","7478d08325084ef28fa9f1c5a6ab18f6","1b0c6cff674c4930b18748d9fa4f9090","73232374fc7f4880a87dec552959d3a4","03dad58a3f004920aff305a2357ad121","fe02f58536de4d49b132a067fc065671","7805d39a07414d199a012bad80e90acb","094a99863ecc425dae15237f990ffb3e","b91817bbe07449b2a9a8d1b6be2ce378","82c6bf962cd04ee8bf02d24b03952b28","3e392b5c457a4ef2b7c883982f60c36b","5ebee37d75004546b316d10399b69431","b7fbdb82ca614754a12635170518e0cb","6cb3f43666734c869fcc960645e129e9","ad4885e1e6ab41f48ac113c30aa13b39","87093d6b3c924e6087e9d2b78ba0c6eb","0a4702427f524e768b8d1b2379e65499","3c9b57c8d7e24cb181e50d3b5e9a89d4","1f6a03090e2f4bd5a08b973a2e31a48e","43ff46cfff674d37a04bf926feca9048","7fb80bf0ab9549989de36323648126cd","34c2e6b19152468e8e8bbdcbf1e7d87e","521b635587644d588e28f0efba61aea2","6408b9869970482d949d6a794700716b","61a742fb4fde49fb9e4026e330cf2159","05bdc0f323064d359a32a0b8d345dd78","424e11f8cff442ee8a7643e56ffb36af","2ac6cf2e1f304f06bdc354d04507fecd","44ac79c4dfec4caa9bd2e4f987aeca9d","ebe6a36807834fb38ff46654e27075c1","ac257e025a9545d5b924d2946b422735","4606964ad6b5447db1d7498178cb5a78","e823a52deb434d5f81deb90ae34adc5e","9dc20f131e8642ddae5a90537309d835","44eea8a841af4ddb92d93bfe06c4c6fc","5a147c6e4b59428ebd7e2e3412cc52fc","49bf717484f84a25a7ac27b01b2606a3","c91bb1aa56074b398362c4326c9b9b13","1aad34c1efeb4af6a9ec6809a12a3569","7214617bec6645ba891a2071f6bc6442","ab827148c2884c728fe50dd09ce55912","d48633090c514ddd9e9fc2baa4bf347d","6315ef0a9eb14469a4de8063b9e745ca","afe817a0babe466cbbf8e4b802ef3360","60f26b29709e462381c221393c45e76b","cd7d3ce3534447b98fafe4d5aab0194b","f44f6872eb08434ea56d62708770cd25","cb38f538344b43928074a1184cd05997","286f2daf07f745d891110f78c116823c","c456a6bba5804e8e911a89abf34e5670","b4b847b559944ce2ba7a4ce34c472120","975f803a94994f91891c3fb7f187a135","47d4f8168a3049b09471968aca76fa08","0af18a7ade304878bdcb8dbca2ef1074","a521fbb49c5946b1afca37cbc4052b52","dac2cbf4f3f2477ab18adce6db8e77a8","5e9949db3b97478a905b23c0a437dd45","544a63b30c714580be37d69eb8669328","33fbfb7b8c2d4a5bb0e979c626862b13"]},"id":"yiA-EylqhE3t","outputId":"aefcb665-c88d-43ab-eb7a-322cbc58a262"},"source":["hybrid_trainer_unfrozen = CollieTrainer(model=hybrid_model_unfrozen, max_epochs=10, deterministic=True)\n","\n","hybrid_trainer_unfrozen.fit(hybrid_model_unfrozen)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["GPU available: True, used: True\n","TPU available: False, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","\n"," | Name | Type | Params\n","--------------------------------------------------------------\n","0 | _trained_model | MatrixFactorizationModel | 74.0 K\n","1 | embeddings | Sequential | 71.6 K\n","2 | dropout | Dropout | 0 \n","3 | metadata_layer_0 | Linear | 160 \n","4 | combined_layer_0 | Linear | 1.1 K \n","5 | combined_layer_1 | Linear | 17 \n","--------------------------------------------------------------\n","75.3 K Trainable params\n","71.6 K Non-trainable params\n","146 K Total params\n","0.588 Total estimated model params size (MB)\n"],"name":"stderr"},{"output_type":"stream","text":["Detected GPU. Setting ``gpus`` to 1.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b776627648254a919b1a18b4025fbcb3","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Global seed set to 22\n"],"name":"stderr"},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"bede57139cc14b599167088250548b43","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"fe7f069fe1f94fe3a98c44c3cf5c3ed1","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3b0d7093adde4f64bfddffdb4444e1f2","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"84cd0ec866694431a1bc3f6eb7686107","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"73232374fc7f4880a87dec552959d3a4","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"5ebee37d75004546b316d10399b69431","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"43ff46cfff674d37a04bf926feca9048","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2ac6cf2e1f304f06bdc354d04507fecd","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 7: reducing learning rate of group 0 to 1.0000e-04.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"5a147c6e4b59428ebd7e2e3412cc52fc","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"afe817a0babe466cbbf8e4b802ef3360","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Epoch 9: reducing learning rate of group 0 to 1.0000e-05.\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"975f803a94994f91891c3fb7f187a135","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"2Txbqf3fbqvD"},"source":["### Evaluate the Model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":117,"referenced_widgets":["0d9f2c4528634f63a26527a755b33773","6d196fda01ee4b63a7731b169c2f36b7","20d9caea7c744c09a8efd05793d2c6db","1760eeca2a084b1c8e57a9989139cdf6","8ccaa38836fd4fa588723ba2516f635b","113a8ae2c462494abf3ca97f65e51d06","c0282a49c81f4dce83322f9734eb0efd","9848dcb0b0e048c88f9c6243dc5ede33"]},"id":"sof4rqMbhE3u","outputId":"a344f1bb-0627-4e46-968e-f89bc81a5224"},"source":["mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc],\n"," val_interactions,\n"," hybrid_model_unfrozen)\n","\n","print(f'Hybrid Unfrozen MAP@10 Score: {mapk_score}')\n","print(f'Hybrid Unfrozen MRR Score: {mrr_score}')\n","print(f'Hybrid Unfrozen AUC Score: {auc_score}')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0d9f2c4528634f63a26527a755b33773","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","Hybrid Unfrozen MAP@10 Score: 0.02789580198163252\n","Hybrid Unfrozen MRR Score: 0.17139103232628614\n","Hybrid Unfrozen AUC Score: 0.8118089364191508\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"0FzTQc6WbtJA"},"source":["### Inference"]},{"cell_type":"code","metadata":{"id":"EwM1pkf_hE3v","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"650e4d07-8566-484c-c7b7-543e7c7428a9"},"source":["user_id = np.random.randint(10, train_interactions.num_users)\n","\n","display(\n"," HTML(\n"," get_recommendation_visualizations(\n"," model=hybrid_model_unfrozen,\n"," user_id=user_id,\n"," filter_films=True,\n"," shuffle=True,\n"," detailed=True,\n"," )\n"," )\n",")"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/html":["

User 895:

\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Willy Wonka and the Chocolate Factory (1971)Mighty Aphrodite (1995)Conspiracy Theory (1997)Sense and Sensibility (1995)Liar Liar (1997)In & Out (1997)Return of the Jedi (1983)Ransom (1996)Emma (1996)Toy Story (1995)
Some loved films:
\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
Jerry Maguire (1996)Bad Boys (1995)Blown Away (1994)Santa Clause, The (1994)Tin Cup (1996)Graduate, The (1967)Cold Comfort Farm (1995)Princess Bride, The (1987)Private Benjamin (1980)True Romance (1993)
Recommended films:
-----

User 895 has rated 12 films with a 4 or 5

User 895 has rated 8 films with a 1, 2, or 3

% of these films rated 5 or 4 appearing in the first 10 recommendations:0.0%

% of these films rated 1, 2, or 3 appearing in the first 10 recommendations: 10.0%

"],"text/plain":[""]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"fNwj-u-AhE3w"},"source":["The metrics and results look great, and we should only see a larger difference compared to a standard model as our data becomes more nuanced and complex (such as with MovieLens 10M data). \n","\n","If we're happy with this model, we can go ahead and save it for later! "]},{"cell_type":"markdown","metadata":{"id":"xYmFQZEhhE3w"},"source":["### Save and Load a Hybrid Model "]},{"cell_type":"code","metadata":{"id":"2ZDlfmAVhE3w"},"source":["# we can save the model with...\n","os.makedirs('models', exist_ok=True)\n","hybrid_model_unfrozen.save_model('models/hybrid_model_unfrozen')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qW3kPpenhE3x","colab":{"base_uri":"https://localhost:8080/"},"outputId":"da7c6d72-d7ca-4913-98fc-e85691a771f4"},"source":["# ... and if we wanted to load that model back in, we can do that easily...\n","hybrid_model_loaded_in = HybridPretrainedModel(load_model_path='models/hybrid_model_unfrozen')\n","\n","\n","hybrid_model_loaded_in"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["HybridPretrainedModel(\n"," (embeddings): Sequential(\n"," (0): ScaledEmbedding(941, 30)\n"," (1): ScaledEmbedding(1447, 30)\n"," )\n"," (dropout): Dropout(p=0.0, inplace=False)\n"," (metadata_layer_0): Linear(in_features=19, out_features=8, bias=True)\n"," (combined_layer_0): Linear(in_features=68, out_features=16, bias=True)\n"," (combined_layer_1): Linear(in_features=16, out_features=1, bias=True)\n",")"]},"metadata":{"tags":[]},"execution_count":98}]},{"cell_type":"markdown","metadata":{"id":"qKn2XvzuSaqU"},"source":["## Yet another Movie Recommender from scratch\n","> Building and training Item-popularity and MLP model on movielens dataset in pure pytorch."]},{"cell_type":"markdown","metadata":{"id":"GUoQMgjCPmkp"},"source":["### Setup"]},{"cell_type":"code","metadata":{"id":"Q6wvep55K6of"},"source":["import math\n","import torch\n","import heapq\n","import pickle\n","import argparse\n","import numpy as np\n","import pandas as pd\n","from torch import nn\n","import seaborn as sns\n","from time import time\n","import scipy.sparse as sp\n","import matplotlib.pyplot as plt\n","import torch.nn.functional as F\n","from torch.autograd import Variable\n","from torch.utils.data import Dataset, DataLoader"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jz4ocKNnLN4P"},"source":["np.random.seed(7)\n","torch.manual_seed(0)\n","\n","_model = None\n","_testRatings = None\n","_testNegatives = None\n","_topk = None\n","\n","use_cuda = torch.cuda.is_available()\n","device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"srnZMdMoPh9V"},"source":["### Data Loading"]},{"cell_type":"code","metadata":{"id":"J52BdmTvKvUv"},"source":["!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.train.rating\n","!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.test.rating\n","!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/u.data"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"csjr7o5wPd7n"},"source":["### Eval Methods"]},{"cell_type":"code","metadata":{"id":"et-6h-pkLLMk"},"source":["def evaluate_model(model, full_dataset: MovieLensDataset, topK: int):\n"," \"\"\"\n"," Evaluate the performance (Hit_Ratio, NDCG) of top-K recommendation\n"," Return: score of each test rating.\n"," \"\"\"\n"," global _model\n"," global _testRatings\n"," global _testNegatives\n"," global _topk\n"," _model = model\n"," _testRatings = full_dataset.testRatings\n"," _testNegatives = full_dataset.testNegatives\n"," _topk = topK\n","\n"," hits, ndcgs = [], []\n"," for idx in range(len(_testRatings)):\n"," (hr, ndcg) = eval_one_rating(idx, full_dataset)\n"," hits.append(hr)\n"," ndcgs.append(ndcg)\n"," return (hits, ndcgs)\n","\n","\n","def eval_one_rating(idx, full_dataset: MovieLensDataset):\n"," rating = _testRatings[idx]\n"," items = _testNegatives[idx]\n"," u = rating[0]\n","\n"," gtItem = rating[1]\n"," items.append(gtItem)\n"," # Get prediction scores\n"," map_item_score = {}\n"," users = np.full(len(items), u, dtype='int32')\n","\n"," feed_dict = {\n"," 'user_id': users,\n"," 'item_id': np.array(items),\n"," }\n"," predictions = _model.predict(feed_dict)\n"," for i in range(len(items)):\n"," item = items[i]\n"," map_item_score[item] = predictions[i]\n","\n"," # Evaluate top rank list\n"," ranklist = heapq.nlargest(_topk, map_item_score, key=map_item_score.get)\n"," hr = getHitRatio(ranklist, gtItem)\n"," ndcg = getNDCG(ranklist, gtItem)\n"," return (hr, ndcg)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"SvpXLWBpPYKk"},"source":["### Eval Metrics"]},{"cell_type":"code","metadata":{"id":"vvU46669Mnmz"},"source":["def getHitRatio(ranklist, gtItem):\n"," for item in ranklist:\n"," if item == gtItem:\n"," return 1\n"," return 0\n","\n","\n","def getNDCG(ranklist, gtItem):\n"," for i in range(len(ranklist)):\n"," item = ranklist[i]\n"," if item == gtItem:\n"," return math.log(2) / math.log(i+2)\n"," return 0"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Rv_-b2rnPQXI"},"source":["### Pytorch Dataset"]},{"cell_type":"code","metadata":{"id":"grg5RywRK1H8"},"source":["class MovieLensDataset(Dataset):\n"," 'Characterizes the dataset for PyTorch, and feeds the (user,item) pairs for training'\n","\n"," def __init__(self, file_name, num_negatives_train=5, num_negatives_test=100):\n"," 'Load the datasets from disk, and store them in appropriate structures'\n","\n"," self.trainMatrix = self.load_rating_file_as_matrix(\n"," file_name + \".train.rating\")\n"," self.num_users, self.num_items = self.trainMatrix.shape\n"," # make training set with negative sampling\n"," self.user_input, self.item_input, self.ratings = self.get_train_instances(\n"," self.trainMatrix, num_negatives_train)\n"," # make testing set with negative sampling\n"," self.testRatings = self.load_rating_file_as_list(\n"," file_name + \".test.rating\")\n"," self.testNegatives = self.create_negative_file(\n"," num_samples=num_negatives_test)\n"," assert len(self.testRatings) == len(self.testNegatives)\n","\n"," def __len__(self):\n"," 'Denotes the total number of rating in test set'\n"," return len(self.user_input)\n","\n"," def __getitem__(self, index):\n"," 'Generates one sample of data'\n","\n"," # get the train data\n"," user_id = self.user_input[index]\n"," item_id = self.item_input[index]\n"," rating = self.ratings[index]\n","\n"," return {'user_id': user_id,\n"," 'item_id': item_id,\n"," 'rating': rating}\n","\n"," def get_train_instances(self, train, num_negatives):\n"," user_input, item_input, ratings = [], [], []\n"," num_users, num_items = train.shape\n"," for (u, i) in train.keys():\n"," # positive instance\n"," user_input.append(u)\n"," item_input.append(i)\n"," ratings.append(1)\n"," # negative instances\n"," for _ in range(num_negatives):\n"," j = np.random.randint(1, num_items)\n"," # while train.has_key((u, j)):\n"," while (u, j) in train:\n"," j = np.random.randint(1, num_items)\n"," user_input.append(u)\n"," item_input.append(j)\n"," ratings.append(0)\n"," return user_input, item_input, ratings\n","\n"," def load_rating_file_as_list(self, filename):\n"," ratingList = []\n"," with open(filename, \"r\") as f:\n"," line = f.readline()\n"," while line != None and line != \"\":\n"," arr = line.split(\"\\t\")\n"," user, item = int(arr[0]), int(arr[1])\n"," ratingList.append([user, item])\n"," line = f.readline()\n"," return ratingList\n","\n"," def create_negative_file(self, num_samples=100):\n"," negativeList = []\n"," for user_item_pair in self.testRatings:\n"," user = user_item_pair[0]\n"," item = user_item_pair[1]\n"," negatives = []\n"," for t in range(num_samples):\n"," j = np.random.randint(1, self.num_items)\n"," while (user, j) in self.trainMatrix or j == item:\n"," j = np.random.randint(1, self.num_items)\n"," negatives.append(j)\n"," negativeList.append(negatives)\n"," return negativeList\n","\n"," def load_rating_file_as_matrix(self, filename):\n"," '''\n"," Read .rating file and Return dok matrix.\n"," The first line of .rating file is: num_users\\t num_items\n"," '''\n"," # Get number of users and items\n"," num_users, num_items = 0, 0\n"," with open(filename, \"r\") as f:\n"," line = f.readline()\n"," while line != None and line != \"\":\n"," arr = line.split(\"\\t\")\n"," u, i = int(arr[0]), int(arr[1])\n"," num_users = max(num_users, u)\n"," num_items = max(num_items, i)\n"," line = f.readline()\n"," # Construct matrix\n"," mat = sp.dok_matrix((num_users+1, num_items+1), dtype=np.float32)\n"," with open(filename, \"r\") as f:\n"," line = f.readline()\n"," while line != None and line != \"\":\n"," arr = line.split(\"\\t\")\n"," user, item, rating = int(arr[0]), int(arr[1]), float(arr[2])\n"," if (rating > 0):\n"," mat[user, item] = 1.0\n"," line = f.readline()\n"," return mat"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"M_CJ2wlKPS-w"},"source":["### Utils"]},{"cell_type":"code","metadata":{"id":"YhQs5tfvK9Pf"},"source":["def train_one_epoch(model, data_loader, loss_fn, optimizer, epoch_no, device, verbose = 1):\n"," 'trains the model for one epoch and returns the loss'\n"," print(\"Epoch = {}\".format(epoch_no))\n"," # Training\n"," # get user, item and rating data\n"," t1 = time()\n"," epoch_loss = []\n"," # put the model in train mode before training\n"," model.train()\n"," # transfer the data to GPU\n"," for feed_dict in data_loader:\n"," for key in feed_dict:\n"," if type(feed_dict[key]) != type(None):\n"," feed_dict[key] = feed_dict[key].to(dtype = torch.long, device = device)\n"," # get the predictions\n"," prediction = model(feed_dict)\n"," # print(prediction.shape)\n"," # get the actual targets\n"," rating = feed_dict['rating']\n"," \n"," \n"," # convert to float and change dim from [batch_size] to [batch_size,1]\n"," rating = rating.float().view(prediction.size()) \n"," loss = loss_fn(prediction, rating)\n"," # clear the gradients\n"," optimizer.zero_grad()\n"," # backpropagate\n"," loss.backward()\n"," # update weights\n"," optimizer.step()\n"," # accumulate the loss for monitoring\n"," epoch_loss.append(loss.item())\n"," epoch_loss = np.mean(epoch_loss)\n"," if verbose:\n"," print(\"Epoch completed {:.1f} s\".format(time() - t1))\n"," print(\"Train Loss: {}\".format(epoch_loss))\n"," return epoch_loss\n"," \n","\n","def test(model, full_dataset : MovieLensDataset, topK):\n"," 'Test the HR and NDCG for the model @topK'\n"," # put the model in eval mode before testing\n"," if hasattr(model,'eval'):\n"," # print(\"Putting the model in eval mode\")\n"," model.eval()\n"," t1 = time()\n"," (hits, ndcgs) = evaluate_model(model, full_dataset, topK)\n"," hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()\n"," print('Eval: HR = %.4f, NDCG = %.4f [%.1f s]' % (hr, ndcg, time()-t1))\n"," return hr, ndcg\n"," \n","\n","def plot_statistics(hr_list, ndcg_list, loss_list, model_alias, path):\n"," 'plots and saves the figures to a local directory'\n"," plt.figure()\n"," hr = np.vstack([np.arange(len(hr_list)),np.array(hr_list)]).T\n"," ndcg = np.vstack([np.arange(len(ndcg_list)),np.array(ndcg_list)]).T\n"," loss = np.vstack([np.arange(len(loss_list)),np.array(loss_list)]).T\n"," plt.plot(hr[:,0], hr[:,1],linestyle='-', marker='o', label = \"HR\")\n"," plt.plot(ndcg[:,0], ndcg[:,1],linestyle='-', marker='v', label = \"NDCG\")\n"," plt.plot(loss[:,0], loss[:,1],linestyle='-', marker='s', label = \"Loss\")\n","\n"," plt.xlabel(\"Epochs\")\n"," plt.ylabel(\"Value\")\n"," plt.legend()\n"," plt.savefig(path+model_alias+\".jpg\")\n"," return\n","\n","\n","def get_items_interacted(user_id, interaction_df):\n"," # returns a set of items the user has interacted with\n"," userid_mask = interaction_df['userid'] == user_id\n"," interacted_items = interaction_df.loc[userid_mask].courseid\n"," return set(interacted_items if type(interacted_items) == pd.Series else [interacted_items])\n","\n","\n","def save_to_csv(df,path, header = False, index = False, sep = '\\t', verbose = False):\n"," if verbose:\n"," print(\"Saving df to path: {}\".format(path))\n"," print(\"Columns in df are: {}\".format(df.columns.tolist()))\n","\n"," df.to_csv(path, header = header, index = index, sep = sep)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"qEl945qyM1FB"},"source":["### Item Popularity Model"]},{"cell_type":"code","metadata":{"id":"dzjxYN-mM3uv"},"source":["def parse_args():\n"," parser = argparse.ArgumentParser(description=\"Run ItemPop\")\n"," parser.add_argument('--path', nargs='?', default='/content/',\n"," help='Input data path.')\n"," parser.add_argument('--dataset', nargs='?', default='movielens',\n"," help='Choose a dataset.')\n"," parser.add_argument('--num_neg_test', type=int, default=100,\n"," help='Number of negative instances to pair with a positive instance while testing')\n"," \n"," return parser.parse_args(args={})"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xAr3XZ4sM73x"},"source":["class ItemPop():\n"," def __init__(self, train_interaction_matrix: sp.dok_matrix):\n"," \"\"\"\n"," Simple popularity based recommender system\n"," \"\"\"\n"," self.__alias__ = \"Item Popularity without metadata\"\n"," # Sum the occurences of each item to get is popularity, convert to array and \n"," # lose the extra dimension\n"," self.item_ratings = np.array(train_interaction_matrix.sum(axis=0, dtype=int)).flatten()\n","\n"," def forward(self):\n"," pass\n","\n"," def predict(self, feeddict) -> np.array:\n"," # returns the prediction score for each (user,item) pair in the input\n"," items = feeddict['item_id']\n"," output_scores = [self.item_ratings[itemid] for itemid in items]\n"," return np.array(output_scores)\n","\n"," def get_alias(self):\n"," return self.__alias__"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0EDavlooLWT0","outputId":"a1f13e86-030e-4530-e2d9-34b0d676570a"},"source":["args = parse_args()\n","path = args.path\n","dataset = args.dataset\n","num_negatives_test = args.num_neg_test\n","print(\"Model arguments: %s \" %(args))\n","\n","topK = 10\n","\n","# Load data\n","\n","t1 = time()\n","full_dataset = MovieLensDataset(path + dataset, num_negatives_test=num_negatives_test)\n","train, testRatings, testNegatives = full_dataset.trainMatrix, full_dataset.testRatings, full_dataset.testNegatives\n","num_users, num_items = train.shape\n","print(\"Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d\"\n"," % (time()-t1, num_users, num_items, train.nnz, len(testRatings)))\n","\n","model = ItemPop(train)\n","test(model, full_dataset, topK)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Model arguments: Namespace(dataset='movielens', num_neg_test=100, path='/content/') \n","Load data done [4.3 s]. #user=944, #item=1683, #train=99057, #test=943\n","Eval: HR = 0.4062, NDCG = 0.2199 [0.1 s]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["(0.4061505832449629, 0.21988638109018463)"]},"metadata":{"tags":[]},"execution_count":20}]},{"cell_type":"markdown","metadata":{"id":"IaJQ8h8kNWmr"},"source":["### MLP Model"]},{"cell_type":"code","metadata":{"id":"F_GYre42NhDX"},"source":["def parse_args():\n"," parser = argparse.ArgumentParser(description=\"Run MLP.\")\n"," parser.add_argument('--path', nargs='?', default='/content/',\n"," help='Input data path.')\n"," parser.add_argument('--dataset', nargs='?', default='movielens',\n"," help='Choose a dataset.')\n"," parser.add_argument('--epochs', type=int, default=30,\n"," help='Number of epochs.')\n"," parser.add_argument('--batch_size', type=int, default=256,\n"," help='Batch size.')\n"," parser.add_argument('--layers', nargs='?', default='[16,32,16,8]',\n"," help=\"Size of each layer. Note that the first layer is the concatenation of user and item embeddings. So layers[0]/2 is the embedding size.\")\n"," parser.add_argument('--weight_decay', type=float, default=0.00001,\n"," help=\"Regularization for each layer\")\n"," parser.add_argument('--num_neg_train', type=int, default=4,\n"," help='Number of negative instances to pair with a positive instance while training')\n"," parser.add_argument('--num_neg_test', type=int, default=100,\n"," help='Number of negative instances to pair with a positive instance while testing')\n"," parser.add_argument('--lr', type=float, default=0.001,\n"," help='Learning rate.')\n"," parser.add_argument('--dropout', type=float, default=0,\n"," help='Add dropout layer after each dense layer, with p = dropout_prob')\n"," parser.add_argument('--learner', nargs='?', default='adam',\n"," help='Specify an optimizer: adagrad, adam, rmsprop, sgd')\n"," parser.add_argument('--verbose', type=int, default=1,\n"," help='Show performance per X iterations')\n"," parser.add_argument('--out', type=int, default=1,\n"," help='Whether to save the trained model.')\n"," return parser.parse_args(args={})"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BJVmkqWGNoXM"},"source":["class MLP(nn.Module):\n","\n"," def __init__(self, n_users, n_items, layers=[16, 8], dropout=False):\n"," \"\"\"\n"," Simple Feedforward network with Embeddings for users and items\n"," \"\"\"\n"," super().__init__()\n"," assert (layers[0] % 2 == 0), \"layers[0] must be an even number\"\n"," self.__alias__ = \"MLP {}\".format(layers)\n"," self.__dropout__ = dropout\n","\n"," # user and item embedding layers\n"," embedding_dim = int(layers[0]/2)\n"," self.user_embedding = torch.nn.Embedding(n_users, embedding_dim)\n"," self.item_embedding = torch.nn.Embedding(n_items, embedding_dim)\n","\n"," # list of weight matrices\n"," self.fc_layers = torch.nn.ModuleList()\n"," # hidden dense layers\n"," for _, (in_size, out_size) in enumerate(zip(layers[:-1], layers[1:])):\n"," self.fc_layers.append(torch.nn.Linear(in_size, out_size))\n"," # final prediction layer\n"," self.output_layer = torch.nn.Linear(layers[-1], 1)\n","\n"," def forward(self, feed_dict):\n"," users = feed_dict['user_id']\n"," items = feed_dict['item_id']\n"," user_embedding = self.user_embedding(users)\n"," item_embedding = self.item_embedding(items)\n"," # concatenate user and item embeddings to form input\n"," x = torch.cat([user_embedding, item_embedding], 1)\n"," for idx, _ in enumerate(range(len(self.fc_layers))):\n"," x = self.fc_layers[idx](x)\n"," x = F.relu(x)\n"," x = F.dropout(x, p=self.__dropout__, training=self.training)\n"," logit = self.output_layer(x)\n"," rating = torch.sigmoid(logit)\n"," return rating\n","\n"," def predict(self, feed_dict):\n"," # return the score, inputs and outputs are numpy arrays\n"," for key in feed_dict:\n"," if type(feed_dict[key]) != type(None):\n"," feed_dict[key] = torch.from_numpy(\n"," feed_dict[key]).to(dtype=torch.long, device=device)\n"," output_scores = self.forward(feed_dict)\n"," return output_scores.cpu().detach().numpy()\n","\n"," def get_alias(self):\n"," return self.__alias__"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"lA9s7rv0LspV","outputId":"57d64091-be5b-493d-d20a-5f8d991e69ef"},"source":["print(\"Device available: {}\".format(device))\n","\n","args = parse_args()\n","path = args.path\n","dataset = args.dataset\n","layers = eval(args.layers)\n","weight_decay = args.weight_decay\n","num_negatives_train = args.num_neg_train\n","num_negatives_test = args.num_neg_test\n","dropout = args.dropout\n","learner = args.learner\n","learning_rate = args.lr\n","batch_size = args.batch_size\n","epochs = args.epochs\n","verbose = args.verbose\n","\n","topK = 10\n","print(\"MLP arguments: %s \" % (args))\n","model_out_file = '%s_MLP_%s_%d.h5' %(args.dataset, args.layers, time())\n","\n","# Load data\n","\n","t1 = time()\n","full_dataset = MovieLensDataset(\n"," path + dataset, num_negatives_train=num_negatives_train, num_negatives_test=num_negatives_test)\n","train, testRatings, testNegatives = full_dataset.trainMatrix, full_dataset.testRatings, full_dataset.testNegatives\n","num_users, num_items = train.shape\n","print(\"Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d\"\n"," % (time()-t1, num_users, num_items, train.nnz, len(testRatings)))\n","\n","training_data_generator = DataLoader(\n"," full_dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n","\n","# Build model\n","model = MLP(num_users, num_items, layers=layers, dropout=dropout)\n","# Transfer the model to GPU, if one is available\n","model.to(device)\n","if verbose:\n"," print(model)\n","\n","loss_fn = torch.nn.BCELoss()\n","# Use Adam optimizer\n","optimizer = torch.optim.Adam(model.parameters(), weight_decay=weight_decay)\n","\n","# Record performance\n","hr_list = []\n","ndcg_list = []\n","BCE_loss_list = []\n","\n","# Check Init performance\n","hr, ndcg = test(model, full_dataset, topK)\n","hr_list.append(hr)\n","ndcg_list.append(ndcg)\n","BCE_loss_list.append(1)\n","\n","# do the epochs now\n","\n","for epoch in range(epochs):\n"," epoch_loss = train_one_epoch( model, training_data_generator, loss_fn, optimizer, epoch, device)\n","\n"," if epoch % verbose == 0:\n"," hr, ndcg = test(model, full_dataset, topK)\n"," hr_list.append(hr)\n"," ndcg_list.append(ndcg)\n"," BCE_loss_list.append(epoch_loss)\n"," if hr > max(hr_list):\n"," if args.out > 0:\n"," model.save(model_out_file, overwrite=True)\n","\n","print(\"hr for epochs: \", hr_list)\n","print(\"ndcg for epochs: \", ndcg_list)\n","print(\"loss for epochs: \", BCE_loss_list)\n","plot_statistics(hr_list, ndcg_list, BCE_loss_list, model.get_alias(), \"/content\")\n","with open(\"metrics\", 'wb') as fp:\n"," pickle.dump(hr_list, fp)\n"," pickle.dump(ndcg_list, fp)\n","\n","best_iter = np.argmax(np.array(hr_list))\n","best_hr = hr_list[best_iter]\n","best_ndcg = ndcg_list[best_iter]\n","print(\"End. Best Iteration %d: HR = %.4f, NDCG = %.4f. \" %\n"," (best_iter, best_hr, best_ndcg))\n","if args.out > 0:\n"," print(\"The best MLP model is saved to %s\" %(model_out_file))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Device available: cpu\n","MLP arguments: Namespace(batch_size=256, dataset='movielens', dropout=0, epochs=30, layers='[16,32,16,8]', learner='adam', lr=0.001, num_neg_test=100, num_neg_train=4, out=1, path='/content/', verbose=1, weight_decay=1e-05) \n","Load data done [3.8 s]. #user=944, #item=1683, #train=99057, #test=943\n","MLP(\n"," (user_embedding): Embedding(944, 8)\n"," (item_embedding): Embedding(1683, 8)\n"," (fc_layers): ModuleList(\n"," (0): Linear(in_features=16, out_features=32, bias=True)\n"," (1): Linear(in_features=32, out_features=16, bias=True)\n"," (2): Linear(in_features=16, out_features=8, bias=True)\n"," )\n"," (output_layer): Linear(in_features=8, out_features=1, bias=True)\n",")\n","Eval: HR = 0.0848, NDCG = 0.0386 [0.6 s]\n","Epoch = 0\n","Epoch completed 5.8 s\n","Train Loss: 0.4429853802195507\n","Eval: HR = 0.3945, NDCG = 0.2187 [0.6 s]\n","Epoch = 1\n","Epoch completed 5.6 s\n","Train Loss: 0.3646208482657292\n","Eval: HR = 0.3818, NDCG = 0.2133 [0.6 s]\n","Epoch = 2\n","Epoch completed 5.6 s\n","Train Loss: 0.35764367812979747\n","Eval: HR = 0.3924, NDCG = 0.2137 [0.6 s]\n","Epoch = 3\n","Epoch completed 5.7 s\n","Train Loss: 0.35384849094297227\n","Eval: HR = 0.3796, NDCG = 0.2103 [0.6 s]\n","Epoch = 4\n","Epoch completed 5.7 s\n","Train Loss: 0.35072445729290175\n","Eval: HR = 0.3818, NDCG = 0.2143 [0.6 s]\n","Epoch = 5\n","Epoch completed 5.8 s\n","Train Loss: 0.3481164647319212\n","Eval: HR = 0.3881, NDCG = 0.2171 [0.7 s]\n","Epoch = 6\n","Epoch completed 5.8 s\n","Train Loss: 0.3454590990638856\n","Eval: HR = 0.4157, NDCG = 0.2292 [0.6 s]\n","Epoch = 7\n","Epoch completed 5.8 s\n","Train Loss: 0.3422531268162321\n","Eval: HR = 0.4231, NDCG = 0.2371 [0.6 s]\n","Epoch = 8\n","Epoch completed 5.8 s\n","Train Loss: 0.3384355346053762\n","Eval: HR = 0.4443, NDCG = 0.2508 [0.6 s]\n","Epoch = 9\n","Epoch completed 5.8 s\n","Train Loss: 0.3335341374156395\n","Eval: HR = 0.4677, NDCG = 0.2598 [0.6 s]\n","Epoch = 10\n","Epoch completed 5.8 s\n","Train Loss: 0.3280563016347491\n","Eval: HR = 0.4719, NDCG = 0.2652 [0.6 s]\n","Epoch = 11\n","Epoch completed 5.7 s\n","Train Loss: 0.3223747977760719\n","Eval: HR = 0.4995, NDCG = 0.2748 [0.6 s]\n","Epoch = 12\n","Epoch completed 5.8 s\n","Train Loss: 0.3164166678753934\n","Eval: HR = 0.5090, NDCG = 0.2817 [0.6 s]\n","Epoch = 13\n","Epoch completed 5.7 s\n","Train Loss: 0.31102338709726507\n","Eval: HR = 0.5143, NDCG = 0.2829 [0.6 s]\n","Epoch = 14\n","Epoch completed 5.7 s\n","Train Loss: 0.30582732322604156\n","Eval: HR = 0.5175, NDCG = 0.2908 [0.6 s]\n","Epoch = 15\n","Epoch completed 5.6 s\n","Train Loss: 0.3016319169092548\n","Eval: HR = 0.5429, NDCG = 0.2963 [0.6 s]\n","Epoch = 16\n","Epoch completed 5.7 s\n","Train Loss: 0.2980319341254789\n","Eval: HR = 0.5493, NDCG = 0.2978 [0.6 s]\n","Epoch = 17\n","Epoch completed 5.7 s\n","Train Loss: 0.29476294266469105\n","Eval: HR = 0.5504, NDCG = 0.3014 [0.6 s]\n","Epoch = 18\n","Epoch completed 5.6 s\n","Train Loss: 0.2921119521985682\n","Eval: HR = 0.5589, NDCG = 0.3108 [0.6 s]\n","Epoch = 19\n","Epoch completed 5.8 s\n","Train Loss: 0.28990745035406845\n","Eval: HR = 0.5620, NDCG = 0.3092 [0.6 s]\n","Epoch = 20\n","Epoch completed 5.7 s\n","Train Loss: 0.2876521824250234\n","Eval: HR = 0.5514, NDCG = 0.3097 [0.6 s]\n","Epoch = 21\n","Epoch completed 5.6 s\n","Train Loss: 0.2858751243245078\n","Eval: HR = 0.5578, NDCG = 0.3122 [0.6 s]\n","Epoch = 22\n","Epoch completed 5.6 s\n","Train Loss: 0.2843063232125546\n","Eval: HR = 0.5567, NDCG = 0.3043 [0.6 s]\n","Epoch = 23\n","Epoch completed 5.6 s\n","Train Loss: 0.28271066885277896\n","Eval: HR = 0.5663, NDCG = 0.3141 [0.6 s]\n","Epoch = 24\n","Epoch completed 5.6 s\n","Train Loss: 0.2813221255630178\n","Eval: HR = 0.5610, NDCG = 0.3070 [0.6 s]\n","Epoch = 25\n","Epoch completed 5.7 s\n","Train Loss: 0.28002421261420235\n","Eval: HR = 0.5610, NDCG = 0.3110 [0.6 s]\n","Epoch = 26\n","Epoch completed 5.9 s\n","Train Loss: 0.27882074906998516\n","Eval: HR = 0.5610, NDCG = 0.3095 [0.6 s]\n","Epoch = 27\n","Epoch completed 5.8 s\n","Train Loss: 0.27783915350449484\n","Eval: HR = 0.5663, NDCG = 0.3115 [0.6 s]\n","Epoch = 28\n","Epoch completed 5.7 s\n","Train Loss: 0.2768868865122783\n","Eval: HR = 0.5631, NDCG = 0.3109 [0.6 s]\n","Epoch = 29\n","Epoch completed 5.8 s\n","Train Loss: 0.2760479487343968\n","Eval: HR = 0.5631, NDCG = 0.3092 [0.6 s]\n","hr for epochs: [0.08483563096500531, 0.3944856839872747, 0.38176033934252385, 0.39236479321314954, 0.3796394485683987, 0.38176033934252385, 0.38812301166489926, 0.41569459172852596, 0.42311770943796395, 0.4443266171792153, 0.4676564156945917, 0.471898197242842, 0.49946977730646874, 0.5090137857900318, 0.5143160127253447, 0.5174973488865323, 0.542948038176034, 0.5493107104984093, 0.5503711558854719, 0.5588547189819725, 0.5620360551431601, 0.5514316012725344, 0.5577942735949099, 0.5567338282078473, 0.5662778366914104, 0.5609756097560976, 0.5609756097560976, 0.5609756097560976, 0.5662778366914104, 0.5630965005302226, 0.5630965005302226]\n","ndcg for epochs: [0.03855482836637224, 0.2186689741068423, 0.21325592738572174, 0.21374918741658008, 0.21033736603276898, 0.21431768576892837, 0.21714573069782853, 0.2292039485312514, 0.23708514689275148, 0.2507826695009706, 0.2598176007060155, 0.2652029648171546, 0.2747717153150814, 0.2817258947342069, 0.28289172403583096, 0.2907608027818361, 0.29626902860751664, 0.29775495439534627, 0.3014327139896777, 0.31075028453364517, 0.30917060839326094, 0.3096903348455541, 0.31217614966561463, 0.3043410687051171, 0.314059797472155, 0.3070033682048637, 0.31104383409268926, 0.3094572048871119, 0.3115140344405953, 0.31090220293994014, 0.3092050624323008]\n","loss for epochs: [1, 0.4429853802195507, 0.3646208482657292, 0.35764367812979747, 0.35384849094297227, 0.35072445729290175, 0.3481164647319212, 0.3454590990638856, 0.3422531268162321, 0.3384355346053762, 0.3335341374156395, 0.3280563016347491, 0.3223747977760719, 0.3164166678753934, 0.31102338709726507, 0.30582732322604156, 0.3016319169092548, 0.2980319341254789, 0.29476294266469105, 0.2921119521985682, 0.28990745035406845, 0.2876521824250234, 0.2858751243245078, 0.2843063232125546, 0.28271066885277896, 0.2813221255630178, 0.28002421261420235, 0.27882074906998516, 0.27783915350449484, 0.2768868865122783, 0.2760479487343968]\n","End. Best Iteration 24: HR = 0.5663, NDCG = 0.3141. \n","The best MLP model is saved to movielens_MLP_[16,32,16,8]_1626069383.h5\n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU1fnA8e87SzKTkIUsEAggKKAiu5FWUVFaXKq11FoU7aK1om3pz4ob1mrRat1aF6zVutO6rxQrVlsBl7ogCMjmkoCyCCQkJCH7Muf3x50kk2RmMgkzmST3/TwPT2bunNx7bkbPe8+5575HjDEopZSyL0e8K6CUUiq+NBAopZTNaSBQSimb00CglFI2p4FAKaVszhXvCnRWVlaWGT58eLyroZRSvcrq1av3GmOyg33W6wLB8OHDWbVqVbyroZRSvYqIfBXqMx0aUkopm9NAoJRSNqeBQCmlbK7X3SNQSqlQ6uvr2bFjBzU1NfGuStx4PB6GDBmC2+2O+Hc0ECil+owdO3aQkpLC8OHDEZF4V6fbGWMoLi5mx44djBgxIuLfi1kgEJFHgdOBQmPM2CCfC3AP8B2gCjjfGPNxtOtxwrMnUFxT3G57pieTFWeviPbhlFJxVFNTY9sgACAiZGZmUlRU1Knfi+U9gseBU8J8fiowyv9vDnB/LCoRLAiE266U6t3sGgSadOX8YxYIjDFvAyVhinwP+LuxfACki8igWNVHKaVUcPGcNZQLbA94v8O/rR0RmSMiq0RkVWe7PEop1Z369evX6v3jjz/O3LlzAViwYAG5ublMnDiRMWPG8PTTT8ejiu30iumjxpgHjTF5xpi87OygT0grpVSnLV6zk6m3LmPE/FeZeusyFq/ZGfNjXnbZZaxdu5Z//vOfXHzxxdTX18f8mB2JZyDYCQwNeD/Ev00ppWJu8ZqdXPPSenaWVmOAnaXVXPPS+m4JBgCjRo0iKSmJffv2dcvxwonn9NElwFwReQb4BlBmjNkV7YNkejJDzhpSSvVdN7yykU1fl4f8fM22Uuoafa22Vdc3ctULn/D0ym1Bf2fM4FR+/90jwh63urqaiRMnNr8vKSnhjDPOaFfu448/ZtSoUQwYMCDs/rpDLKePPg2cAGSJyA7g94AbwBjzALAUa+poPtb00QtiUY+mKaLF1cWc8NwJXDPlGs49/NxYHEop1Yu0DQIdbY+U1+tl7dq1ze8ff/zxVoky77rrLh577DE+//xzXnnllQM6VrTELBAYY2Z38LkBfhWr47eVnpiOQxw6bVQpm+joyn3qrcvYWVrdbntuupdnLz46VtXisssu44orrmDJkiVceOGFFBQU4PF4Yna8SPSKm8XR4HQ4SU9Mp6Qm3IxWpZRdXHnyoXjdzlbbvG4nV558aLcc/4wzziAvL49FixZ1y/HCsU0gAMj0ZlJcrT0CpRTMnJTLLWeOIzfdi2D1BG45cxwzJwWdxR4T119/PXfeeSc+34ENRx0osUZoeo+8vDzT1YVpLnrjIqoaqnjyO09GuVZKqZ5g8+bNHH744fGuRtwF+zuIyGpjTF6w8rbrEZRU69CQUkoFslUgyPBk6M1ipZRqw1aBINOTSXVDNVX1VfGuilJK9Rj2CgRe6yEynTmklFItbBUIMjwZgKagVkqpQLYKBE09Ap1CqpRSLewVCDw6NKSUii0R4fLLL29+/6c//YkFCxYArdNQjxo1ijPPPJNNmzY1l62vr2f+/PmMGjWKyZMnc/TRR/Paa68BUFFRwS9+8QsOOeQQJk+ezJFHHslDDz0UlTrbas3i5qEh7REopR44Fnavb789Zxxc8m6Xd5uYmMhLL73ENddcQ1ZWVrvPm1JMADz77LNMnz6d9evXk52dzXXXXceuXbvYsGEDiYmJ7Nmzh7feeguAn//85xx88MF88cUXOBwOioqKePTRR7tcz0C26hEkOBNISUjRewRKKRgyBZwJrbc5E6ztB8DlcjFnzhzuuuuuDsueffbZnHTSSTz11FNUVVXx0EMPce+995KYmAjAwIEDmTVrFgUFBaxcuZKbbroJh8NqtrOzs7n66qsPqK7NdY7KXnqRTE+mDg0pZQevzQ9+xd+koQ58Da23+Rqs33nstOC/kzMOTr21w0P/6le/Yvz48Vx11VUdlp08eTKffvop+fn5DBs2jNTU1HZlNm7cyIQJE5qDQLTZqkcA/ofKdGhIKeVKgOQBQNNi72K9b9tL6ILU1FR+8pOfsHDhwg7LdiXNz80338zEiRMZPHhwV6rXjv16BN5M8kvz410NpVSsRXDlzv7dcM8EaKgBVyJc/DakDIzK4X/zm98wefJkLrgg/FIra9asIS8vj5EjR7Jt2zbKy8vb9QrGjBnDunXr8Pl8OBwOrr32Wq699tp26yN3le16BDo0pJRqlpIDE88DcVg/oxQEADIyMpg1axaPPPJIyDIvvvgib7zxBrNnzyYpKYkLL7yQSy+9lLq6OgCKiop4/vnnGTlyJHl5efzud7+jsbERgJqami71JoKxXSDI8GZQVltGvS/+C0YrpXqAaVfBsG/CtOjceA10+eWXs3fv3lbb7rrrrubpo0888QTLli0jOzsbgJtuuons7GzGjBnD2LFjOf3005t7Bw8//DDFxcXNQWHGjBncfvvtUamnrdJQAzz32XP84YM/8N+z/svA5OhFf6VU/Gkaaoumoe6A5htSSqnW7BcI/E8X67MESillsW8g0CmkSikF2DEQ6NCQUkq1YrtA4HV58Tg92iNQSik/2wUCESHTm6n3CJRSys92gQD0oTKlVOxE62nf7mS7FBNg5RvaVbkr3tVQSsXRCc+eEHRkINOTyYqzV3R/heLInj0CHRpSyvZCtQGxaBvWrl3LN7/5TcaPH8/3v/999u3bB8DChQsZM2YM48eP55xzzgHgrbfeYuLEiUycOJFJkyaxf//+qNenLdv2CPbV7MNnfDjElrFQqT7vtpW38WnJp1363Qv+HTxR3GEZh3H1lM6novjJT37Cvffey7Rp07j++uu54YYbuPvuu7n11lvZunUriYmJlJaWAtaKZvfddx9Tp06loqICj8fTpXPoDFu2gpneTBpNI2W1ZfGuilKqjysrK6O0tJRp06YB8NOf/pS3334bgPHjx3PeeefxxBNP4HJZ1+VTp05l3rx5LFy4kNLS0ubtsWTLHkHgQ2X9Pf3jXBulVCx0dOU+btG4kJ89dspj0a5OUK+++ipvv/02r7zyCjfffDPr169n/vz5nHbaaSxdupSpU6fy+uuvc9hhh8W0HrbtEYCmmVBKxV5aWhr9+/fnnXfeAeAf//gH06ZNw+fzsX37dk488URuu+02ysrKqKiooKCggHHjxnH11Vdz1FFH8emnXRve6gxb9giaFrHXKaRK2VemJ/ikkaYRg66qqqpiyJAhze/nzZvHokWLuOSSS6iqquLggw/mscceo7GxkR/96EeUlZVhjOH//u//SE9P57rrrmP58uU4HA6OOOIITj311AOqTyRiGghE5BTgHsAJPGyMubXN58OARUC6v8x8Y8zSWNYJNN+QUoqYTRH1+XxBt3/wwQfttr377rvttt17771Rr1NHYjY0JCJO4D7gVGAMMFtExrQp9jvgOWPMJOAc4K+xqk+g1MRUXOLSoSGllCK29wimAPnGmC3GmDrgGeB7bcoYoGlxzjTg6xjWp5lDHGR4MnRoSCmliG0gyAW2B7zf4d8WaAHwIxHZASwFfh1sRyIyR0RWiciqoqKiqFQuw5uhQ0NK9UG9bdXFaOvK+cd71tBs4HFjzBDgO8A/RNo/4WWMedAYk2eMyWta2/NAZXoyNRAo1cd4PB6Ki4ttGwyMMRQXF3f6IbRY3izeCQwNeD/Evy3QhcApAMaY90XEA2QBhTGsF2BNId1atjXWh1FKdaMhQ4awY8cOojVy0Bt5PJ5Ws5YiEctA8BEwSkRGYAWAc4Bz25TZBnwLeFxEDgc8QLd8gxmeDIprrCsHEemOQyqlYsztdjNixIh4V6PXidnQkDGmAZgLvA5sxpodtFFEbhSRM/zFLgcuEpF1wNPA+aab+nSZnkxqG2uprK/sjsMppVSPFdPnCPzPBCxts+36gNebgKmxrEMogUtW9kvoffnDlVIqWuJ9szhump4u1mcJlFJ2Z9tA0JxvSGcOKaVszr6BwNMyNKSUUnZm20CQ7kkHtEeglFK2DQRuh5v0xHS9R6CUsj3bBgKwhod0aEgpZXe2DgSab0gppWweCEItTKGUUnZi70DgzaSkWoeGlFL2ZutAkOHJYH/9fmoba+NdFaWUihtbB4LmZwm0V6CUsjF7BwKvPlSmlFK2DgSab0gppWweCDTfkFJK2TwQaI9AKaVsHgi8Li9JriTtESilbM3WgQCs4SHtESil7EwDgeYbUkrZnO0DQYZH8w0ppezN9oEg06s9AqWUvWkg8Gayr2YfDb6GeFdFKaXiwvaBIMOTgcFQWlsa76oopVRc2D4QNOUb0vsESim70kDQ9HSxTiFVStmUBgKPJp5TStmb7QNBhtefZkKHhpRSNmX7QJDiTsHtcOvQkFLKtmwfCEREl6xUStma7QMB+J8u1h6BUsqmNBBg3TDWewRKKbvSQICmmVBK2ZsGAqyhoZKaEowx8a6KUkp1u5gGAhE5RUQ+E5F8EZkfoswsEdkkIhtF5KlY1ieUTE8m9b56yuvK43F4pZSKK1esdiwiTuA+YAawA/hIRJYYYzYFlBkFXANMNcbsE5EBsapPOE1PF5fUlJCWmBaPKiilVNzEskcwBcg3xmwxxtQBzwDfa1PmIuA+Y8w+AGNMYQzrE1Lz2sV6w1gpZUOxDAS5wPaA9zv82wKNBkaLyP9E5AMROSXYjkRkjoisEpFVRUVFUa+o5htSStlZvG8Wu4BRwAnAbOAhEUlvW8gY86AxJs8Yk5ednR31Smi+IaWUncUyEOwEhga8H+LfFmgHsMQYU2+M2Qp8jhUYulV6YjoOcejQkFLKlmIZCD4CRonICBFJAM4BlrQpsxirN4CIZGENFW2JYZ2CcjqcpCem69CQUsqWYjZryBjTICJzgdcBJ/CoMWajiNwIrDLGLPF/dpKIbAIagSuNMXFpjTXfkFK9w+I1O7nj9c/4urSawelerjz5UGZOanv7MX56ev2CiVkgADDGLAWWttl2fcBrA8zz/4srzTekVM9vxBav2ck1L62nur4RgJ2l1Vzz0nqAHlHPztYv0r93rL+XiAOBiCQZY6qiduQeJtOTySdFn8S7GkrFRCQNSSwa2c40YJGUveP1T5vr16S6vpE7Xv8s6PlEs5GNpNzt/w5ev1te28wJh2aT4nHjdEjz/iL5e3dH8JOO0iqIyDHAw0A/Y8wwEZkAXGyM+WVUatBJeXl5ZtWqVVHf7+0f3c6Ln7/Ih+d9GPV9KxVPbRsSAK/byY3fO4Jpo7Mpq66nvKaeOX9fTXFlXbvfz0338r/509vts7OBpem4t5w5LsKyDq446VAGpXvZsLOMjV+X89bnoaePjxrQj5w0DzmpHsqq61j+WRH1jS3tW4LLwZzjRnDcqJaZh+98UcSD72ylrsHX6XIuh3DU8P4kup3sLqthd3kNpVX1IevXJCXRRarXzZ7yGhp87dtfj9vBtNEtx37r8yJq6n3tygX7XsIRkdXGmLygn0UQCD4EzsKa3TPJv22DMWZsxDWIolgFgofXP8w9H9/DyvNW4nV5o75/pTojGleplbUNfFFYwfmPrqS0uuMGKpyfTR3BxGHpTBqazqovS/jtyxtCNvDGGIor6zj17ncoqqhtt69+iS5+NnU4bqcDt8uByyHcuyyfsjB1dDmEkQP6sa2kiqq6xnafJyc6OXZkFrvLa9ldVs2e8vbHjQWHwJjBqeSkeslJS2TJ2q8pr2loV65/kptfTx9FeU29FXirG3jx4x0h93tYTkrz60937w9aRoCtt54WcV3DBYKIhoaMMdtFJHBT+2+il2t6lqC4upghKUPiXBvVV0VriMYYw8sf7+C3izc0Xy3uLK3miufX8dA7BZTXNLC9pLrD+tw0cyxpXjepXjdXPLcuaMOd4HTw5Idf8ej/tgJW49f2Qra6vpH5L37CX1fks2NfddDGuklFbQMLl+V3WLcmS+ZOZfTAFDxuZ8hexs0zW/cyRsx/lWCXuAI8+fNvNL8/9+HgIwCRljMG/vXr45rf5x2UEbR+v//uEe2+5w+2FLOztP13lJvu5d+/Ob75/dRblwUtNzg9eheskQSC7f7hISMibuBSYHPUatBDBOYb0kCgYiFUA19b38iEYelsL6lme0kVf/7PZ0HHmS97bi3Xvrye+kZDvc9HsM58g8/w2e4KThmbw6wjhzI6J4Xr/7kh6BVybrqXH33zoOb31552eMihnNPGD+Kz3ftZs72U6xZvCHp+NQ0+hmUkM3VkFsMykrh3WT4lIYaa3r36RBp9pvlcTrrrbXaX1QQtO35IyzOmTY1pR8F0cLo3ZON5zMisVvs/0HKBIq0fwJUnHxr0733lyYd2qdyBiCQQXALcg5UeYifwBvCrqNWghwjsESgVC3e8HryBv9p/td8RY2D2lGG4nA4SnBLyqrrRZ/jLuZNbjlHXGFFD0lEjNjY3jbG5aTywoiDklezDP20ZeeiflBDyuCKCyym4nODFyfxTDou4sZs5KbfDm6TRbmQ70xhHUr+mctBx0OhMcOmqDgOBMWYvcF7UjthDab4hFUsllXVBG88m986exNCMJIb293LGX95lZ2nwq+PfnT6m+f2LH++M+lVqNBvZzh430rKRiHYjG6vGuDNBI5bTYyO5WfwYtB9uM8b8LFaVCidWN4vrGus48okjmTtxLhdPuDjq+1f2VFZdzyPvbOGRd7dSGWLcvO3sj0hn23RmVk609fTnDVR7B3qz+F8Brz3A94Gvo1GxniTBmUCKO0UTz6lOC9YozhgzkMff+5K/vWXduD1t3CDG5aZyz5v5UbuS7o4hg1BifYWqulckQ0MvBr4XkaeBd2NWozjK9Gbq0JBq1tUZPle+sI4Ep4PKuka+ffgALpsxmiMGWwse5aR5ozZE05lySoXTlRQTo4C4rCQWaxmeDL1ZbAPRmMJpjKGitoFbXtvc7gZwfaPBIYaXf3kMk4b1b/WZNtyqJ+owEIjIfqx7BOL/uRu4Osb1iotMbyYFpQXxroaKoXAN/Cljc9hTXsOushpueGVj0Bk+lz+/jgWvbKS8ur7dXPpAdQ2+dkFAqZ4qkqGhlI7K9BUZngxW1qyMdzVUF0VypX9biFww855by2+e7fgYjT7Dd8cPJtXrIs3r5q/LC4I+tRvNh32UirWQgUBEJof6DMAY83H0qxNfmd5MymrLqPfV43a4410d1QnBrvTnv/gJm3aVkeZNYOPXZWzYWc6uIA8tgfWk7BUnjSYnzUtOqod5z62lcH/wh7D+MLMlu8qAFE/MH/ZRKtbC9Qj+HOYzA0Se7aiXaHqobF/NPgYk9cnbIH1WsIe1ahp8PPi2lRbhoMwkxuamUlpVFzQXTG66l7nTWxbH++13gj9leyBz5ZXqqUIGAmPMid1ZkZ4g8OliDQS9hzEm5MNaAqxbcBKpHquHF2ru/YE+DKUNv+rNIpo1JCJjgTFYzxEAYIz5e6wqFS/6dHHvs/HrMm5+NXTqq8Hp3uYgANrAKxVMJLOGfo+1rvAYrNXGTsV6jqDvBQJPS+I51bPtLqvhT298xosf7yDN6+bMSYNZumF3q7ztB5oLRim7iKRHcBYwAVhjjLlARAYCT8S2WvGR4c0ANPFcTxM4GygnzcP43DTe/mIvDT4fPz92BHNPHEVakpvjR2vaA6W6IpJAUGOM8YlIg4ikAoXA0BjXKy6SXEl4nB4NBN2kKw927Sqz5vlPHJLGwtmTGZaZ1FxWr/SV6ppw00fvA54GVopIOvAQsBqoAN7vnup1LxEh05upQ0PdINh0z6te+IT3t+xlWEYyxRV1FFfW8u8Nu6ltaL9MX1FFXasgoJTqunA9gs+BO4DBQCVWUJgBpBpj+uwq7xmeDL1Z3A1ufa39g111jT6e/chavi85wUlmv8SgQQDg6zApnZVSnRNu+ug9wD0ichBwDvAo4AWeFpFqY8wX3VTHbpXpyWRX5a54V6PP2rq3kr+9VcDu8uAPdgmw6cZT8CY4ge5Zpk8pu4skxcRXwG3AbSIyCSsgXA84Y1y3uMj0ZrKxeGO8q9GrBRv7HzmgH/e/VcBr63fhcjpITnAGzc8/ON3bHASge5bpU8ruIpk+6sKaMnoO8C1gBbAgprWKgxOePaHVkNC4ReMAq4ew4uwVcapV7xNs7H/ec2vxGeiX6GLO8Yfws2OH815+sT65q1QPEe5m8QxgNvAdYCXwDDDHGFPZTXXrVqHuC+j9gs4JlurBZyDV4+Kdq6eT5rUe7tIHu5TqOcL1CK4BngIuN8bs66b6qF5sf019yFQP+2samoNAE23gleoZwt0s7nNJ5VRsbN1byaL3vuSF1TtCltGbu0r1XF1ZoUzZUNsbwFfMGE1GSiKP/28ryz8rwu0UTh8/mBFZSdy/Yove3FWqF9FAYHNdXbZx3vPrMEBWv0R+8+1RnPuNYQxIsXISDstI1pu7SvUiGgj8Mj3BF65PcCTEoTbdI9yyjaeOy2FbcRVb9lZy/T83tLsBbID+SW7emz+dBJej1Wc69q9U76KBwC/YFNGH1z/MPR/fw/JtyzlxWN9bniHYDJ+mdXmbpnyGU1pV3y4IKKV6n5j+Xywip4jIZyKSLyLzw5T7gYgYEcmLZX0666dH/JRR/Udx84c3U1nf92bNhkrT0OgzzJ0+irvPnsiSuVMZlOYJWk5vACvVN8QsEIiIE7gP62G0McBsERkTpFwKcCnwYazq0lVuh5vfH/17CqsKuXfNvfGuTtT4fIaX1+zAIcE/z033Mm/GaGZOymX8kHSuPuUwvO7WD5LrDWCl+o5YDg1NAfKNMVsAROQZ4HvApjbl/oCVwuLKGNYlIsFvnE5g1qGzeGrzU5x+8OmMzRrb8Y463Gf8xs9Xbi3hplc38cmOMoakeyisqKOuIfxiLvp0r1J9WywDQS6wPeD9DuAbgQVEZDIw1BjzqojENRCEu3F66eRLWb5tOTe8fwNPn/Y0Lkdkf7Zw+4x1I9o2AJ1/zHBWfVXC6xv3MCjNw52zJjBzYi5L1n2tT/cqZXNxu1ksIg7gTuD8CMrOAeYADBs2LCb1CXXj9KZXN3HK2Olc841ruGzFZTyx6QnOH3t+h1f6xhhueW1z0H3e8fpnXW5Uuzrd8+alm0lwClecNJoLjz24ObGbNvBKqVgGgp20XslsiH9bkxRgLLBCRABygCUicoYxZlXgjowxDwIPAuTl5XUwl6VrQt043VtRx4Qb3uDI4ekMTz2Ke9f8hcb9R/CnpcWtGtr5L33C53v2k5zoYu32UtZuL6Vof23Qfe4sreb5Vds5ZmQWuf4brl1t4Jt6GN+dMJg95TVsL6liwSsb2wUggIzkROZOH9W1P5BSqs8SY2LSrjZlLf0cK2PpTuAj4FxjTNAczyKyAriibRBoKy8vz6xaFbZIl4TKe5+ZnMDMSbm8V1DMp0XbSD74Thqrh1O9/QKs7PntjchKZuLQdJZ9WkhZdX27zx1C89TMgzKTGJzmYdVX+6hvbPkuPG4HV518KMeNyqaqrpGqukbmPvUxxZV17fbndAgOodXvByPA1ltPC1tGKdU3ichqY0zQmZkx6xEYYxpEZC7wOtbaBY8aYzaKyI3AKmPMklgduytC5b2/7vQxzVfmxRW1/PnDMl7ZcT+u1E9oKJ/Qbj9rr59BepL1EFrbK/imff5x5lgOz03lvfxi3iso5s3Ne2jbhNfU+7jxX5uBzR3WvdFnuGjaIQzN8DK0fxJXvrCOPeXteyM63VMpFUzMegSxEqseAVgNd9ODVLkhhmcafY1M/PtkkPZLKEpjCp/87L12++xoyGfE/FfbBYIm986eRFKCE2+Ck0ufXktRRfsGPjfdy//mt+QIDBWAbjlznN4PUMqm4tIj6I2OG5WFz8DvTjucnx93cNAyToczaBAAMM79rd43L3YzCPoNgnLguk/g7s9bL3YzON0bdFgqN93LdycMbn5/7WmH62IuSqmo00AQIL+wAoCRA/p1eR+flnzK0JShJLuTI17sxjd0ASmD2i/54HP1B1qu9HUxF6VULGggCJBfdOCB4Iev/BCALG9W2HKlNaWkJKTgdDipaAi+7k/b7ZH2MNouu9lEl91USgWjgSBAfmEFXreTwWldv6n652l/Ztv+bWwr38bL+S+HLHfcs8chCGmJaWH398/8f5KakEpKQkrEPYxIy0UaMDSwKNW3aSAIUFBUySEDknGESsITgZOGn9T8OlwgmD9lPvtq9lFaW8qznz0bstzv/ve7iI573qvnkeROop87fG/mo90fkexOpp+7X9QDC2jQUKo30kAQoKCwgqOG9++wXKi1CzI9mREf67zDz2t+HS4QLP3+Usrry9lft5+L3rgoZLl+Cf2orK9kb/XesMf92es/i6h+81bMIzUhldSE1LDljDH4HwgEot8bUUrFngYCv8raBnaWVnNO9tAOy0baUEUjYAxN7bg+AH+b8bfm1+MWjQtZ7pGTHqGivoLK+kp+++5vQ5bLL82nvLac8rrysMc96smjSE9Mp7+nf4fDXA2+huY8TRowlOo5NBD4bSmy1hs4kBvFbXVnwIjUlEFTml+HCwRLZrY87xcusJx72Lnsq91HaU0ppbWlYY89+R+T6e/pz4CkAWHLBfYyOjMspZTqGg0EfvlF1jMA0QwEkYp2wOjOwDIvb16r9+GCxiUTLqGwqpC91Xv5tOTTkOWmPDmFgckDyUnKibge2nNQqus0EPgVFFbidAgHZSbHuyohRdqg9dTA8suJv2x+HS5gzDp0Fnuq9rC7cnfY/c1cPJOc5BxyknN0qMkuHjgWdq9vvz1nHFzybvfXp4/QQOCXX1jBQZlJtlqDN9qBBaITNK48qmVpinABY3jacHZV7mJzSfh8TFe+dSWDkgcxqN8gewaMSBvPeJXrjCFToOgzaAxIvuhMsLZ3RTzPOZ5/xzY0EPjlF1VwSHb3Dwv1Nd15Xw7+qXIAABcSSURBVOTuE+9ufh0uYGwq3sSb296k3tc+E2yg+9fez8DkgQxMGti3AkakjWe8ykHkjd03fwFr/tG6jAhMu6pr+4vnOcdin12kgQCob/Tx5d5KZowZGO+q2EZ3BoxXz3wVn/FRXF3M9Oenhyx3/7r7MSHT/7V4avNTDEwayICkAfF9xqKjxs7ng31brfemzfoUvgbY9yU8fwG4POBKBF9j+3LGB64EePPGls+Nz3rddn+7N8CDJ0BtBdRVQE1568YLoLEeqkrgnTsh+1DIOhT6Dw/R2LkhbQj87x7YtQ6+XgslBe3Pt6EWFk6GjIMhYwRkHgKeNHC4rHo1cbggeQB89AhUFUPlXti/q3WZpjruXg9P/KDlb2NMkHNuhLLt8MRZUF8FdZVQuz/IOdfBxpdg8yvgcII4rZ8Y61ht/44lW+H588Hl9R/b1/57EQdMu7r936KLNBAAXxVX0eAzjNQeQY8TratqhzjITsoOW2b1j1ZTVF3Enqo9/OS1n4Qsd8vKWyI65qKNi8jwZDT/i7iX8fh4iqV9QMo0worzP2kpl1RN8Yj2K/Zl+spY8cjJsGeD1SC3I5CUCRWFULrNakgbqq2fvjbH9TXABw/4GzBHSyMmjtaNkyfdujJPyoT0gyCxHySkwLb3rUbVNFq/02+gtW1TwMOWDrf1O8Ea5M9es/6lDYVBE2DCbEgfBkt+DY211pXxCfOtRr24AAo3W+WD9f58DVDwpvUPIDENkjMhKQsqiwBj/W1SB1tBqHqf/29TY/10uqEh4JzdXuvv506y/vUbYAU1ESjZYjXg4oDsw2DY0dbfwBcQSH0N1t9z35ctx07KgqoiK8A01EB9tfXTF5Do0pkAE8+DlOhduGogAAqikGNIxVc0eg5up5vB/QYzuN/gsOWWz1pOYVUhhVWF/HrZr0OW+9OqP0V87Fs+vIX0xDRSXclBgwBgbX/rDqgugaoSik37RYoAih2AD5h4LuSM44TPHqS4rqxduUyPixVnt6R071SPZf9uuGeC1Ui5PPDLD4I3TM3lGq0GbM5bVrmaMtj7hdUL2Pu59a+yEGqbnlsRyD0STvytFQCS2+Tu2v4hrH4MJv0Yjru89WeNDVZD+trVkP9fqwF2OGHkDJj+O6uxTcq0ejrtziURLlrewbn4z/nXH3dczpkAP14cutFue+xL3u14n1HuDYAGAqAl6+ghGgh6re4casryZpHlzWJM5piw5d47+UlKSj5n376tFJdv5zeFy0OWfWXTE+x3dDxR4eQvHiXVQKq4wB263PunLmhJJbL+jqBlDjhH1dCA50Fe+nboHFWhyg3Js/41lR2cDqQHHKmQzNV/CP7dTrsKijYHbxCdLmuI6IyFLUHI4YbvLgzeyKbkWFfYqx8Lf6Ud7XKx2mcXaCDASi2Rk+qhX6L+Ofq6FfsaYPe29h/ktH4qOtNIyOGZZtXBs8Y2SXngOFKAg5o2BBnGafLekLNocLopdwjTvnomZLkjR59BeX2l9cR34ZqQ5eb8Z07YujX58dIf43V5SXInhS3336/+S5IriSR3UkxyVHXp5rwDeOnbQJhEiZEGK4DhQ6D4P7BoXOj7NuECUFfKxWqfnaQtH9aMIR0W6uWiPEtkRea3YO0TrcuJE7JGweOnW/uoLCRzaC7FLme7w2Y6PDDzAUjLhdRca9z5qTCzPGbcgAvIAFgUOhD88fjbml+Hmyn1+CmPU1lfSWV9JVe9fVXIcomuRCobKimqLgpdN+CyFZeF/bzJic+dSKIzEY/TE7bcbStvI9GZaP1zJYYtu2Hvhuay0Q5CMQtAHZSL1T67yvaBwBhDQWEFP8yLLKeP6qHCNfB1ldZMjJIt4O3ffvZHYx1sXAwbXrBuUDbWtb9xCdZYc/lOSEyF0SdB1qGsyBoNSf1h0RktY8eXfhK0696dT3wfOfDI5tfhAsHDJz3c/DpcYHnhuy9Q1VBFVX0Vl/z3kpDlpg2ZRm1jLTUNNRSUBZnh4/dy/svUNtbSEOzv3MbsV2d3WAasIJTgSCDBmRC23LXvXovb4cbtCDO2BizOX4zb4SbBmRA2YGwt24rL4cLtcMekF9QdaVZsHwh2l9dQWdeo9wd6qkiv9I+/EtY80bqMrwE2LYFVj7Te7kyERh/WTA0HZI6EEcdagcPp9v9MgC/esKYsmkZr6uG4WTDzr9askLYiGL9tvnrbvxteuADOevyAAkZ3BpZDMw7tuBCw4JgFza/DBZYPzv0AsNYAr/PVMeXJ0L2lv0z/C7WNtdQ21obNjzVtyDTqffXUNdbxZfmXIcut2r3KKucLfrO9yXX/uy7s503OWHxGROWmPDkFl7hwOVzNyRdDOf/f51vlpHuaaNsHgublKXXqaPc60KGcrMNg/QvW/PLdn1g/G2tb7ys525opkjHCP8fcP8+8vjpgpkYCnP+v4I33kecH3Gx0wbcXBA8C0Lnx25QcuOC1kB/39lQineF0OPE6wi8ENW3otObX4QJBYBD695f/Dlnu9bNeb34dLli9duZr1PvqqffV84MlPwhZ7pbjbqHB10CDr4Eb3r8hZLlZo2fRYBqay774xYshywpCXWMdVb6qkGWiSQNB84yhnptjqE/qaKzeGOtm7OFnwJq/t/7dxjrY8Lz1z5kAA8bA4d+F/iNgxS3W5y4PXPxO8Abekxab2R9hGvd46g2BJR5BqCNDUoZEVO70g09vfh0uEFxx1BWt3ocLBI+d8ljz63DBKlo0EBRWkOpxkd0v/A0rFaEDGspptJ5OXTgZyr+2HnJqR2DA4XD0ryBnvPWwjitgTLhsR2QNdw+YqdFbxSJHVbyCUE8MQPGggaDQmjEkobr8qnNCXelnHGw1/Ls3WE+87tnQfijH7bWGXgaNh0NPtWbapA62ru6f+6lV3pUY/gGdSBvuSK/ge/CVvh1FOwj1hl5QdwQrMabj3Co9SV5enlm1alXHBSPd303/Zfph2dx+1oSo7dPWyr6GhRPa51tp4k6yhnJyxkLaMHjr1pahnBCzbQD41zzrSv/IC+D0O2NXf6X6KBFZbYzJC/aZrXsEZVX17K2o1WcIOtLRcE9FIRQsg/w3YcvyNkHAf4V/7GUwcJx1s9YRMO++fGd0h3KUUp1m60DQtCqZpp/uQLDhHocLkNZBIikTDpkOuXnwn+tbhnLOfb77hnKUUp1m70BQaPNkc5Hc2K2rgsNOg4/bzNzxNcCejTDsmzD9Ohj5LciZAE35cvZ+3utn2yhlF7YPBAkuB0P6h8+z0mcFu9IXpzV184kfQNHnUBYkL484rCv/sx4DT2rwfetQjlK9hq0DQUFRJQdnJeN0SN9aCzWSc9n3FWSNbp9uwTTC3nxr9s7QKTD5x1Y5bwY89cOW1Lrf+2voIAB6pa9UL2LrQJBfWMH4If6sk7FYVi/aweVAnsZ1uK2FOF6aA1+9Z+VrB+scm9ItOFww9kyY+beWIZ5AMUyDq5SKH9sGgpr6Rrbvq+LMybnWhmlXwdonWxfyNVpXxh/+rWVOe2ou5B4V3bVGD7SBzxwF2z6wlgasLYf0odYqSK3OpR6+etdKu3DQVJh6KRx0DHj6w72TrCt9hwtm3BQ8CDT9jXS4R6k+x7aBYEtRJcYEzBhqTifwuH8JPrEecPr47+3nxIuz/RqijfVWY/zAcf41Rn3W8nbB1iSt2w//XWBlwvSkW41z2/VVxWmtWfqf31vrq1YVW0/btttfvbUe6saXwpytAw46Gr57j5Vgre3Dc30gjYJSqutiGghE5BTgHsAJPGyMubXN5/OAnwMNQBHwM2PMV7GsU5P8YMtTTrsKVj1qvXYlWkvR9Rvgb4R3Wg1x089N/7TWSA1c5zQtt2VdVxFrvrzxWePx+KxyCSlQsNzKoxMuBa9phB0r4es11jJ9SVmQlAEZw639GZ91nIOOhqMugsQUK4dOYqr1uqEa/np0S2K1sx478CmcSqk+KWaBQEScwH3ADGAH8JGILDHGbAootgbIM8ZUicgvgNuBs2NVp0AFhRU4BEZkBSSbcybQ3LAHXh0nZ1n/BgU8fTxlThfWOU2EuR9Z5Yyx8uRX77P+rfgjfPEfKzg4XHDEmXDan61GPfAKvtV6qG74waOhG3i90ldKRaDjRVK7bgqQb4zZYoypA54BvhdYwBiz3BjTlGf1AyCydH9RkF9UwdCMJDzugKdct75l/cwZG9kDThPPs3oAkWSwbFtOBBL7WeP5g8bD6Xf7H9LC+nnSTdasnLbDOJEeF6wr/WHf1Ct9pVRYsQwEucD2gPc7/NtCuRAIelkqInNEZJWIrCoqCr+kXqQKCivar0FQsMyaWXPRishmxUTa0EZSLhYNfNOVvs7wUUqF0SNuFovIj4A8YFqwz40xDwIPgpV07kCP1+gzbNlbyfGjswMPYo3dH3w8OCP8s0Q7g6WmW1BKxUEsewQ7gcCFgIf4t7UiIt8GrgXOMMbUtv08FraXVFHX4GvdIyjOt+bWHzK9O6oQnF7BK6XiIJaB4CNglIiMEJEE4BxgSWABEZkE/A0rCBTGsC6tFBQ1rUoWEAjy37R+xjMQKKVUHMQsEBhjGoC5wOvAZuA5Y8xGEblRRJpWe74D6Ac8LyJrRWRJiN1FVdBkcwXLIOMQ6D+8O6qglFI9RkzvERhjlgJL22y7PuD1t2N5/FDyCyvITkkkzeu2NjTUwpfvWDdplVLKZmI5NNRj5RdVcEh2wPMD21dCfZUOCymlbMl2gcAY07xOcbOCZdbc/eHHxq9iSikVJ7YLBEUVteyvaWg9Y6hgmZXQLVxaZaWU6qNsFwhabhSnWBsq98KudTospJSyLdsFgoK2M4a2rACMBgKllG3ZLhDkF1bQL9HFwNREa0PBMisd9OCJ8a2YUkrFif0CgX/GkIj400osg4NPsFJGK6WUDdkvEBRWtDxRXPQp7N+lw0JKKVuzVSDYX1PPnvLalvsDBcusnwefGL9KKaVUnNkqEBQUVQK0TB0tWAZZo601AZRSyqZsFQha5Riqr4Ev/6fDQkop27NdIHA7hWEZSbD9A2td30O+Fe9qKaVUXNkuEAzPTMbldFhppx1uGD413tVSSqm4slUg2FIUkGOoYLm13GNCcvhfUkqpPs4WgWDxmp0cc8ubbNlbybtf7OW199fCnvV6f0AppbBBIFi8ZifXvLSer8tqANhf28Dy1563PtRAoJRSfT8Q3PH6Z1TXN7ba9g2zln2kQs74ONVKKaV6jj4fCL4urW6zxXC8Yz1vN44FR58/faWU6lCfbwkHp3tbvT9MtpMtZaz3HBmnGimlVM/S5wPBlScfitfdklDueMc6APJO/EG8qqSUUj1KTBev7wlmTsoFrHsFX5dWMyNxE2XJozjlmElxrplSSvUMfT4QgBUMZk7KhboquO0CGHtRvKuklFI9Rp8fGmpl23vQWAuHaLZRpZRqYq9AULAcnIkw7Jh410QppXqMvj809MCxsHt9621/HAQ54+CSd+NTJ6WU6kH6fo9gyBRwJrTe5kywtiullLJBIJh2FUib0xQHTLs6PvVRSqkepu8HgpQcmHgeOPyjYM4E633KwPjWSymleoi+HwjA6hU0BQLtDSilVCv2CARNvQJxaG9AKaXa6PuzhppMuwqKNmtvQCml2rBPIEjJgQtei3ctlFKqx4np0JCInCIin4lIvojMD/J5oog86//8QxEZHsv6KKWUai9mgUBEnMB9wKnAGGC2iIxpU+xCYJ8xZiRwF3BbrOqjlFIquFj2CKYA+caYLcaYOuAZ4HttynwPWOR//QLwLRGRGNZJKaVUG7EMBLnA9oD3O/zbgpYxxjQAZUBm2x2JyBwRWSUiq4qKimJUXaWUsqdeMX3UGPOgMSbPGJOXnZ0d7+oopVSfEstZQzuBoQHvh/i3BSuzQ0RcQBpQHG6nq1ev3isiX3WxTlnA3i7+bk+j59Lz9JXzAD2XnupAzuWgUB/EMhB8BIwSkRFYDf45wLltyiwBfgq8D5wFLDPGmHA7NcZ0uUsgIquMMXld/f2eRM+l5+kr5wF6Lj1VrM4lZoHAGNMgInOB1wEn8KgxZqOI3AisMsYsAR4B/iEi+UAJVrBQSinVjWL6QJkxZimwtM226wNe1wA/jGUdlFJKhdcrbhZH0YPxrkAU6bn0PH3lPEDPpaeKyblIB0PySiml+ji79QiUUkq1oYFAKaVszjaBoKMEeL2JiHwpIutFZK2IrIp3fTpDRB4VkUIR2RCwLUNE/iMiX/h/9o9nHSMR4jwWiMhO//eyVkS+E886RkpEhorIchHZJCIbReRS//Ze9b2EOY9e972IiEdEVorIOv+53ODfPsKfoDPfn7AzoaN9RXQ8O9wj8CfA+xyYgZXq4iNgtjFmU1wr1kUi8iWQZ4zpdQ/JiMjxQAXwd2PMWP+224ESY8yt/iDd3xjToxeOCHEeC4AKY8yf4lm3zhKRQcAgY8zHIpICrAZmAufTi76XMOcxi172vfhzriUbYypExA28C1wKzANeMsY8IyIPAOuMMfcf6PHs0iOIJAGe6gbGmLexnhkJFJh8cBHW/7w9Wojz6JWMMbuMMR/7X+8HNmPlAetV30uY8+h1jKXC/9bt/2eA6VgJOiGK34ldAkEkCfB6EwO8ISKrRWROvCsTBQONMbv8r3cDvXkt0bki8ol/6KhHD6UE418TZBLwIb34e2lzHtALvxcRcYrIWqAQ+A9QAJT6E3RCFNsxuwSCvuZYY8xkrLUefuUfpugT/ClGeut45f3AIcBEYBfw5/hWp3NEpB/wIvAbY0x54Ge96XsJch698nsxxjQaYyZi5WmbAhwWq2PZJRBEkgCv1zDG7PT/LARexvqPpDfb4x/fbRrnLYxzfbrEGLPH/z+vD3iIXvS9+MehXwSeNMa85N/c676XYOfRm78XAGNMKbAcOBpI9yfohCi2Y3YJBM0J8Px32c/BSnjX64hIsv9GGCKSDJwEbAj/Wz1eU/JB/D//Gce6dFlTo+n3fXrJ9+K/MfkIsNkYc2fAR73qewl1Hr3xexGRbBFJ97/2Yk102YwVEM7yF4vad2KLWUMA/iljd9OSAO/mOFepS0TkYKxeAFi5op7qTeciIk8DJ2Cl090D/B5YDDwHDAO+AmYZY3r0jdgQ53EC1vCDAb4ELg4YY++xRORY4B1gPeDzb/4t1vh6r/lewpzHbHrZ9yIi47FuBjuxLtifM8bc6P///xkgA1gD/MgYU3vAx7NLIFBKKRWcXYaGlFJKhaCBQCmlbE4DgVJK2ZwGAqWUsjkNBEopZXMaCJTyE5HGgAyVa6OZpVZEhgdmKlWqJ4npmsVK9TLV/kf6lbIV7REo1QH/+g+3+9eAWCkiI/3bh4vIMn8yszdFZJh/+0ARedmfS36diBzj35VTRB7y55d/w//EKCLyf/4c+p+IyDNxOk1lYxoIlGrhbTM0dHbAZ2XGmHHAX7CeUAe4F1hkjBkPPAks9G9fCLxljJkATAY2+rePAu4zxhwBlAI/8G+fD0zy7+eSWJ2cUqHok8VK+YlIhTGmX5DtXwLTjTFb/EnNdhtjMkVkL9ZCKPX+7buMMVkiUgQMCXz0358W+T/GmFH+91cDbmPMTSLyb6xFbhYDiwPy0CvVLbRHoFRkTIjXnRGYE6aRlnt0pwH3YfUePgrILqlUt9BAoFRkzg74+b7/9XtYmWwBzsNKeAbwJvALaF5cJC3UTkXEAQw1xiwHrgbSgHa9EqViSa88lGrh9a8I1eTfxpimKaT9ReQTrKv62f5tvwYeE5ErgSLgAv/2S4EHReRCrCv/X2AtiBKME3jCHywEWOjPP69Ut9F7BEp1wH+PIM8YszfedVEqFnRoSCmlbE57BEopZXPaI1BKKZvTQKCUUjangUAppWxOA4FSStmcBgKllLK5/weA/uHgFsh5DAAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"1rDDCMkehE3x"},"source":["Thus far, we keep our focus only on the implicit feedback based matrix factorization model on small movielens dataset. In future, we will be expanding this MVP in the following directions:\n","1. Large scale industrial datasets - Yoochoose, Trivago\n","2. Other available models in [this](https://github.com/ShopRunner/collie_recs/tree/main/collie_recs/model) repo\n","3. Really liked the poster carousel. Put it in dash/streamlit app."]},{"cell_type":"markdown","metadata":{"id":"thC-jHYLJKkz"},"source":["## Training neural factorization model on movielens dataset\n","> Training MF, MF+bias, and MLP model on movielens-100k dataset in PyTorch."]},{"cell_type":"code","metadata":{"id":"U9XYsONJClRh"},"source":["!pip install -q git+https://github.com/sparsh-ai/recochef.git"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2LS69WtgCuxJ"},"source":["import torch\n","import torch.nn.functional as F\n","\n","from recochef.datasets.synthetic import Synthetic\n","from recochef.datasets.movielens import MovieLens\n","from recochef.preprocessing.split import chrono_split\n","from recochef.preprocessing.encode import label_encode as le\n","from recochef.models.factorization import MF, MF_bias\n","from recochef.models.dnn import CollabFNet"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"X7-2sy7dDJte"},"source":["# # generate synthetic implicit data\n","# synt = Synthetic()\n","# df = synt.implicit()\n","\n","movielens = MovieLens()\n","df = movielens.load_interactions()\n","\n","# changing rating colname to event following implicit naming conventions\n","df = df.rename(columns={'RATING': 'EVENT'})"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EGLNfBJBCw38","outputId":"06429212-3b1c-4a95-df70-927b8e8a3e43"},"source":["# drop duplicates\n","df = df.drop_duplicates()\n","\n","# chronological split\n","df_train, df_valid = chrono_split(df, ratio=0.8, min_rating=10)\n","print(f\"Train set:\\n\\n{df_train}\\n{'='*100}\\n\")\n","print(f\"Validation set:\\n\\n{df_valid}\\n{'='*100}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Train set:\n","\n"," USERID ITEMID EVENT TIMESTAMP\n","59972 1 168 5.0 874965478\n","92487 1 172 5.0 874965478\n","74577 1 165 5.0 874965518\n","48214 1 156 4.0 874965556\n","22971 1 166 5.0 874965677\n","... ... ... ... ...\n","98752 943 139 1.0 888640027\n","89336 943 426 4.0 888640027\n","80660 943 720 1.0 888640048\n","93177 943 80 2.0 888640048\n","87415 943 53 3.0 888640067\n","\n","[80000 rows x 4 columns]\n","====================================================================================================\n","\n","Validation set:\n","\n"," USERID ITEMID EVENT TIMESTAMP\n","10508 1 208 5.0 878542960\n","83307 1 3 4.0 878542960\n","8976 1 12 5.0 878542960\n","78171 1 58 4.0 878542960\n","9811 1 201 3.0 878542960\n","... ... ... ... ...\n","81005 943 450 1.0 888693158\n","92536 943 227 1.0 888693158\n","95003 943 230 1.0 888693158\n","94914 943 229 2.0 888693158\n","92880 943 234 3.0 888693184\n","\n","[20000 rows x 4 columns]\n","====================================================================================================\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"68zLUPlvC5LK","outputId":"46c0f8b6-dd84-4c54-8d55-8eb61fb3fc47"},"source":["# label encoding\n","df_train, uid_maps = le(df_train, col='USERID')\n","df_train, iid_maps = le(df_train, col='ITEMID')\n","df_valid = le(df_valid, col='USERID', maps=uid_maps)\n","df_valid = le(df_valid, col='ITEMID', maps=iid_maps)\n","\n","# # event implicit to rating conversion\n","# event_weights = {'click':1, 'add':2, 'purchase':4}\n","# event_maps = dict({'EVENT_TO_IDX':event_weights})\n","# df_train = le(df_train, col='EVENT', maps=event_maps)\n","# df_valid = le(df_valid, col='EVENT', maps=event_maps)\n","\n","print(f\"Processed Train set:\\n\\n{df_train}\\n{'='*100}\\n\")\n","print(f\"Processed Validation set:\\n\\n{df_valid}\\n{'='*100}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Processed Train set:\n","\n"," USERID ITEMID EVENT TIMESTAMP\n","59972 0 0 5.0 874965478\n","92487 0 1 5.0 874965478\n","74577 0 2 5.0 874965518\n","48214 0 3 4.0 874965556\n","22971 0 4 5.0 874965677\n","... ... ... ... ...\n","98752 942 933 1.0 888640027\n","89336 942 990 4.0 888640027\n","80660 942 643 1.0 888640048\n","93177 942 155 2.0 888640048\n","87415 942 166 3.0 888640067\n","\n","[80000 rows x 4 columns]\n","====================================================================================================\n","\n","Processed Validation set:\n","\n"," USERID ITEMID EVENT TIMESTAMP\n","10508 0 341.0 5.0 878542960\n","83307 0 983.0 4.0 878542960\n","8976 0 425.0 5.0 878542960\n","78171 0 639.0 4.0 878542960\n","9811 0 490.0 3.0 878542960\n","... ... ... ... ...\n","81005 942 314.0 1.0 888693158\n","92536 942 154.0 1.0 888693158\n","95003 942 183.0 1.0 888693158\n","94914 942 176.0 2.0 888693158\n","92880 942 132.0 3.0 888693184\n","\n","[19917 rows x 4 columns]\n","====================================================================================================\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VnhEaj5QC8j1","outputId":"f15ef434-6b1d-4f51-f11c-4bfbe4af649b"},"source":["# get number of unique users and items\n","num_users = len(df_train.USERID.unique())\n","num_items = len(df_train.ITEMID.unique())\n","\n","num_users_t = len(df_valid.USERID.unique())\n","num_items_t = len(df_valid.ITEMID.unique())\n","\n","print(f\"There are {num_users} users and {num_items} items in the train set.\\n{'='*100}\\n\")\n","print(f\"There are {num_users_t} users and {num_items_t} items in the validation set.\\n{'='*100}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["There are 943 users and 1613 items in the train set.\n","====================================================================================================\n","\n","There are 943 users and 1429 items in the validation set.\n","====================================================================================================\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"xTiGbb5UCpwM"},"source":["# training and testing related helper functions\n","def train_epocs(model, epochs=10, lr=0.01, wd=0.0, unsqueeze=False):\n"," optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n"," model.train()\n"," for i in range(epochs):\n"," users = torch.LongTensor(df_train.USERID.values) # .cuda()\n"," items = torch.LongTensor(df_train.ITEMID.values) #.cuda()\n"," ratings = torch.FloatTensor(df_train.EVENT.values) #.cuda()\n"," if unsqueeze:\n"," ratings = ratings.unsqueeze(1)\n"," y_hat = model(users, items)\n"," loss = F.mse_loss(y_hat, ratings)\n"," optimizer.zero_grad()\n"," loss.backward()\n"," optimizer.step()\n"," print(loss.item()) \n"," test_loss(model, unsqueeze)\n","\n","def test_loss(model, unsqueeze=False):\n"," model.eval()\n"," users = torch.LongTensor(df_valid.USERID.values) #.cuda()\n"," items = torch.LongTensor(df_valid.ITEMID.values) #.cuda()\n"," ratings = torch.FloatTensor(df_valid.EVENT.values) #.cuda()\n"," if unsqueeze:\n"," ratings = ratings.unsqueeze(1)\n"," y_hat = model(users, items)\n"," loss = F.mse_loss(y_hat, ratings)\n"," print(\"test loss %.3f \" % loss.item())"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"LxhbI4ECC_Jb","outputId":"fa326841-1e15-4900-c0e4-fc7790beb762"},"source":["# training MF model\n","model = MF(num_users, num_items, emb_size=100) # .cuda() if you have a GPU\n","print(f\"Training MF model:\\n\")\n","train_epocs(model, epochs=10, lr=0.1)\n","print(f\"\\n{'='*100}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Training MF model:\n","\n","13.594555854797363\n","5.292399883270264\n","2.558849573135376\n","3.584117889404297\n","1.0360910892486572\n","1.9875222444534302\n","2.920832633972168\n","2.4130148887634277\n","1.2886441946029663\n","1.112807273864746\n","test loss 2.085 \n","\n","====================================================================================================\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fnbkknGIDAs6","outputId":"e8466582-7078-49ab-dda7-eeffaa65c8de"},"source":["# training MF with bias model\n","model = MF_bias(num_users, num_items, emb_size=100) #.cuda()\n","print(f\"Training MF+bias model:\\n\")\n","train_epocs(model, epochs=10, lr=0.05, wd=1e-5)\n","print(f\"\\n{'='*100}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Training MF+bias model:\n","\n","13.59664535522461\n","9.730958938598633\n","4.798837184906006\n","1.3603413105010986\n","2.697232723236084\n","4.214857578277588\n","2.871798276901245\n","1.3329992294311523\n","0.9624974727630615\n","1.459389328956604\n","test loss 2.269 \n","\n","====================================================================================================\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"N9ltu-ISDCUY","outputId":"06c01140-05a4-4b06-9819-546a7ecdba66"},"source":["# training MLP model\n","model = CollabFNet(num_users, num_items, emb_size=100) #.cuda()\n","print(f\"Training MLP model:\\n\")\n","train_epocs(model, epochs=15, lr=0.05, wd=1e-6, unsqueeze=True)\n","print(f\"\\n{'='*100}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Training MLP model:\n","\n","12.962654113769531\n","1.4028953313827515\n","15.373563766479492\n","2.177295207977295\n","2.6291019916534424\n","5.752542495727539\n","6.88251256942749\n","6.2746357917785645\n","4.8090314865112305\n","3.095308303833008\n","1.6791961193084717\n","1.1257785558700562\n","1.678966760635376\n","2.615834951400757\n","2.80102276802063\n","test loss 2.559 \n","\n","====================================================================================================\n","\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"UxWgUAp7vnIM"},"source":["## Neural Matrix Factorization on Movielens\n","> Experiments with different variations of Neural matrix factorization model in PyTorch on movielens dataset."]},{"cell_type":"code","metadata":{"id":"hfac3W-Z4yEs"},"source":["import numpy as np\n","import pandas as pd\n","\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"74OaJCfn5H4Z","outputId":"9ec26cd4-a004-49cf-aefb-9a24c670bf11"},"source":["data = pd.read_csv(\"https://raw.githubusercontent.com/sparsh-ai/reco-data/master/MovieLens_LatestSmall_ratings.csv\")\n","data.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdratingtimestamp
0114.0964982703
1134.0964981247
2164.0964982224
31475.0964983815
41505.0964982931
\n","
"],"text/plain":[" userId movieId rating timestamp\n","0 1 1 4.0 964982703\n","1 1 3 4.0 964981247\n","2 1 6 4.0 964982224\n","3 1 47 5.0 964983815\n","4 1 50 5.0 964982931"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"F0Irmpk2oUna","outputId":"082a3deb-3f36-4d93-8917-77c27db5fc55"},"source":["data.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(100836, 4)"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"markdown","metadata":{"id":"AsSMbrTG6LQr"},"source":["Data encoding"]},{"cell_type":"code","metadata":{"id":"m_wEgrHx5U93"},"source":["np.random.seed(3)\n","msk = np.random.rand(len(data)) < 0.8\n","train = data[msk].copy()\n","valid = data[~msk].copy()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fmeQCQXP6Vtv"},"source":["# here is a handy function modified from fast.ai\n","def proc_col(col, train_col=None):\n"," \"\"\"Encodes a pandas column with continous ids. \n"," \"\"\"\n"," if train_col is not None:\n"," uniq = train_col.unique()\n"," else:\n"," uniq = col.unique()\n"," name2idx = {o:i for i,o in enumerate(uniq)}\n"," return name2idx, np.array([name2idx.get(x, -1) for x in col]), len(uniq)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OUfpCvFJ6W72"},"source":["def encode_data(df, train=None):\n"," \"\"\" Encodes rating data with continous user and movie ids. \n"," If train is provided, encodes df with the same encoding as train.\n"," \"\"\"\n"," df = df.copy()\n"," for col_name in [\"userId\", \"movieId\"]:\n"," train_col = None\n"," if train is not None:\n"," train_col = train[col_name]\n"," _,col,_ = proc_col(df[col_name], train_col)\n"," df[col_name] = col\n"," df = df[df[col_name] >= 0]\n"," return df"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TZDUY2rt6Z9B"},"source":["# encoding the train and validation data\n","df_train = encode_data(train)\n","df_valid = encode_data(valid, train)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"c9VIAfR6otTe","outputId":"dc5ad891-2004-4fda-acfa-22d04525df3b"},"source":["df_train.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdratingtimestamp
0004.0964982703
1014.0964981247
2024.0964982224
3035.0964983815
6045.0964980868
\n","
"],"text/plain":[" userId movieId rating timestamp\n","0 0 0 4.0 964982703\n","1 0 1 4.0 964981247\n","2 0 2 4.0 964982224\n","3 0 3 5.0 964983815\n","6 0 4 5.0 964980868"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iy77qSeCo2WY","outputId":"c4ce1748-6c39-44f6-c094-476873135fdb"},"source":["df_train.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(80450, 4)"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"sOSEDMQPo2Tq","outputId":"4327e4a5-b01b-4d03-c829-0220e7c8b36c"},"source":["df_valid.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdratingtimestamp
403885.0964982931
509953.0964982400
2908414.0964981179
3005674.0964982653
3204024.0964982546
\n","
"],"text/plain":[" userId movieId rating timestamp\n","4 0 388 5.0 964982931\n","5 0 995 3.0 964982400\n","29 0 841 4.0 964981179\n","30 0 567 4.0 964982653\n","32 0 402 4.0 964982546"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2uiymy6po2Lo","outputId":"f1ebfdf2-a139-46d4-d8e7-850701cb57a2"},"source":["df_valid.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(19591, 4)"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"markdown","metadata":{"id":"y3QSDyZj61Iy"},"source":["Matrix factorization model"]},{"cell_type":"code","metadata":{"id":"HBPnUZl-6z1g"},"source":["class MF(nn.Module):\n"," def __init__(self, num_users, num_items, emb_size=100):\n"," super(MF, self).__init__()\n"," self.user_emb = nn.Embedding(num_users, emb_size)\n"," self.item_emb = nn.Embedding(num_items, emb_size)\n"," self.user_emb.weight.data.uniform_(0, 0.05)\n"," self.item_emb.weight.data.uniform_(0, 0.05)\n"," \n"," def forward(self, u, v):\n"," u = self.user_emb(u)\n"," v = self.item_emb(v)\n"," return (u*v).sum(1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"dBQhfy2l7AAn","outputId":"d667d3a3-2baa-467b-e8ab-905c3282780f"},"source":["# unit testing the architecture\n","sample = encode_data(train.sample(5))\n","display(sample)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdratingtimestamp
63234003.0961596392
96012113.0840875700
31417222.0955944735
17473330.51516141230
66983444.01328232721
\n","
"],"text/plain":[" userId movieId rating timestamp\n","63234 0 0 3.0 961596392\n","96012 1 1 3.0 840875700\n","31417 2 2 2.0 955944735\n","17473 3 3 0.5 1516141230\n","66983 4 4 4.0 1328232721"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"9tmmtDTuqnIB"},"source":["num_users = 5\n","num_items = 5\n","emb_size = 3\n","\n","user_emb = nn.Embedding(num_users, emb_size)\n","item_emb = nn.Embedding(num_items, emb_size)\n","\n","users = torch.LongTensor(sample.userId.values)\n","items = torch.LongTensor(sample.movieId.values)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":102},"id":"H557JwqSqimK","outputId":"b45ad07e-0cf7-4cb0-f9ad-5de2f8a86c08"},"source":["U = user_emb(users)\n","display(U)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["tensor([[ 2.2694, 0.9679, 0.3305],\n"," [-1.1478, -0.7004, -0.8113],\n"," [-1.2287, -0.7210, 0.3875],\n"," [ 0.9106, 0.0427, -0.7128],\n"," [-1.0396, -0.2739, 0.7271]], grad_fn=)"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":102},"id":"IsdtlmtBq3cj","outputId":"418f529d-1481-475e-ad2c-39578baa4cdf"},"source":["V = item_emb(items)\n","display(V)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["tensor([[-1.9371, -1.1172, -1.5967],\n"," [-2.4336, -1.1177, 0.6197],\n"," [ 0.5889, 1.4830, -1.0103],\n"," [-0.8294, 0.5744, -1.7315],\n"," [-1.6733, -0.2447, -0.2630]], grad_fn=)"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":102},"id":"TKtMvkNfq0q2","outputId":"c7bb19a7-3a7b-45db-f4c1-b95f0994750f"},"source":["display(U*V) # element wise multiplication"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["tensor([[-4.3959, -1.0813, -0.5278],\n"," [ 2.7932, 0.7828, -0.5027],\n"," [-0.7236, -1.0693, -0.3915],\n"," [-0.7552, 0.0246, 1.2343],\n"," [ 1.7397, 0.0670, -0.1912]], grad_fn=)"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"id":"5_AN_dhQq0nE","outputId":"7cc9d053-c9f3-49d0-e2a6-f0ea809ecd8f"},"source":["display((U*V).sum(1))"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["tensor([-6.0050, 3.0733, -2.1844, 0.5036, 1.6155], grad_fn=)"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"W01e58dr86WY"},"source":["Model training"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VC5vARcP7QAc","outputId":"b4cea803-bbac-4301-bb1c-12683689948d"},"source":["num_users = len(df_train.userId.unique())\n","num_items = len(df_train.movieId.unique())\n","print(\"{} users and {} items in the training set\".format(num_users, num_items))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["610 users and 8998 items in the training set\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"yRi5sy-K8-fr"},"source":["model = MF(num_users, num_items, emb_size=100) # .cuda() if you have a GPU"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"4nAGJ4l08_83"},"source":["def train_epocs(model, epochs=10, lr=0.01, wd=0.0, unsqueeze=False):\n"," optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n"," model.train()\n"," for i in range(epochs):\n"," users = torch.LongTensor(df_train.userId.values) # .cuda()\n"," items = torch.LongTensor(df_train.movieId.values) #.cuda()\n"," ratings = torch.FloatTensor(df_train.rating.values) #.cuda()\n"," if unsqueeze:\n"," ratings = ratings.unsqueeze(1)\n"," y_hat = model(users, items)\n"," loss = F.mse_loss(y_hat, ratings)\n"," optimizer.zero_grad()\n"," loss.backward()\n"," optimizer.step()\n"," print(loss.item()) \n"," test_loss(model, unsqueeze)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7l_3G5gn9GH3"},"source":["def test_loss(model, unsqueeze=False):\n"," model.eval()\n"," users = torch.LongTensor(df_valid.userId.values) #.cuda()\n"," items = torch.LongTensor(df_valid.movieId.values) #.cuda()\n"," ratings = torch.FloatTensor(df_valid.rating.values) #.cuda()\n"," if unsqueeze:\n"," ratings = ratings.unsqueeze(1)\n"," y_hat = model(users, items)\n"," loss = F.mse_loss(y_hat, ratings)\n"," print(\"test loss %.3f \" % loss.item())"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EztQtZKl9M53","outputId":"25713f3c-edff-4333-e45b-ffb2c79375fb"},"source":["train_epocs(model, epochs=10, lr=0.1)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["12.914263725280762\n","4.8582916259765625\n","2.5804786682128906\n","3.109440565109253\n","0.850287139415741\n","1.819737195968628\n","2.657919406890869\n","2.138274908065796\n","1.0904945135116577\n","0.9722878932952881\n","test loss 1.851 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AoSgUhWV9O1q","outputId":"48e887fa-b55c-4465-e2d0-05789b5a7419"},"source":["train_epocs(model, epochs=10, lr=0.01)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["1.6430705785751343\n","1.0046814680099487\n","0.712002694606781\n","0.6611021757125854\n","0.7258523106575012\n","0.803934633731842\n","0.843424379825592\n","0.8351688981056213\n","0.7928505539894104\n","0.737376868724823\n","test loss 0.827 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"erRnsApY9Q7e","outputId":"1e6a68f3-13b1-4963-b187-5d1b1bc3e438"},"source":["train_epocs(model, epochs=10, lr=0.01)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0.6877127289772034\n","0.6256141066551208\n","0.6374999284744263\n","0.6272100210189819\n","0.6171814799308777\n","0.614914059638977\n","0.6113061308860779\n","0.6033822298049927\n","0.595890998840332\n","0.592114269733429\n","test loss 0.764 \n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"HAwA9Rts9UI1"},"source":["MF with bias"]},{"cell_type":"code","metadata":{"id":"Dur1n3lo9S3C"},"source":["class MF_bias(nn.Module):\n"," def __init__(self, num_users, num_items, emb_size=100):\n"," super(MF_bias, self).__init__()\n"," self.user_emb = nn.Embedding(num_users, emb_size)\n"," self.user_bias = nn.Embedding(num_users, 1)\n"," self.item_emb = nn.Embedding(num_items, emb_size)\n"," self.item_bias = nn.Embedding(num_items, 1)\n"," self.user_emb.weight.data.uniform_(0,0.05)\n"," self.item_emb.weight.data.uniform_(0,0.05)\n"," self.user_bias.weight.data.uniform_(-0.01,0.01)\n"," self.item_bias.weight.data.uniform_(-0.01,0.01)\n"," \n"," def forward(self, u, v):\n"," U = self.user_emb(u)\n"," V = self.item_emb(v)\n"," b_u = self.user_bias(u).squeeze()\n"," b_v = self.item_bias(v).squeeze()\n"," return (U*V).sum(1) + b_u + b_v"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WAyaietL9ZAq"},"source":["model = MF_bias(num_users, num_items, emb_size=100) #.cuda()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5nEO-IVp9acn","outputId":"773ee576-ea2e-40ac-97c7-7d78606595e1"},"source":["train_epocs(model, epochs=10, lr=0.05, wd=1e-5)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["12.91020393371582\n","9.150527954101562\n","4.3840012550354\n","1.1575191020965576\n","2.46807861328125\n","3.7430803775787354\n","2.4481022357940674\n","1.0781667232513428\n","0.816169023513794\n","1.3183783292770386\n","test loss 2.069 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nD2fD5A59cK4","outputId":"03df230c-70ff-4767-e088-928d54f6cd37"},"source":["train_epocs(model, epochs=10, lr=0.01, wd=1e-5)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["1.8935126066207886\n","1.3250681161880493\n","0.9350242614746094\n","0.7446779012680054\n","0.722224235534668\n","0.7774652242660522\n","0.8231741189956665\n","0.8222126364707947\n","0.7816660404205322\n","0.727698802947998\n","test loss 0.798 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Os53hZxr9e_T","outputId":"d9bf93d3-8411-4535-dcc7-ebcf4a3fbc48"},"source":["train_epocs(model, epochs=10, lr=0.001, wd=1e-5)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0.6853442788124084\n","0.6711287498474121\n","0.6592414975166321\n","0.6495122909545898\n","0.6417150497436523\n","0.6356027722358704\n","0.6309247612953186\n","0.6274365186691284\n","0.6249085068702698\n","0.6231329441070557\n","test loss 0.751 \n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"-XhFy6bU9h48"},"source":["Note that these models are susceptible to weight initialization, optimization algorithm and regularization.\n","\n"]},{"cell_type":"markdown","metadata":{"id":"NugoowzF9kCk"},"source":["### Neural Network Model\n","Note here there is no matrix multiplication, we could potentially make the embeddings of different sizes. Here we could get better results by keep playing with regularization."]},{"cell_type":"code","metadata":{"id":"qLVWHOxQ9fVX"},"source":["class CollabFNet(nn.Module):\n"," def __init__(self, num_users, num_items, emb_size=100, n_hidden=10):\n"," super(CollabFNet, self).__init__()\n"," self.user_emb = nn.Embedding(num_users, emb_size)\n"," self.item_emb = nn.Embedding(num_items, emb_size)\n"," self.lin1 = nn.Linear(emb_size*2, n_hidden)\n"," self.lin2 = nn.Linear(n_hidden, 1)\n"," self.drop1 = nn.Dropout(0.1)\n"," \n"," def forward(self, u, v):\n"," U = self.user_emb(u)\n"," V = self.item_emb(v)\n"," x = F.relu(torch.cat([U, V], dim=1))\n"," x = self.drop1(x)\n"," x = F.relu(self.lin1(x))\n"," x = self.lin2(x)\n"," return x"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ljjju7Yy9x7b"},"source":["model = CollabFNet(num_users, num_items, emb_size=100) #.cuda()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YuG2Hz5e9yyl","outputId":"67b716ed-84e4-4dcf-eaf0-f06609542d0d"},"source":["train_epocs(model, epochs=15, lr=0.05, wd=1e-6, unsqueeze=True)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["14.657201766967773\n","2.586819648742676\n","6.025796890258789\n","2.89852237701416\n","1.1256697177886963\n","2.0860772132873535\n","2.9243881702423096\n","2.806140422821045\n","1.9981783628463745\n","1.1265769004821777\n","0.8947575092315674\n","1.4373805522918701\n","1.795198678970337\n","1.4024922847747803\n","0.8697773218154907\n","test loss 0.797 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JYyXb1qO90vb","outputId":"b3bcc463-51a8-4625-f547-38ebbf105a7f"},"source":["train_epocs(model, epochs=10, lr=0.001, wd=1e-6, unsqueeze=True)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0.7495059967041016\n","0.7382366061210632\n","0.731941282749176\n","0.7295416593551636\n","0.7321946024894714\n","0.7312469482421875\n","0.731982409954071\n","0.7298287153244019\n","0.7264290452003479\n","0.7244617938995361\n","test loss 0.774 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"o0d-WRvW92h-","outputId":"20f81203-7fbb-44db-b421-c6c20239cd22"},"source":["train_epocs(model, epochs=10, lr=0.001, wd=1e-6, unsqueeze=True)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0.7242854833602905\n","0.7213587760925293\n","0.7197834849357605\n","0.7182263135910034\n","0.7177621722221375\n","0.7155387997627258\n","0.7147852182388306\n","0.7143447995185852\n","0.7133223414421082\n","0.712261974811554\n","test loss 0.766 \n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"6fpmHCaYAn2d"},"source":["### Neural network model - different approach\n","> youtube: https://youtu.be/MVB1cbe923A"]},{"cell_type":"markdown","metadata":{"id":"SMuCwuPWPfGz"},"source":["### Ethan Rosenthal\n","\n","Ref - https://github.com/EthanRosenthal/torchmf"]},{"cell_type":"code","metadata":{"id":"J31f-camBorB"},"source":["import os\n","import requests\n","import zipfile\n","import collections\n","\n","import numpy as np\n","import pandas as pd\n","import scipy.sparse as sp\n","from sklearn.metrics import roc_auc_score\n","\n","import torch\n","from torch import nn\n","import torch.multiprocessing as mp\n","import torch.utils.data as data\n","from tqdm import tqdm"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ahu8EWCGQJkI"},"source":["def _get_data_path():\n"," \"\"\"\n"," Get path to the movielens dataset file.\n"," \"\"\"\n"," data_path = '/content/data'\n"," if not os.path.exists(data_path):\n"," print('Making data path')\n"," os.mkdir(data_path)\n"," return data_path\n","\n","\n","def _download_movielens(dest_path):\n"," \"\"\"\n"," Download the dataset.\n"," \"\"\"\n","\n"," url = 'http://files.grouplens.org/datasets/movielens/ml-100k.zip'\n"," req = requests.get(url, stream=True)\n","\n"," print('Downloading MovieLens data')\n","\n"," with open(os.path.join(dest_path, 'ml-100k.zip'), 'wb') as fd:\n"," for chunk in req.iter_content(chunk_size=None):\n"," fd.write(chunk)\n","\n"," with zipfile.ZipFile(os.path.join(dest_path, 'ml-100k.zip'), 'r') as z:\n"," z.extractall(dest_path)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DouK7x1nPsNb"},"source":["def read_movielens_df():\n"," path = _get_data_path()\n"," zipfile = os.path.join(path, 'ml-100k.zip')\n"," if not os.path.isfile(zipfile):\n"," _download_movielens(path)\n"," fname = os.path.join(path, 'ml-100k', 'u.data')\n"," names = ['user_id', 'item_id', 'rating', 'timestamp']\n"," df = pd.read_csv(fname, sep='\\t', names=names)\n"," return df\n","\n","\n","def get_movielens_interactions():\n"," df = read_movielens_df()\n","\n"," n_users = df.user_id.unique().shape[0]\n"," n_items = df.item_id.unique().shape[0]\n","\n"," interactions = np.zeros((n_users, n_items))\n"," for row in df.itertuples():\n"," interactions[row[1] - 1, row[2] - 1] = row[3]\n"," return interactions\n","\n","\n","def train_test_split(interactions, n=10):\n"," \"\"\"\n"," Split an interactions matrix into training and test sets.\n"," Parameters\n"," ----------\n"," interactions : np.ndarray\n"," n : int (default=10)\n"," Number of items to select / row to place into test.\n","\n"," Returns\n"," -------\n"," train : np.ndarray\n"," test : np.ndarray\n"," \"\"\"\n"," test = np.zeros(interactions.shape)\n"," train = interactions.copy()\n"," for user in range(interactions.shape[0]):\n"," if interactions[user, :].nonzero()[0].shape[0] > n:\n"," test_interactions = np.random.choice(interactions[user, :].nonzero()[0],\n"," size=n,\n"," replace=False)\n"," train[user, test_interactions] = 0.\n"," test[user, test_interactions] = interactions[user, test_interactions]\n","\n"," # Test and training are truly disjoint\n"," assert(np.all((train * test) == 0))\n"," return train, test\n","\n","\n","def get_movielens_train_test_split(implicit=False):\n"," interactions = get_movielens_interactions()\n"," if implicit:\n"," interactions = (interactions >= 4).astype(np.float32)\n"," train, test = train_test_split(interactions)\n"," train = sp.coo_matrix(train)\n"," test = sp.coo_matrix(test)\n"," return train, test"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0x9xMyW6PsK6","outputId":"00096a84-7542-4d07-920d-48830410244e"},"source":["%%writefile metrics.py\n","\n","import numpy as np\n","from sklearn.metrics import roc_auc_score\n","from torch import multiprocessing as mp\n","import torch\n","\n","def get_row_indices(row, interactions):\n"," start = interactions.indptr[row]\n"," end = interactions.indptr[row + 1]\n"," return interactions.indices[start:end]\n","\n","\n","def auc(model, interactions, num_workers=1):\n"," aucs = []\n"," processes = []\n"," n_users = interactions.shape[0]\n"," mp_batch = int(np.ceil(n_users / num_workers))\n","\n"," queue = mp.Queue()\n"," rows = np.arange(n_users)\n"," np.random.shuffle(rows)\n"," for rank in range(num_workers):\n"," start = rank * mp_batch\n"," end = np.min((start + mp_batch, n_users))\n"," p = mp.Process(target=batch_auc,\n"," args=(queue, rows[start:end], interactions, model))\n"," p.start()\n"," processes.append(p)\n","\n"," while True:\n"," is_alive = False\n"," for p in processes:\n"," if p.is_alive():\n"," is_alive = True\n"," break\n"," if not is_alive and queue.empty():\n"," break\n","\n"," while not queue.empty():\n"," aucs.append(queue.get())\n","\n"," queue.close()\n"," for p in processes:\n"," p.join()\n"," return np.mean(aucs)\n","\n","\n","def batch_auc(queue, rows, interactions, model):\n"," n_items = interactions.shape[1]\n"," items = torch.arange(0, n_items).long()\n"," users_init = torch.ones(n_items).long()\n"," for row in rows:\n"," row = int(row)\n"," users = users_init.fill_(row)\n","\n"," preds = model.predict(users, items)\n"," actuals = get_row_indices(row, interactions)\n","\n"," if len(actuals) == 0:\n"," continue\n"," y_test = np.zeros(n_items)\n"," y_test[actuals] = 1\n"," queue.put(roc_auc_score(y_test, preds.data.numpy()))\n","\n","\n","def patk(model, interactions, num_workers=1, k=5):\n"," patks = []\n"," processes = []\n"," n_users = interactions.shape[0]\n"," mp_batch = int(np.ceil(n_users / num_workers))\n","\n"," queue = mp.Queue()\n"," rows = np.arange(n_users)\n"," np.random.shuffle(rows)\n"," for rank in range(num_workers):\n"," start = rank * mp_batch\n"," end = np.min((start + mp_batch, n_users))\n"," p = mp.Process(target=batch_patk,\n"," args=(queue, rows[start:end], interactions, model),\n"," kwargs={'k': k})\n"," p.start()\n"," processes.append(p)\n","\n"," while True:\n"," is_alive = False\n"," for p in processes:\n"," if p.is_alive():\n"," is_alive = True\n"," break\n"," if not is_alive and queue.empty():\n"," break\n","\n"," while not queue.empty():\n"," patks.append(queue.get())\n","\n"," queue.close()\n"," for p in processes:\n"," p.join()\n"," return np.mean(patks)\n","\n","\n","def batch_patk(queue, rows, interactions, model, k=5):\n"," n_items = interactions.shape[1]\n","\n"," items = torch.arange(0, n_items).long()\n"," users_init = torch.ones(n_items).long()\n"," for row in rows:\n"," row = int(row)\n"," users = users_init.fill_(row)\n","\n"," preds = model.predict(users, items)\n"," actuals = get_row_indices(row, interactions)\n","\n"," if len(actuals) == 0:\n"," continue\n","\n"," top_k = np.argpartition(-np.squeeze(preds.data.numpy()), k)\n"," top_k = set(top_k[:k])\n"," true_pids = set(actuals)\n"," if true_pids:\n"," queue.put(len(top_k & true_pids) / float(k))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Writing metrics.py\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"I6IqWKWFhAQh","outputId":"3bef0869-34b5-488c-ede7-bf9be08c115f"},"source":["import metrics\n","import importlib\n","importlib.reload(metrics)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":55}]},{"cell_type":"code","metadata":{"id":"UEhEE_GhPsH2"},"source":["class Interactions(data.Dataset):\n"," \"\"\"\n"," Hold data in the form of an interactions matrix.\n"," Typical use-case is like a ratings matrix:\n"," - Users are the rows\n"," - Items are the columns\n"," - Elements of the matrix are the ratings given by a user for an item.\n"," \"\"\"\n","\n"," def __init__(self, mat):\n"," self.mat = mat.astype(np.float32).tocoo()\n"," self.n_users = self.mat.shape[0]\n"," self.n_items = self.mat.shape[1]\n","\n"," def __getitem__(self, index):\n"," row = self.mat.row[index]\n"," col = self.mat.col[index]\n"," val = self.mat.data[index]\n"," return (row, col), val\n","\n"," def __len__(self):\n"," return self.mat.nnz\n","\n","\n","class PairwiseInteractions(data.Dataset):\n"," \"\"\"\n"," Sample data from an interactions matrix in a pairwise fashion. The row is\n"," treated as the main dimension, and the columns are sampled pairwise.\n"," \"\"\"\n","\n"," def __init__(self, mat):\n"," self.mat = mat.astype(np.float32).tocoo()\n","\n"," self.n_users = self.mat.shape[0]\n"," self.n_items = self.mat.shape[1]\n","\n"," self.mat_csr = self.mat.tocsr()\n"," if not self.mat_csr.has_sorted_indices:\n"," self.mat_csr.sort_indices()\n","\n"," def __getitem__(self, index):\n"," row = self.mat.row[index]\n"," found = False\n","\n"," while not found:\n"," neg_col = np.random.randint(self.n_items)\n"," if self.not_rated(row, neg_col, self.mat_csr.indptr,\n"," self.mat_csr.indices):\n"," found = True\n","\n"," pos_col = self.mat.col[index]\n"," val = self.mat.data[index]\n","\n"," return (row, (pos_col, neg_col)), val\n","\n"," def __len__(self):\n"," return self.mat.nnz\n","\n"," @staticmethod\n"," def not_rated(row, col, indptr, indices):\n"," # similar to use of bsearch in lightfm\n"," start = indptr[row]\n"," end = indptr[row + 1]\n"," searched = np.searchsorted(indices[start:end], col, 'right')\n"," if searched >= (end - start):\n"," # After the array\n"," return False\n"," return col != indices[searched] # Not found\n","\n"," def get_row_indices(self, row):\n"," start = self.mat_csr.indptr[row]\n"," end = self.mat_csr.indptr[row + 1]\n"," return self.mat_csr.indices[start:end]\n","\n","\n","class BaseModule(nn.Module):\n"," \"\"\"\n"," Base module for explicit matrix factorization.\n"," \"\"\"\n"," \n"," def __init__(self,\n"," n_users,\n"," n_items,\n"," n_factors=40,\n"," dropout_p=0,\n"," sparse=False):\n"," \"\"\"\n","\n"," Parameters\n"," ----------\n"," n_users : int\n"," Number of users\n"," n_items : int\n"," Number of items\n"," n_factors : int\n"," Number of latent factors (or embeddings or whatever you want to\n"," call it).\n"," dropout_p : float\n"," p in nn.Dropout module. Probability of dropout.\n"," sparse : bool\n"," Whether or not to treat embeddings as sparse. NOTE: cannot use\n"," weight decay on the optimizer if sparse=True. Also, can only use\n"," Adagrad.\n"," \"\"\"\n"," super(BaseModule, self).__init__()\n"," self.n_users = n_users\n"," self.n_items = n_items\n"," self.n_factors = n_factors\n"," self.user_biases = nn.Embedding(n_users, 1, sparse=sparse)\n"," self.item_biases = nn.Embedding(n_items, 1, sparse=sparse)\n"," self.user_embeddings = nn.Embedding(n_users, n_factors, sparse=sparse)\n"," self.item_embeddings = nn.Embedding(n_items, n_factors, sparse=sparse)\n"," \n"," self.dropout_p = dropout_p\n"," self.dropout = nn.Dropout(p=self.dropout_p)\n","\n"," self.sparse = sparse\n"," \n"," def forward(self, users, items):\n"," \"\"\"\n"," Forward pass through the model. For a single user and item, this\n"," looks like:\n","\n"," user_bias + item_bias + user_embeddings.dot(item_embeddings)\n","\n"," Parameters\n"," ----------\n"," users : np.ndarray\n"," Array of user indices\n"," items : np.ndarray\n"," Array of item indices\n","\n"," Returns\n"," -------\n"," preds : np.ndarray\n"," Predicted ratings.\n","\n"," \"\"\"\n"," ues = self.user_embeddings(users)\n"," uis = self.item_embeddings(items)\n","\n"," preds = self.user_biases(users)\n"," preds += self.item_biases(items)\n"," preds += (self.dropout(ues) * self.dropout(uis)).sum(dim=1, keepdim=True)\n","\n"," return preds.squeeze()\n"," \n"," def __call__(self, *args):\n"," return self.forward(*args)\n","\n"," def predict(self, users, items):\n"," return self.forward(users, items)\n","\n","\n","def bpr_loss(preds, vals):\n"," sig = nn.Sigmoid()\n"," return (1.0 - sig(preds)).pow(2).sum()\n","\n","\n","class BPRModule(nn.Module):\n"," \n"," def __init__(self,\n"," n_users,\n"," n_items,\n"," n_factors=40,\n"," dropout_p=0,\n"," sparse=False,\n"," model=BaseModule):\n"," super(BPRModule, self).__init__()\n","\n"," self.n_users = n_users\n"," self.n_items = n_items\n"," self.n_factors = n_factors\n"," self.dropout_p = dropout_p\n"," self.sparse = sparse\n"," self.pred_model = model(\n"," self.n_users,\n"," self.n_items,\n"," n_factors=n_factors,\n"," dropout_p=dropout_p,\n"," sparse=sparse\n"," )\n","\n"," def forward(self, users, items):\n"," assert isinstance(items, tuple), \\\n"," 'Must pass in items as (pos_items, neg_items)'\n"," # Unpack\n"," (pos_items, neg_items) = items\n"," pos_preds = self.pred_model(users, pos_items)\n"," neg_preds = self.pred_model(users, neg_items)\n"," return pos_preds - neg_preds\n","\n"," def predict(self, users, items):\n"," return self.pred_model(users, items)\n","\n","\n","class BasePipeline:\n"," \"\"\"\n"," Class defining a training pipeline. Instantiates data loaders, model,\n"," and optimizer. Handles training for multiple epochs and keeping track of\n"," train and test loss.\n"," \"\"\"\n","\n"," def __init__(self,\n"," train,\n"," test=None,\n"," model=BaseModule,\n"," n_factors=40,\n"," batch_size=32,\n"," dropout_p=0.02,\n"," sparse=False,\n"," lr=0.01,\n"," weight_decay=0.,\n"," optimizer=torch.optim.Adam,\n"," loss_function=nn.MSELoss(reduction='sum'),\n"," n_epochs=10,\n"," verbose=False,\n"," random_seed=None,\n"," interaction_class=Interactions,\n"," hogwild=False,\n"," num_workers=0,\n"," eval_metrics=None,\n"," k=5):\n"," self.train = train\n"," self.test = test\n","\n"," if hogwild:\n"," num_loader_workers = 0\n"," else:\n"," num_loader_workers = num_workers\n"," self.train_loader = data.DataLoader(\n"," interaction_class(train), batch_size=batch_size, shuffle=True,\n"," num_workers=num_loader_workers)\n"," if self.test is not None:\n"," self.test_loader = data.DataLoader(\n"," interaction_class(test), batch_size=batch_size, shuffle=True,\n"," num_workers=num_loader_workers)\n"," self.num_workers = num_workers\n"," self.n_users = self.train.shape[0]\n"," self.n_items = self.train.shape[1]\n"," self.n_factors = n_factors\n"," self.batch_size = batch_size\n"," self.dropout_p = dropout_p\n"," self.lr = lr\n"," self.weight_decay = weight_decay\n"," self.loss_function = loss_function\n"," self.n_epochs = n_epochs\n"," if sparse:\n"," assert weight_decay == 0.0\n"," self.model = model(self.n_users,\n"," self.n_items,\n"," n_factors=self.n_factors,\n"," dropout_p=self.dropout_p,\n"," sparse=sparse)\n"," self.optimizer = optimizer(self.model.parameters(),\n"," lr=self.lr,\n"," weight_decay=self.weight_decay)\n"," self.warm_start = False\n"," self.losses = collections.defaultdict(list)\n"," self.verbose = verbose\n"," self.hogwild = hogwild\n"," if random_seed is not None:\n"," if self.hogwild:\n"," random_seed += os.getpid()\n"," torch.manual_seed(random_seed)\n"," np.random.seed(random_seed)\n","\n"," if eval_metrics is None:\n"," eval_metrics = []\n"," self.eval_metrics = eval_metrics\n"," self.k = k\n","\n"," def break_grads(self):\n"," for param in self.model.parameters():\n"," # Break gradient sharing\n"," if param.grad is not None:\n"," param.grad.data = param.grad.data.clone()\n","\n"," def fit(self):\n"," for epoch in range(1, self.n_epochs + 1):\n","\n"," if self.hogwild:\n"," self.model.share_memory()\n"," processes = []\n"," train_losses = []\n"," queue = mp.Queue()\n"," for rank in range(self.num_workers):\n"," p = mp.Process(target=self._fit_epoch,\n"," kwargs={'epoch': epoch,\n"," 'queue': queue})\n"," p.start()\n"," processes.append(p)\n"," for p in processes:\n"," p.join()\n","\n"," while True:\n"," is_alive = False\n"," for p in processes:\n"," if p.is_alive():\n"," is_alive = True\n"," break\n"," if not is_alive and queue.empty():\n"," break\n","\n"," while not queue.empty():\n"," train_losses.append(queue.get())\n"," queue.close()\n"," train_loss = np.mean(train_losses)\n"," else:\n"," train_loss = self._fit_epoch(epoch)\n","\n"," self.losses['train'].append(train_loss)\n"," row = 'Epoch: {0:^3} train: {1:^10.5f}'.format(epoch, self.losses['train'][-1])\n"," if self.test is not None:\n"," self.losses['test'].append(self._validation_loss())\n"," row += 'val: {0:^10.5f}'.format(self.losses['test'][-1])\n"," for metric in self.eval_metrics:\n"," func = getattr(metrics, metric)\n"," res = func(self.model, self.test_loader.dataset.mat_csr,\n"," num_workers=self.num_workers)\n"," self.losses['eval-{}'.format(metric)].append(res)\n"," row += 'eval-{0}: {1:^10.5f}'.format(metric, res)\n"," self.losses['epoch'].append(epoch)\n"," if self.verbose:\n"," print(row)\n","\n"," def _fit_epoch(self, epoch=1, queue=None):\n"," if self.hogwild:\n"," self.break_grads()\n","\n"," self.model.train()\n"," total_loss = torch.Tensor([0])\n"," pbar = tqdm(enumerate(self.train_loader),\n"," total=len(self.train_loader),\n"," desc='({0:^3})'.format(epoch))\n"," for batch_idx, ((row, col), val) in pbar:\n"," self.optimizer.zero_grad()\n","\n"," row = row.long()\n"," # TODO: turn this into a collate_fn like the data_loader\n"," if isinstance(col, list):\n"," col = tuple(c.long() for c in col)\n"," else:\n"," col = col.long()\n"," val = val.float()\n","\n"," preds = self.model(row, col)\n"," loss = self.loss_function(preds, val)\n"," loss.backward()\n","\n"," self.optimizer.step()\n","\n"," total_loss += loss.item()\n"," batch_loss = loss.item() / row.size()[0]\n"," pbar.set_postfix(train_loss=batch_loss)\n"," total_loss /= self.train.nnz\n"," if queue is not None:\n"," queue.put(total_loss[0])\n"," else:\n"," return total_loss[0]\n","\n"," def _validation_loss(self):\n"," self.model.eval()\n"," total_loss = torch.Tensor([0])\n"," for batch_idx, ((row, col), val) in enumerate(self.test_loader):\n"," row = row.long()\n"," if isinstance(col, list):\n"," col = tuple(c.long() for c in col)\n"," else:\n"," col = col.long()\n"," val = val.float()\n","\n"," preds = self.model(row, col)\n"," loss = self.loss_function(preds, val)\n"," total_loss += loss.item()\n","\n"," total_loss /= self.test.nnz\n"," return total_loss[0]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"iKeUFiRNPsFY"},"source":["def explicit():\n"," train, test = get_movielens_train_test_split()\n"," pipeline = BasePipeline(train, test=test, model=BaseModule,\n"," n_factors=10, batch_size=1024, dropout_p=0.02,\n"," lr=0.02, weight_decay=0.1,\n"," optimizer=torch.optim.Adam, n_epochs=40,\n"," verbose=True, random_seed=2017)\n"," pipeline.fit()\n","\n","\n","def implicit():\n"," train, test = get_movielens_train_test_split(implicit=True)\n","\n"," pipeline = BasePipeline(train, test=test, verbose=True,\n"," batch_size=1024, num_workers=4,\n"," n_factors=20, weight_decay=0,\n"," dropout_p=0., lr=.2, sparse=True,\n"," optimizer=torch.optim.SGD, n_epochs=40,\n"," random_seed=2017, loss_function=bpr_loss,\n"," model=BPRModule,\n"," interaction_class=PairwiseInteractions,\n"," eval_metrics=('auc', 'patk'))\n"," pipeline.fit()\n","\n","\n","def hogwild():\n"," train, test = get_movielens_train_test_split(implicit=True)\n","\n"," pipeline = BasePipeline(train, test=test, verbose=True,\n"," batch_size=1024, num_workers=4,\n"," n_factors=20, weight_decay=0,\n"," dropout_p=0., lr=.2, sparse=True,\n"," optimizer=torch.optim.SGD, n_epochs=40,\n"," random_seed=2017, loss_function=bpr_loss,\n"," model=BPRModule, hogwild=True,\n"," interaction_class=PairwiseInteractions,\n"," eval_metrics=('auc', 'patk'))\n"," pipeline.fit()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dPQaj1FjPsCo","outputId":"42275f83-f4e9-43cc-a105-3ecde82efaa4"},"source":["explicit()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Making data path\n","Downloading MovieLens data\n"],"name":"stdout"},{"output_type":"stream","text":["( 1 ): 100%|██████████| 89/89 [00:01<00:00, 53.63it/s, train_loss=6.88]\n","( 2 ): 7%|▋ | 6/89 [00:00<00:01, 57.03it/s, train_loss=6.06]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 1 train: 14.42120 val: 8.68083 \n"],"name":"stdout"},{"output_type":"stream","text":["( 2 ): 100%|██████████| 89/89 [00:01<00:00, 63.13it/s, train_loss=2.27]\n","( 3 ): 8%|▊ | 7/89 [00:00<00:01, 62.84it/s, train_loss=2.23]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 2 train: 4.15028 val: 3.99969 \n"],"name":"stdout"},{"output_type":"stream","text":["( 3 ): 100%|██████████| 89/89 [00:01<00:00, 59.57it/s, train_loss=1.67]\n","( 4 ): 7%|▋ | 6/89 [00:00<00:01, 59.43it/s, train_loss=1.33]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 3 train: 1.84903 val: 2.41240 \n"],"name":"stdout"},{"output_type":"stream","text":["( 4 ): 100%|██████████| 89/89 [00:01<00:00, 59.96it/s, train_loss=1.05]\n","( 5 ): 8%|▊ | 7/89 [00:00<00:01, 61.59it/s, train_loss=0.982]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 4 train: 1.20266 val: 1.78271 \n"],"name":"stdout"},{"output_type":"stream","text":["( 5 ): 100%|██████████| 89/89 [00:01<00:00, 57.47it/s, train_loss=0.917]\n","( 6 ): 8%|▊ | 7/89 [00:00<00:01, 62.99it/s, train_loss=0.861]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 5 train: 0.98022 val: 1.48147 \n"],"name":"stdout"},{"output_type":"stream","text":["( 6 ): 100%|██████████| 89/89 [00:01<00:00, 61.39it/s, train_loss=0.9]\n","( 7 ): 8%|▊ | 7/89 [00:00<00:01, 65.11it/s, train_loss=0.77] "],"name":"stderr"},{"output_type":"stream","text":["Epoch: 6 train: 0.88477 val: 1.32482 \n"],"name":"stdout"},{"output_type":"stream","text":["( 7 ): 100%|██████████| 89/89 [00:01<00:00, 62.83it/s, train_loss=0.806]\n","( 8 ): 7%|▋ | 6/89 [00:00<00:01, 54.86it/s, train_loss=0.766]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 7 train: 0.83306 val: 1.22818 \n"],"name":"stdout"},{"output_type":"stream","text":["( 8 ): 100%|██████████| 89/89 [00:01<00:00, 58.63it/s, train_loss=0.776]\n","( 9 ): 3%|▎ | 3/89 [00:00<00:03, 25.32it/s, train_loss=0.722]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 8 train: 0.80015 val: 1.16457 \n"],"name":"stdout"},{"output_type":"stream","text":["( 9 ): 100%|██████████| 89/89 [00:01<00:00, 59.21it/s, train_loss=0.871]\n","(10 ): 2%|▏ | 2/89 [00:00<00:04, 19.07it/s, train_loss=0.708]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 9 train: 0.77529 val: 1.12250 \n"],"name":"stdout"},{"output_type":"stream","text":["(10 ): 100%|██████████| 89/89 [00:01<00:00, 60.45it/s, train_loss=0.749]\n","(11 ): 2%|▏ | 2/89 [00:00<00:04, 19.87it/s, train_loss=0.735]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 10 train: 0.75322 val: 1.09408 \n"],"name":"stdout"},{"output_type":"stream","text":["(11 ): 100%|██████████| 89/89 [00:01<00:00, 60.82it/s, train_loss=0.728]\n","(12 ): 8%|▊ | 7/89 [00:00<00:01, 62.74it/s, train_loss=0.655]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 11 train: 0.73431 val: 1.06755 \n"],"name":"stdout"},{"output_type":"stream","text":["(12 ): 100%|██████████| 89/89 [00:01<00:00, 64.48it/s, train_loss=0.729]\n","(13 ): 8%|▊ | 7/89 [00:00<00:01, 61.52it/s, train_loss=0.706]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 12 train: 0.71816 val: 1.05441 \n"],"name":"stdout"},{"output_type":"stream","text":["(13 ): 100%|██████████| 89/89 [00:01<00:00, 63.59it/s, train_loss=0.804]\n","(14 ): 7%|▋ | 6/89 [00:00<00:01, 57.44it/s, train_loss=0.658]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 13 train: 0.70331 val: 1.04291 \n"],"name":"stdout"},{"output_type":"stream","text":["(14 ): 100%|██████████| 89/89 [00:01<00:00, 62.10it/s, train_loss=0.648]\n","(15 ): 7%|▋ | 6/89 [00:00<00:01, 55.63it/s, train_loss=0.662]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 14 train: 0.69230 val: 1.03409 \n"],"name":"stdout"},{"output_type":"stream","text":["(15 ): 100%|██████████| 89/89 [00:01<00:00, 59.82it/s, train_loss=0.71]\n","(16 ): 8%|▊ | 7/89 [00:00<00:01, 63.50it/s, train_loss=0.648]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 15 train: 0.68174 val: 1.02946 \n"],"name":"stdout"},{"output_type":"stream","text":["(16 ): 100%|██████████| 89/89 [00:01<00:00, 63.41it/s, train_loss=0.762]\n","(17 ): 8%|▊ | 7/89 [00:00<00:01, 66.62it/s, train_loss=0.6] "],"name":"stderr"},{"output_type":"stream","text":["Epoch: 16 train: 0.67185 val: 1.02574 \n"],"name":"stdout"},{"output_type":"stream","text":["(17 ): 100%|██████████| 89/89 [00:01<00:00, 61.57it/s, train_loss=0.709]\n","(18 ): 7%|▋ | 6/89 [00:00<00:01, 59.98it/s, train_loss=0.647]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 17 train: 0.66559 val: 1.01690 \n"],"name":"stdout"},{"output_type":"stream","text":["(18 ): 100%|██████████| 89/89 [00:01<00:00, 59.60it/s, train_loss=0.657]\n","(19 ): 7%|▋ | 6/89 [00:00<00:01, 58.13it/s, train_loss=0.609]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 18 train: 0.65754 val: 1.01814 \n"],"name":"stdout"},{"output_type":"stream","text":["(19 ): 100%|██████████| 89/89 [00:01<00:00, 58.23it/s, train_loss=0.609]\n","(20 ): 8%|▊ | 7/89 [00:00<00:01, 64.70it/s, train_loss=0.636]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 19 train: 0.65179 val: 1.01196 \n"],"name":"stdout"},{"output_type":"stream","text":["(20 ): 100%|██████████| 89/89 [00:01<00:00, 58.38it/s, train_loss=0.693]\n","(21 ): 8%|▊ | 7/89 [00:00<00:01, 68.79it/s, train_loss=0.607]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 20 train: 0.64911 val: 1.00926 \n"],"name":"stdout"},{"output_type":"stream","text":["(21 ): 100%|██████████| 89/89 [00:01<00:00, 60.85it/s, train_loss=0.75]\n","(22 ): 7%|▋ | 6/89 [00:00<00:01, 52.77it/s, train_loss=0.635]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 21 train: 0.64537 val: 1.01296 \n"],"name":"stdout"},{"output_type":"stream","text":["(22 ): 100%|██████████| 89/89 [00:01<00:00, 59.46it/s, train_loss=0.702]\n","(23 ): 4%|▍ | 4/89 [00:00<00:02, 39.91it/s, train_loss=0.588]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 22 train: 0.64303 val: 1.00838 \n"],"name":"stdout"},{"output_type":"stream","text":["(23 ): 100%|██████████| 89/89 [00:01<00:00, 56.49it/s, train_loss=0.683]\n","(24 ): 7%|▋ | 6/89 [00:00<00:01, 59.61it/s, train_loss=0.633]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 23 train: 0.63932 val: 0.99910 \n"],"name":"stdout"},{"output_type":"stream","text":["(24 ): 100%|██████████| 89/89 [00:01<00:00, 58.42it/s, train_loss=0.709]\n","(25 ): 7%|▋ | 6/89 [00:00<00:01, 52.67it/s, train_loss=0.594]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 24 train: 0.63549 val: 1.01004 \n"],"name":"stdout"},{"output_type":"stream","text":["(25 ): 100%|██████████| 89/89 [00:01<00:00, 57.48it/s, train_loss=0.786]\n","(26 ): 7%|▋ | 6/89 [00:00<00:01, 58.84it/s, train_loss=0.59] "],"name":"stderr"},{"output_type":"stream","text":["Epoch: 25 train: 0.63468 val: 1.00146 \n"],"name":"stdout"},{"output_type":"stream","text":["(26 ): 100%|██████████| 89/89 [00:01<00:00, 55.84it/s, train_loss=0.64]\n","(27 ): 7%|▋ | 6/89 [00:00<00:01, 58.98it/s, train_loss=0.603]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 26 train: 0.63316 val: 1.00257 \n"],"name":"stdout"},{"output_type":"stream","text":["(27 ): 100%|██████████| 89/89 [00:01<00:00, 60.23it/s, train_loss=0.682]\n","(28 ): 8%|▊ | 7/89 [00:00<00:01, 67.37it/s, train_loss=0.584]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 27 train: 0.63269 val: 1.00099 \n"],"name":"stdout"},{"output_type":"stream","text":["(28 ): 100%|██████████| 89/89 [00:01<00:00, 59.51it/s, train_loss=0.721]\n","(29 ): 7%|▋ | 6/89 [00:00<00:01, 57.41it/s, train_loss=0.573]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 28 train: 0.63194 val: 0.99549 \n"],"name":"stdout"},{"output_type":"stream","text":["(29 ): 100%|██████████| 89/89 [00:01<00:00, 58.52it/s, train_loss=0.759]\n","(30 ): 7%|▋ | 6/89 [00:00<00:01, 58.95it/s, train_loss=0.564]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 29 train: 0.63050 val: 1.00029 \n"],"name":"stdout"},{"output_type":"stream","text":["(30 ): 100%|██████████| 89/89 [00:01<00:00, 59.03it/s, train_loss=0.718]\n","(31 ): 8%|▊ | 7/89 [00:00<00:01, 65.42it/s, train_loss=0.563]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 30 train: 0.63016 val: 0.99232 \n"],"name":"stdout"},{"output_type":"stream","text":["(31 ): 100%|██████████| 89/89 [00:01<00:00, 57.36it/s, train_loss=0.699]\n","(32 ): 8%|▊ | 7/89 [00:00<00:01, 62.85it/s, train_loss=0.58] "],"name":"stderr"},{"output_type":"stream","text":["Epoch: 31 train: 0.63022 val: 0.99609 \n"],"name":"stdout"},{"output_type":"stream","text":["(32 ): 100%|██████████| 89/89 [00:01<00:00, 56.56it/s, train_loss=0.743]\n","(33 ): 7%|▋ | 6/89 [00:00<00:01, 59.53it/s, train_loss=0.576]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 32 train: 0.63043 val: 0.99635 \n"],"name":"stdout"},{"output_type":"stream","text":["(33 ): 100%|██████████| 89/89 [00:01<00:00, 57.91it/s, train_loss=0.643]\n","(34 ): 8%|▊ | 7/89 [00:00<00:01, 64.98it/s, train_loss=0.625]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 33 train: 0.63210 val: 0.99697 \n"],"name":"stdout"},{"output_type":"stream","text":["(34 ): 100%|██████████| 89/89 [00:01<00:00, 58.12it/s, train_loss=0.641]\n","(35 ): 6%|▌ | 5/89 [00:00<00:01, 49.84it/s, train_loss=0.546]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 34 train: 0.63177 val: 0.99458 \n"],"name":"stdout"},{"output_type":"stream","text":["(35 ): 100%|██████████| 89/89 [00:01<00:00, 54.93it/s, train_loss=0.654]\n","(36 ): 7%|▋ | 6/89 [00:00<00:01, 57.96it/s, train_loss=0.543]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 35 train: 0.63137 val: 1.00267 \n"],"name":"stdout"},{"output_type":"stream","text":["(36 ): 100%|██████████| 89/89 [00:01<00:00, 58.59it/s, train_loss=0.742]\n","(37 ): 7%|▋ | 6/89 [00:00<00:01, 59.93it/s, train_loss=0.553]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 36 train: 0.63002 val: 0.99718 \n"],"name":"stdout"},{"output_type":"stream","text":["(37 ): 100%|██████████| 89/89 [00:01<00:00, 58.76it/s, train_loss=0.733]\n","(38 ): 7%|▋ | 6/89 [00:00<00:01, 57.61it/s, train_loss=0.56]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 37 train: 0.62959 val: 0.99938 \n"],"name":"stdout"},{"output_type":"stream","text":["(38 ): 100%|██████████| 89/89 [00:01<00:00, 59.98it/s, train_loss=0.638]\n","(39 ): 8%|▊ | 7/89 [00:00<00:01, 61.75it/s, train_loss=0.599]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 38 train: 0.63083 val: 1.00133 \n"],"name":"stdout"},{"output_type":"stream","text":["(39 ): 100%|██████████| 89/89 [00:01<00:00, 61.77it/s, train_loss=0.724]\n","(40 ): 8%|▊ | 7/89 [00:00<00:01, 60.35it/s, train_loss=0.573]"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 39 train: 0.63185 val: 0.99541 \n"],"name":"stdout"},{"output_type":"stream","text":["(40 ): 100%|██████████| 89/89 [00:01<00:00, 61.02it/s, train_loss=0.69]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 40 train: 0.63168 val: 0.99467 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mtI0DewsPr_0","outputId":"6cc428e1-211f-4e7e-8264-e8332ad47e8b"},"source":["implicit()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," cpuset_checked))\n","( 1 ): 100%|██████████| 46/46 [00:02<00:00, 21.50it/s, train_loss=0.361]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 1 train: 0.42040 val: 0.40008 eval-auc: 0.55278 eval-patk: 0.00776 \n"],"name":"stdout"},{"output_type":"stream","text":["( 2 ): 100%|██████████| 46/46 [00:02<00:00, 22.72it/s, train_loss=0.298]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 2 train: 0.34066 val: 0.35044 eval-auc: 0.60807 eval-patk: 0.01164 \n"],"name":"stdout"},{"output_type":"stream","text":["( 3 ): 100%|██████████| 46/46 [00:02<00:00, 22.89it/s, train_loss=0.303]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 3 train: 0.27492 val: 0.31180 eval-auc: 0.65543 eval-patk: 0.01804 \n"],"name":"stdout"},{"output_type":"stream","text":["( 4 ): 100%|██████████| 46/46 [00:01<00:00, 23.75it/s, train_loss=0.192]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 4 train: 0.22703 val: 0.29160 eval-auc: 0.69006 eval-patk: 0.02694 \n"],"name":"stdout"},{"output_type":"stream","text":["( 5 ): 100%|██████████| 46/46 [00:02<00:00, 21.58it/s, train_loss=0.17]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 5 train: 0.19465 val: 0.27365 eval-auc: 0.71412 eval-patk: 0.03265 \n"],"name":"stdout"},{"output_type":"stream","text":["( 6 ): 100%|██████████| 46/46 [00:02<00:00, 22.30it/s, train_loss=0.176]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 6 train: 0.17487 val: 0.25775 eval-auc: 0.73276 eval-patk: 0.03973 \n"],"name":"stdout"},{"output_type":"stream","text":["( 7 ): 100%|██████████| 46/46 [00:02<00:00, 22.14it/s, train_loss=0.202]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 7 train: 0.16267 val: 0.25430 eval-auc: 0.74666 eval-patk: 0.04201 \n"],"name":"stdout"},{"output_type":"stream","text":["( 8 ): 100%|██████████| 46/46 [00:02<00:00, 22.22it/s, train_loss=0.17]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 8 train: 0.15176 val: 0.24547 eval-auc: 0.75858 eval-patk: 0.04429 \n"],"name":"stdout"},{"output_type":"stream","text":["( 9 ): 100%|██████████| 46/46 [00:02<00:00, 22.55it/s, train_loss=0.141]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 9 train: 0.14359 val: 0.23771 eval-auc: 0.76822 eval-patk: 0.04589 \n"],"name":"stdout"},{"output_type":"stream","text":["(10 ): 100%|██████████| 46/46 [00:01<00:00, 23.32it/s, train_loss=0.151]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 10 train: 0.13715 val: 0.22593 eval-auc: 0.77713 eval-patk: 0.04361 \n"],"name":"stdout"},{"output_type":"stream","text":["(11 ): 100%|██████████| 46/46 [00:01<00:00, 23.04it/s, train_loss=0.115]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 11 train: 0.13167 val: 0.22131 eval-auc: 0.78402 eval-patk: 0.04772 \n"],"name":"stdout"},{"output_type":"stream","text":["(12 ): 100%|██████████| 46/46 [00:02<00:00, 22.63it/s, train_loss=0.134]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 12 train: 0.12781 val: 0.22118 eval-auc: 0.79055 eval-patk: 0.04749 \n"],"name":"stdout"},{"output_type":"stream","text":["(13 ): 100%|██████████| 46/46 [00:01<00:00, 23.33it/s, train_loss=0.128]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 13 train: 0.12185 val: 0.21263 eval-auc: 0.79726 eval-patk: 0.05228 \n"],"name":"stdout"},{"output_type":"stream","text":["(14 ): 100%|██████████| 46/46 [00:02<00:00, 22.32it/s, train_loss=0.109]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 14 train: 0.11865 val: 0.20135 eval-auc: 0.80326 eval-patk: 0.04977 \n"],"name":"stdout"},{"output_type":"stream","text":["(15 ): 100%|██████████| 46/46 [00:01<00:00, 23.13it/s, train_loss=0.117]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 15 train: 0.11352 val: 0.20501 eval-auc: 0.80805 eval-patk: 0.05434 \n"],"name":"stdout"},{"output_type":"stream","text":["(16 ): 100%|██████████| 46/46 [00:01<00:00, 23.17it/s, train_loss=0.113]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 16 train: 0.11156 val: 0.20189 eval-auc: 0.81208 eval-patk: 0.05753 \n"],"name":"stdout"},{"output_type":"stream","text":["(17 ): 100%|██████████| 46/46 [00:02<00:00, 22.15it/s, train_loss=0.127]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 17 train: 0.10898 val: 0.19678 eval-auc: 0.81534 eval-patk: 0.05936 \n"],"name":"stdout"},{"output_type":"stream","text":["(18 ): 100%|██████████| 46/46 [00:01<00:00, 23.03it/s, train_loss=0.13]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 18 train: 0.10363 val: 0.19250 eval-auc: 0.81967 eval-patk: 0.05890 \n"],"name":"stdout"},{"output_type":"stream","text":["(19 ): 100%|██████████| 46/46 [00:02<00:00, 22.78it/s, train_loss=0.121]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 19 train: 0.10260 val: 0.18791 eval-auc: 0.82216 eval-patk: 0.06416 \n"],"name":"stdout"},{"output_type":"stream","text":["(20 ): 100%|██████████| 46/46 [00:02<00:00, 22.97it/s, train_loss=0.121]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 20 train: 0.10081 val: 0.18382 eval-auc: 0.82357 eval-patk: 0.06370 \n"],"name":"stdout"},{"output_type":"stream","text":["(21 ): 100%|██████████| 46/46 [00:02<00:00, 22.89it/s, train_loss=0.0978]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 21 train: 0.09957 val: 0.18360 eval-auc: 0.82604 eval-patk: 0.06667 \n"],"name":"stdout"},{"output_type":"stream","text":["(22 ): 100%|██████████| 46/46 [00:02<00:00, 22.88it/s, train_loss=0.105]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 22 train: 0.09936 val: 0.17989 eval-auc: 0.82805 eval-patk: 0.06667 \n"],"name":"stdout"},{"output_type":"stream","text":["(23 ): 100%|██████████| 46/46 [00:01<00:00, 23.03it/s, train_loss=0.102]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 23 train: 0.09896 val: 0.17684 eval-auc: 0.83031 eval-patk: 0.07123 \n"],"name":"stdout"},{"output_type":"stream","text":["(24 ): 100%|██████████| 46/46 [00:01<00:00, 23.09it/s, train_loss=0.116]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 24 train: 0.09503 val: 0.18290 eval-auc: 0.83277 eval-patk: 0.06758 \n"],"name":"stdout"},{"output_type":"stream","text":["(25 ): 100%|██████████| 46/46 [00:02<00:00, 22.64it/s, train_loss=0.081]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 25 train: 0.09565 val: 0.17506 eval-auc: 0.83462 eval-patk: 0.07511 \n"],"name":"stdout"},{"output_type":"stream","text":["(26 ): 100%|██████████| 46/46 [00:02<00:00, 22.48it/s, train_loss=0.102]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 26 train: 0.09337 val: 0.17530 eval-auc: 0.83571 eval-patk: 0.07169 \n"],"name":"stdout"},{"output_type":"stream","text":["(27 ): 100%|██████████| 46/46 [00:02<00:00, 21.46it/s, train_loss=0.0837]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 27 train: 0.09035 val: 0.17689 eval-auc: 0.83655 eval-patk: 0.07420 \n"],"name":"stdout"},{"output_type":"stream","text":["(28 ): 100%|██████████| 46/46 [00:02<00:00, 20.81it/s, train_loss=0.0846]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 28 train: 0.08635 val: 0.17874 eval-auc: 0.83849 eval-patk: 0.07420 \n"],"name":"stdout"},{"output_type":"stream","text":["(29 ): 100%|██████████| 46/46 [00:02<00:00, 21.13it/s, train_loss=0.107]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 29 train: 0.08961 val: 0.17910 eval-auc: 0.83905 eval-patk: 0.07237 \n"],"name":"stdout"},{"output_type":"stream","text":["(30 ): 100%|██████████| 46/46 [00:02<00:00, 21.09it/s, train_loss=0.0935]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 30 train: 0.08822 val: 0.17294 eval-auc: 0.84065 eval-patk: 0.07717 \n"],"name":"stdout"},{"output_type":"stream","text":["(31 ): 100%|██████████| 46/46 [00:02<00:00, 21.52it/s, train_loss=0.0926]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 31 train: 0.08964 val: 0.16762 eval-auc: 0.84098 eval-patk: 0.07466 \n"],"name":"stdout"},{"output_type":"stream","text":["(32 ): 100%|██████████| 46/46 [00:02<00:00, 21.57it/s, train_loss=0.0708]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 32 train: 0.08982 val: 0.16215 eval-auc: 0.84217 eval-patk: 0.07055 \n"],"name":"stdout"},{"output_type":"stream","text":["(33 ): 100%|██████████| 46/46 [00:02<00:00, 20.14it/s, train_loss=0.106]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 33 train: 0.08753 val: 0.16941 eval-auc: 0.84282 eval-patk: 0.07352 \n"],"name":"stdout"},{"output_type":"stream","text":["(34 ): 100%|██████████| 46/46 [00:02<00:00, 20.73it/s, train_loss=0.0781]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 34 train: 0.08659 val: 0.17334 eval-auc: 0.84284 eval-patk: 0.07489 \n"],"name":"stdout"},{"output_type":"stream","text":["(35 ): 100%|██████████| 46/46 [00:02<00:00, 20.66it/s, train_loss=0.0971]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 35 train: 0.08623 val: 0.17476 eval-auc: 0.84393 eval-patk: 0.07443 \n"],"name":"stdout"},{"output_type":"stream","text":["(36 ): 100%|██████████| 46/46 [00:02<00:00, 20.77it/s, train_loss=0.0864]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 36 train: 0.08559 val: 0.17291 eval-auc: 0.84470 eval-patk: 0.07397 \n"],"name":"stdout"},{"output_type":"stream","text":["(37 ): 100%|██████████| 46/46 [00:02<00:00, 20.11it/s, train_loss=0.0751]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 37 train: 0.08506 val: 0.16872 eval-auc: 0.84690 eval-patk: 0.07648 \n"],"name":"stdout"},{"output_type":"stream","text":["(38 ): 100%|██████████| 46/46 [00:02<00:00, 18.27it/s, train_loss=0.0964]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 38 train: 0.08522 val: 0.16541 eval-auc: 0.84715 eval-patk: 0.07991 \n"],"name":"stdout"},{"output_type":"stream","text":["(39 ): 100%|██████████| 46/46 [00:02<00:00, 19.55it/s, train_loss=0.0962]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 39 train: 0.08316 val: 0.16021 eval-auc: 0.84812 eval-patk: 0.07991 \n"],"name":"stdout"},{"output_type":"stream","text":["(40 ): 100%|██████████| 46/46 [00:02<00:00, 19.17it/s, train_loss=0.0943]\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch: 40 train: 0.08459 val: 0.16542 eval-auc: 0.84809 eval-patk: 0.07237 \n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"tbXR1BvPWXKO"},"source":["## Neural Graph Collaborative Filtering on MovieLens\n","> Applying NGCF PyTorch version on Movielens-100k."]},{"cell_type":"markdown","metadata":{"id":"sG-h_5yQQEvQ"},"source":["### Libraries"]},{"cell_type":"code","metadata":{"id":"3QJEhOlDwIR7"},"source":["!pip install -q git+https://github.com/sparsh-ai/recochef"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"08tq9wC8pdD1"},"source":["import os\n","import csv \n","import argparse\n","import numpy as np\n","import pandas as pd\n","import random as rd\n","from time import time\n","from pathlib import Path\n","import scipy.sparse as sp\n","from datetime import datetime\n","\n","import torch\n","from torch import nn\n","import torch.nn.functional as F\n","\n","from recochef.preprocessing.split import chrono_split"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"F_-NBXzHS0_o"},"source":["device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","use_cuda = torch.cuda.is_available()\n","device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","torch.cuda.set_device(0)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"uuM8Q9GlP8RN"},"source":["### Data Loading\n","\n","The MovieLens 100K data set consists of 100,000 ratings from 1000 users on 1700 movies as described on [their website](https://grouplens.org/datasets/movielens/100k/)."]},{"cell_type":"code","metadata":{"id":"Q-yPokpipXsY"},"source":["!wget http://files.grouplens.org/datasets/movielens/ml-100k.zip"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6CLWhSTXpbvW"},"source":["!unzip ml-100k.zip"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"KS9OJY75pgwq","outputId":"0c1ec91e-2ef6-4b5a-e613-fa4501e2737f"},"source":["df = pd.read_csv('ml-100k/u.data', sep='\\t', header=None, names=['USERID','ITEMID','RATING','TIMESTAMP'])\n","df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
01962423881250949
11863023891717742
2223771878887116
3244512880606923
41663461886397596
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","0 196 242 3 881250949\n","1 186 302 3 891717742\n","2 22 377 1 878887116\n","3 244 51 2 880606923\n","4 166 346 1 886397596"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"markdown","metadata":{"id":"Mo6kTBolQYca"},"source":["### Train/Test Split\n","\n","We split the data chronologically in 80:20 ratio. Validated the split for user 4."]},{"cell_type":"code","metadata":{"id":"XTJpJj2uvpD2"},"source":["df_train, df_test = chrono_split(df, ratio=0.8)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"Dpeu1H5xxXcR","outputId":"d94c244d-ad25-47c1-d084-e51f6b015645"},"source":["userid = 4\n","\n","query = \"USERID==@userid\"\n","display(df.query(query))\n","display(df_train.query(query))\n","display(df_test.query(query))"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
125042643892004275
132943035892002352
220443615892002353
252643574892003525
327742604892004275
596043563892003459
1215142945892004409
1389342884892001445
163054505892003526
1893043545892002353
2008242714892001690
2038343005892001445
2451943283892001537
2474342585892001374
2486642103892003374
3531343295892002352
488264114892004520
5120343275892002352
6409143245892002353
6827343595892002352
7105543625892002352
7672243582892004275
8681543605892002352
8889143015892002353
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","1250 4 264 3 892004275\n","1329 4 303 5 892002352\n","2204 4 361 5 892002353\n","2526 4 357 4 892003525\n","3277 4 260 4 892004275\n","5960 4 356 3 892003459\n","12151 4 294 5 892004409\n","13893 4 288 4 892001445\n","16305 4 50 5 892003526\n","18930 4 354 5 892002353\n","20082 4 271 4 892001690\n","20383 4 300 5 892001445\n","24519 4 328 3 892001537\n","24743 4 258 5 892001374\n","24866 4 210 3 892003374\n","35313 4 329 5 892002352\n","48826 4 11 4 892004520\n","51203 4 327 5 892002352\n","64091 4 324 5 892002353\n","68273 4 359 5 892002352\n","71055 4 362 5 892002352\n","76722 4 358 2 892004275\n","86815 4 360 5 892002352\n","88891 4 301 5 892002353"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
2474342585892001374
1389342884892001445
2038343005892001445
2451943283892001537
2008242714892001690
6827343595892002352
7105543625892002352
132943035892002352
5120343275892002352
3531343295892002352
8681543605892002352
220443615892002353
1893043545892002353
8889143015892002353
6409143245892002353
2486642103892003374
596043563892003459
252643574892003525
163054505892003526
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","24743 4 258 5 892001374\n","13893 4 288 4 892001445\n","20383 4 300 5 892001445\n","24519 4 328 3 892001537\n","20082 4 271 4 892001690\n","68273 4 359 5 892002352\n","71055 4 362 5 892002352\n","1329 4 303 5 892002352\n","51203 4 327 5 892002352\n","35313 4 329 5 892002352\n","86815 4 360 5 892002352\n","2204 4 361 5 892002353\n","18930 4 354 5 892002353\n","88891 4 301 5 892002353\n","64091 4 324 5 892002353\n","24866 4 210 3 892003374\n","5960 4 356 3 892003459\n","2526 4 357 4 892003525\n","16305 4 50 5 892003526"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMIDRATINGTIMESTAMP
7672243582892004275
327742604892004275
125042643892004275
1215142945892004409
488264114892004520
\n","
"],"text/plain":[" USERID ITEMID RATING TIMESTAMP\n","76722 4 358 2 892004275\n","3277 4 260 4 892004275\n","1250 4 264 3 892004275\n","12151 4 294 5 892004409\n","48826 4 11 4 892004520"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"qny2aVoWQknP"},"source":["### Preprocessing\n","\n","1. Sort by User ID and Timestamp\n","2. Label encode user and item id - in this case, already label encoded starting from 1, so decreasing ids by 1 as a proxy for label encode\n","3. Remove Timestamp and Rating column. The reason is that we are training a recall-maximing model where the objective is to correctly retrieve the items that users can interact with. We can select a rating threshold also\n","4. Convert Item IDs into list format\n","5. Store as a space-seperated txt file"]},{"cell_type":"code","metadata":{"id":"1iSOiyCqpmYE"},"source":["def preprocess(data):\n"," data = data.copy()\n"," data = data.sort_values(by=['USERID','TIMESTAMP'])\n"," data['USERID'] = data['USERID'] - 1\n"," data['ITEMID'] = data['ITEMID'] - 1\n"," data.drop(['TIMESTAMP','RATING'], axis=1, inplace=True)\n"," data = data.groupby('USERID')['ITEMID'].apply(list).reset_index(name='ITEMID')\n"," return data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"D7ZtrUPp22dO","outputId":"56290e63-dcf3-448b-b3a5-ce60bd2c23db"},"source":["preprocess(df_train).head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USERIDITEMID
00[167, 171, 164, 155, 165, 195, 186, 13, 249, 1...
11[285, 257, 304, 306, 287, 311, 300, 305, 291, ...
22[301, 332, 343, 299, 267, 336, 302, 344, 353, ...
33[257, 287, 299, 327, 270, 358, 361, 302, 326, ...
44[266, 454, 221, 120, 404, 362, 256, 249, 24, 2...
\n","
"],"text/plain":[" USERID ITEMID\n","0 0 [167, 171, 164, 155, 165, 195, 186, 13, 249, 1...\n","1 1 [285, 257, 304, 306, 287, 311, 300, 305, 291, ...\n","2 2 [301, 332, 343, 299, 267, 336, 302, 344, 353, ...\n","3 3 [257, 287, 299, 327, 270, 358, 361, 302, 326, ...\n","4 4 [266, 454, 221, 120, 404, 362, 256, 249, 24, 2..."]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"yDMAhrig1Lde"},"source":["def store(data, target_file='./data/movielens/train.txt'):\n"," Path(target_file).parent.mkdir(parents=True, exist_ok=True)\n"," with open(target_file, 'w+') as f:\n"," writer = csv.writer(f, delimiter=' ')\n"," for USERID, row in zip(data.USERID.values,data.ITEMID.values):\n"," row = [USERID] + row\n"," writer.writerow(row)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XUIFrKsavRzV"},"source":["store(preprocess(df_train), '/content/data/ml-100k/train.txt')\n","store(preprocess(df_test), '/content/data/ml-100k/test.txt')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vq2IwCkJtUTy","outputId":"bdd5df6b-213f-4e87-deae-b8f29e42ec87"},"source":["!head /content/data/ml-100k/train.txt"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0 167 171 164 155 165 195 186 13 249 126 180 116 108 0 245 256 247 49 248 252 261 92 223 123 18 122 136 145 6 234 14 244 259 23 263 125 236 12 24 120 250 235 239 117 129 64 189 46 30 27 113 38 51 237 198 182 10 68 160 94 59 82 178 21 97 63 134 162 25 201 88 7 213 181 47 98 159 174 191 179 127 142 184 67 54 203 55 95 80 78 150 211 22 69 83 93 196 190 183 133 206 144 187 185 96 84 35 143 158 16 173 251 104 147 107 146 219 105 242 121 106 103 246 119 44 267 266 258 260 262 9 149 233 91 70 41 175 90 192 216 176 215 193 72 58 132 40 194 217 169 212 156 222 26 226 79 230 66 118 199 3 214 163 1 205 76 52 135 45 39 152 268 253 114 172 210 228 154 202 61 89 218 166 229 34 161 60 264 111 56 48 29 232 130 151 81 140 71 32 157 197 224 112 20 148 87 100 109 102 238 33 28 42 131 209 204 115 124\r\n","1 285 257 304 306 287 311 300 305 291 302 268 298 314 295 0 18 296 292 274 256 294 276 286 254 297 289 279 273 275 272 290 277 293 24 278 13 110 9 281 12 236 283 99 126 312 284 301 282 250 310\r\n","2 301 332 343 299 267 336 302 344 353 257 287 318 340 351 271 349 352 333 342 338 341 335 298 325 293 306 331 270 244 354 323 348 322 321 334 263 324 337 329 350 346 339 328\r\n","3 257 287 299 327 270 358 361 302 326 328 359 360 353 300 323 209 355 356 49\r\n","4 266 454 221 120 404 362 256 249 24 20 99 108 368 234 411 406 410 104 367 224 150 0 180 49 405 423 412 78 396 372 230 398 228 225 175 449 182 434 88 1 227 229 226 448 209 430 173 171 143 402 397 390 384 16 371 385 392 395 166 366 89 400 389 41 152 185 455 69 383 109 79 380 363 208 450 381 427 382 429 210 432 238 172 207 203 413 167 153 421 431 422 418 142 416 414 373 28 433 364 365 379 391 386 428 424 213 134 61 374 97 447 184 233 435 199 442 444 446 218 443 378 369 440 144 445 401 240 370 215 65 420 426 377 94 419 101 415 417 98 403\r\n","5 285 241 301 268 305 257 339 302 303 320 309 258 267 308 537 260 181 247 407 274 6 296 126 275 99 8 458 123 13 514 14 136 292 533 535 12 116 284 220 474 0 256 110 476 245 507 470 150 297 283 124 409 532 236 457 293 471 459 534 404 531 472 20 475 300 307 63 523 7 513 97 426 164 222 134 78 530 509 177 486 176 135 88 49 526 204 512 480 461 186 191 46 168 173 519 317 483 488 497 142 70 11 478 190 496 479 491 503 511 495 210 468 524 196 481 198 529 55 536 499 473 68 520 203 506 31 179 182 518 489 188 22 193 463 494 510 460 184 165 174 487 485 69 132 528 482 215 521 522 434 192 462 431 492 237 493 58 208 130 21 94 502 527 86 490 316 467 505 155\r\n","6 268 677 681 258 680 306 265 285 267 682 299 287 263 679 308 63 173 186 602 514 175 179 85 366 264 227 522 434 617 185 418 215 446 529 177 642 31 650 171 649 181 100 473 172 615 428 97 233 487 49 92 525 196 88 99 481 495 513 21 611 203 610 652 658 96 143 483 190 402 656 131 170 180 633 7 494 222 95 22 645 654 635 55 8 490 195 435 81 200 498 655 632 167 422 603 134 272 430 67 204 384 165 614 660 510 182 214 155 607 647 643 197 212 670 68 126 189 43 595 355 542 236 526 512 3 284 135 163 497 237 151 484 592 193 482 612 91 478 491 191 317 392 156 381 583 479 509 420 496 202 429 486 590 6 426 662 152 207 501 567 587 631 78 460 178 629 504 480 27 506 130 228 213 503 433 657 10 588 24 646 160 613 549 469 628 206 210 98 69 626 601 194 528 80 555 673 527 651 70 26 46 547 608 150 536 187 508 216 667 618 274 518 431 627 606 634 9 470 176 120 401 605 209 403 674 201 50 630 89 621 377 211 659 415 454 609 548 153 139 520 678 464 124 462 132 419 125 648 442 543 669 404 604 76 378 229 471 565 117 500 162 161 545 140 580 488 199 636 639 505 644 38 225 591 638 280 383 546 600 596 672 364 231 594 597 51 28 572 447 451 598 90 105 623 619 450 570 586 502 663 71 240 379 561 141 577 599 388 593 77 622 440 400 395 443 571 398 79 414 664 563 676\r\n","7 257 293 300 258 335 259 687 242 357 456 340 686 337 688 650 171 186 126 49 384 88 21 55 189 181 180 95 173 510 509 567 176 10 434 175 182 402 143 54 227 78 272 209 6 194 228 685\r\n","8 339 241 478 520 401 506 614 526 689 275 293 6 370 49 384 5 297 200\r\n","9 301 285 268 288 318 244 333 332 653 526 429 55 512 662 63 31 173 126 492 193 152 557 58 185 517 701 610 628 417 473 384 692 602 706 155 504 655 587 485 663 22 11 403 530 685 370 81 222 123 474 49 708 190 272 609 487 220 705 174 274 696 10 177 21 710 700 167 204 650 184 605 181 233 15 510 697 0 479 196 460 159 115 477 134 178 156 8 508 495 197 601 47 194 210 651 175 3 98 133 68 136 284 356 154 462 97 434 501 496 217 691 199 481 482 215 179 273 163 169 497 69 446 461 99 469 275 132 466 654 483 588 191 478 413 128 202 704 12 198 703 160 694 518 702 656 520 603\r\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"E7YYuu2XuVQa","outputId":"fd6fc81c-8ee7-4a34-cb7f-ae5c9ffb27d6"},"source":["!head /content/data/ml-100k/test.txt"],"execution_count":null,"outputs":[{"output_type":"stream","text":["0 207 2 11 57 200 137 65 36 37 139 240 75 225 77 62 231 138 141 74 50 53 43 86 99 8 227 153 85 168 15 177 221 257 265 254 271 270 19 128 220 243 5 17 269 208 31 188 241 170 110 4 255 101 73\r\n","1 49 241 271 309 303 299 288 315 307 308 313 280\r\n","2 320 327 326 345 347 259 330 317 316 319 180\r\n","3 357 259 263 293 10\r\n","4 138 388 453 68 161 232 242 258 451 439 437 436 438 188 168 407 100 425 376 62 399 93 408 193 162 393 375 39 409 23 441 387 394 452 456\r\n","5 516 80 133 498 194 466 418 131 504 207 356 465 199 172 187 212 273 422 169 525 484 508 515 501 201 469 366 185 500 153 477 202 167 424 18 85 152 27 517 538 464 271\r\n","6 675 61 144 551 616 560 569 553 585 540 52 138 589 448 544 558 333 293 259 624 417 541 557 218 671 439 568 562 550 564 566 668 445 637 556 579 665 641 226 582 53 573 449 142 416 625 620 559 230 575 390 539 427 174 72 584 574 576 385 578 519 386 316 661 552 554 30 133 323 257 11 192 640 184 198 356 666 653 432 581 340\r\n","7 549 81 187 221 430 683 517 226 240 232 565 684\r\n","8 690 285 482 486\r\n","9 143 503 59 616 709 431 494 92 509 524 488 229 529 161 6 237 614 695 581 282 699 693 366 84 39 419 711 707 528 182 698 131 32 498 320 293 339\r\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"C-f9mzEf4Ow6"},"source":["Path('/content/results').mkdir(parents=True, exist_ok=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Bxz7aV0Sws1S"},"source":["def parse_args():\n"," parser = argparse.ArgumentParser(description=\"Run NGCF.\")\n"," parser.add_argument('--data_dir', type=str,\n"," default='./data/',\n"," help='Input data path.')\n"," parser.add_argument('--dataset', type=str, default='ml-100k',\n"," help='Dataset name: Amazond-book, Gowella, ml-100k')\n"," parser.add_argument('--results_dir', type=str, default='results',\n"," help='Store model to path.')\n"," parser.add_argument('--n_epochs', type=int, default=400,\n"," help='Number of epoch.')\n"," parser.add_argument('--reg', type=float, default=1e-5,\n"," help='l2 reg.')\n"," parser.add_argument('--lr', type=float, default=0.0001,\n"," help='Learning rate.')\n"," parser.add_argument('--emb_dim', type=int, default=64,\n"," help='number of embeddings.')\n"," parser.add_argument('--layers', type=str, default='[64,64]',\n"," help='Output sizes of every layer')\n"," parser.add_argument('--batch_size', type=int, default=512,\n"," help='Batch size.')\n"," parser.add_argument('--node_dropout', type=float, default=0.,\n"," help='Graph Node dropout.')\n"," parser.add_argument('--mess_dropout', type=float, default=0.1,\n"," help='Message dropout.')\n"," parser.add_argument('--k', type=str, default=20,\n"," help='k order of metric evaluation (e.g. NDCG@k)')\n"," parser.add_argument('--eval_N', type=int, default=5,\n"," help='Evaluate every N epochs')\n"," parser.add_argument('--save_results', type=int, default=1,\n"," help='Save model and results')\n","\n"," return parser.parse_args(args={})"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"twi1ZIucR0ga"},"source":["### Helper Functions\n","\n","- early_stopping()\n","- train()\n","- split_matrix()\n","- ndcg_k()\n","- eval_model"]},{"cell_type":"markdown","metadata":{"id":"aCShFbsCTPzw"},"source":["#### Early Stopping\n","Premature stopping is applied if *recall@20* on the test set does not increase for 5 successive epochs."]},{"cell_type":"code","metadata":{"id":"tHVTudWxTVZo"},"source":["def early_stopping(log_value, best_value, stopping_step, flag_step, expected_order='asc'):\n"," \"\"\"\n"," Check if early_stopping is needed\n"," Function copied from original code\n"," \"\"\"\n"," assert expected_order in ['asc', 'des']\n"," if (expected_order == 'asc' and log_value >= best_value) or (expected_order == 'des' and log_value <= best_value):\n"," stopping_step = 0\n"," best_value = log_value\n"," else:\n"," stopping_step += 1\n","\n"," if stopping_step >= flag_step:\n"," print(\"Early stopping at step: {} log:{}\".format(flag_step, log_value))\n"," should_stop = True\n"," else:\n"," should_stop = False\n","\n"," return best_value, stopping_step, should_stop"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6JEG5Jlpw3Nw"},"source":["def train(model, data_generator, optimizer):\n"," \"\"\"\n"," Train the model PyTorch style\n"," Arguments:\n"," ---------\n"," model: PyTorch model\n"," data_generator: Data object\n"," optimizer: PyTorch optimizer\n"," \"\"\"\n"," model.train()\n"," n_batch = data_generator.n_train // data_generator.batch_size + 1\n"," running_loss=0\n"," for _ in range(n_batch):\n"," u, i, j = data_generator.sample()\n"," optimizer.zero_grad()\n"," loss = model(u,i,j)\n"," loss.backward()\n"," optimizer.step()\n"," running_loss += loss.item()\n"," return running_loss\n","\n","def split_matrix(X, n_splits=100):\n"," \"\"\"\n"," Split a matrix/Tensor into n_folds (for the user embeddings and the R matrices)\n"," Arguments:\n"," ---------\n"," X: matrix to be split\n"," n_folds: number of folds\n"," Returns:\n"," -------\n"," splits: split matrices\n"," \"\"\"\n"," splits = []\n"," chunk_size = X.shape[0] // n_splits\n"," for i in range(n_splits):\n"," start = i * chunk_size\n"," end = X.shape[0] if i == n_splits - 1 else (i + 1) * chunk_size\n"," splits.append(X[start:end])\n"," return splits\n","\n","def compute_ndcg_k(pred_items, test_items, test_indices, k):\n"," \"\"\"\n"," Compute NDCG@k\n"," \n"," Arguments:\n"," ---------\n"," pred_items: binary tensor with 1s in those locations corresponding to the predicted item interactions\n"," test_items: binary tensor with 1s in locations corresponding to the real test interactions\n"," test_indices: tensor with the location of the top-k predicted items\n"," k: k'th-order \n"," Returns:\n"," -------\n"," NDCG@k\n"," \"\"\"\n"," r = (test_items * pred_items).gather(1, test_indices)\n"," f = torch.from_numpy(np.log2(np.arange(2, k+2))).float().cuda()\n"," dcg = (r[:, :k]/f).sum(1)\n"," dcg_max = (torch.sort(r, dim=1, descending=True)[0][:, :k]/f).sum(1)\n"," ndcg = dcg/dcg_max\n"," ndcg[torch.isnan(ndcg)] = 0\n"," return ndcg"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"sx-Vzl2vTeWN"},"source":["#### Eval Model\n","\n","At every N epoch, the model is evaluated on the test set. From this evaluation, we compute the recall and normal discounted cumulative gain (ndcg) at the top-20 predictions. It is important to note that in order to evaluate the model on the test set we have to ‘unpack’ the sparse matrix (torch.sparse.todense()), and thus load a bunch of ‘zeros’ on memory. In order to prevent memory overload, we split the sparse matrices into 100 chunks, unpack the sparse chunks one by one, compute the metrics we need, and compute the mean value of all chunks."]},{"cell_type":"code","metadata":{"id":"1dysqVKGTjm6"},"source":["def eval_model(u_emb, i_emb, Rtr, Rte, k):\n"," \"\"\"\n"," Evaluate the model\n"," \n"," Arguments:\n"," ---------\n"," u_emb: User embeddings\n"," i_emb: Item embeddings\n"," Rtr: Sparse matrix with the training interactions\n"," Rte: Sparse matrix with the testing interactions\n"," k : kth-order for metrics\n"," \n"," Returns:\n"," --------\n"," result: Dictionary with lists correponding to the metrics at order k for k in Ks\n"," \"\"\"\n"," # split matrices\n"," ue_splits = split_matrix(u_emb)\n"," tr_splits = split_matrix(Rtr)\n"," te_splits = split_matrix(Rte)\n","\n"," recall_k, ndcg_k= [], []\n"," # compute results for split matrices\n"," for ue_f, tr_f, te_f in zip(ue_splits, tr_splits, te_splits):\n","\n"," scores = torch.mm(ue_f, i_emb.t())\n","\n"," test_items = torch.from_numpy(te_f.todense()).float().cuda()\n"," non_train_items = torch.from_numpy(1-(tr_f.todense())).float().cuda()\n"," scores = scores * non_train_items\n","\n"," _, test_indices = torch.topk(scores, dim=1, k=k)\n","\n"," # If you want to use a as the index in dim1 for t, this code should work:\n"," #t[torch.arange(t.size(0)), a]\n","\n"," pred_items = torch.zeros_like(scores).float()\n"," # pred_items.scatter_(dim=1,index=test_indices,src=torch.tensor(1.0).cuda())\n"," pred_items.scatter_(dim=1,index=test_indices,src=torch.ones_like(test_indices, dtype=torch.float).cuda())\n","\n"," topk_preds = torch.zeros_like(scores).float()\n"," # topk_preds.scatter_(dim=1,index=test_indices[:, :k],src=torch.tensor(1.0))\n"," _idx = test_indices[:, :k]\n"," topk_preds.scatter_(dim=1,index=_idx,src=torch.ones_like(_idx, dtype=torch.float))\n","\n"," TP = (test_items * topk_preds).sum(1)\n"," rec = TP/test_items.sum(1)\n"," ndcg = compute_ndcg_k(pred_items, test_items, test_indices, k)\n","\n"," recall_k.append(rec)\n"," ndcg_k.append(ndcg)\n","\n"," return torch.cat(recall_k).mean(), torch.cat(ndcg_k).mean()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"mvvKJOlpSLn4"},"source":["### Dataset Class"]},{"cell_type":"markdown","metadata":{"id":"AzoIqHHuUADD"},"source":["#### Laplacian matrix\n","\n","The components of the Laplacian matrix are as follows,\n","\n","- **D**: a diagonal degree matrix, where D{t,t} is |N{t}|, which is the amount of first-hop neighbors for either item or user t,\n","- **R**: the user-item interaction matrix,\n","- **0**: an all-zero matrix,\n","- **A**: the adjacency matrix,\n","\n","#### Interaction and Adjacency Matrix\n","\n","We create the sparse interaction matrix R, the adjacency matrix A, the degree matrix D, and the Laplacian matrix L, using the SciPy library. The adjacency matrix A is then transferred onto PyTorch tensor objects."]},{"cell_type":"code","metadata":{"id":"s0w9GTdKw7Vj"},"source":["class Data(object):\n"," def __init__(self, path, batch_size):\n"," self.path = path\n"," self.batch_size = batch_size\n","\n"," train_file = path + '/train.txt'\n"," test_file = path + '/test.txt'\n","\n"," #get number of users and items\n"," self.n_users, self.n_items = 0, 0\n"," self.n_train, self.n_test = 0, 0\n"," self.neg_pools = {}\n","\n"," self.exist_users = []\n","\n"," # search train_file for max user_id/item_id\n"," with open(train_file) as f:\n"," for l in f.readlines():\n"," if len(l) > 0:\n"," l = l.strip('\\n').split(' ')\n"," items = [int(i) for i in l[1:]]\n"," # first element is the user_id, rest are items\n"," uid = int(l[0])\n"," self.exist_users.append(uid)\n"," # item/user with highest number is number of items/users\n"," self.n_items = max(self.n_items, max(items))\n"," self.n_users = max(self.n_users, uid)\n"," # number of interactions\n"," self.n_train += len(items)\n","\n"," # search test_file for max item_id\n"," with open(test_file) as f:\n"," for l in f.readlines():\n"," if len(l) > 0:\n"," l = l.strip('\\n')\n"," try:\n"," items = [int(i) for i in l.split(' ')[1:]]\n"," except Exception:\n"," continue\n"," if not items:\n"," print(\"empyt test exists\")\n"," pass\n"," else:\n"," self.n_items = max(self.n_items, max(items))\n"," self.n_test += len(items)\n"," # adjust counters: user_id/item_id starts at 0\n"," self.n_items += 1\n"," self.n_users += 1\n","\n"," self.print_statistics()\n","\n"," # create interactions/ratings matrix 'R' # dok = dictionary of keys\n"," print('Creating interaction matrices R_train and R_test...')\n"," t1 = time()\n"," self.R_train = sp.dok_matrix((self.n_users, self.n_items), dtype=np.float32) \n"," self.R_test = sp.dok_matrix((self.n_users, self.n_items), dtype=np.float32)\n","\n"," self.train_items, self.test_set = {}, {}\n"," with open(train_file) as f_train:\n"," with open(test_file) as f_test:\n"," for l in f_train.readlines():\n"," if len(l) == 0: break\n"," l = l.strip('\\n')\n"," items = [int(i) for i in l.split(' ')]\n"," uid, train_items = items[0], items[1:]\n"," # enter 1 if user interacted with item\n"," for i in train_items:\n"," self.R_train[uid, i] = 1.\n"," self.train_items[uid] = train_items\n","\n"," for l in f_test.readlines():\n"," if len(l) == 0: break\n"," l = l.strip('\\n')\n"," try:\n"," items = [int(i) for i in l.split(' ')]\n"," except Exception:\n"," continue\n"," uid, test_items = items[0], items[1:]\n"," for i in test_items:\n"," self.R_test[uid, i] = 1.0\n"," self.test_set[uid] = test_items\n"," print('Complete. Interaction matrices R_train and R_test created in', time() - t1, 'sec')\n","\n"," # if exist, get adjacency matrix\n"," def get_adj_mat(self):\n"," try:\n"," t1 = time()\n"," adj_mat = sp.load_npz(self.path + '/s_adj_mat.npz')\n"," print('Loaded adjacency-matrix (shape:', adj_mat.shape,') in', time() - t1, 'sec.')\n","\n"," except Exception:\n"," print('Creating adjacency-matrix...')\n"," adj_mat = self.create_adj_mat()\n"," sp.save_npz(self.path + '/s_adj_mat.npz', adj_mat)\n"," return adj_mat\n"," \n"," # create adjancency matrix\n"," def create_adj_mat(self):\n"," t1 = time()\n"," \n"," adj_mat = sp.dok_matrix((self.n_users + self.n_items, self.n_users + self.n_items), dtype=np.float32)\n"," adj_mat = adj_mat.tolil()\n"," R = self.R_train.tolil() # to list of lists\n","\n"," adj_mat[:self.n_users, self.n_users:] = R\n"," adj_mat[self.n_users:, :self.n_users] = R.T\n"," adj_mat = adj_mat.todok()\n"," print('Complete. Adjacency-matrix created in', adj_mat.shape, time() - t1, 'sec.')\n","\n"," t2 = time()\n","\n"," # normalize adjacency matrix\n"," def normalized_adj_single(adj):\n"," rowsum = np.array(adj.sum(1))\n","\n"," d_inv = np.power(rowsum, -.5).flatten()\n"," d_inv[np.isinf(d_inv)] = 0.\n"," d_mat_inv = sp.diags(d_inv)\n","\n"," norm_adj = d_mat_inv.dot(adj).dot(d_mat_inv)\n"," return norm_adj.tocoo()\n","\n"," print('Transforming adjacency-matrix to NGCF-adjacency matrix...')\n"," ngcf_adj_mat = normalized_adj_single(adj_mat) + sp.eye(adj_mat.shape[0])\n","\n"," print('Complete. Transformed adjacency-matrix to NGCF-adjacency matrix in', time() - t2, 'sec.')\n"," return ngcf_adj_mat.tocsr()\n","\n"," # create collections of N items that users never interacted with\n"," def negative_pool(self):\n"," t1 = time()\n"," for u in self.train_items.keys():\n"," neg_items = list(set(range(self.n_items)) - set(self.train_items[u]))\n"," pools = [rd.choice(neg_items) for _ in range(100)]\n"," self.neg_pools[u] = pools\n"," print('refresh negative pools', time() - t1)\n","\n"," # sample data for mini-batches\n"," def sample(self):\n"," if self.batch_size <= self.n_users:\n"," users = rd.sample(self.exist_users, self.batch_size)\n"," else:\n"," users = [rd.choice(self.exist_users) for _ in range(self.batch_size)]\n","\n"," def sample_pos_items_for_u(u, num):\n"," pos_items = self.train_items[u]\n"," n_pos_items = len(pos_items)\n"," pos_batch = []\n"," while True:\n"," if len(pos_batch) == num: break\n"," pos_id = np.random.randint(low=0, high=n_pos_items, size=1)[0]\n"," pos_i_id = pos_items[pos_id]\n","\n"," if pos_i_id not in pos_batch:\n"," pos_batch.append(pos_i_id)\n"," return pos_batch\n","\n"," def sample_neg_items_for_u(u, num):\n"," neg_items = []\n"," while True:\n"," if len(neg_items) == num: break\n"," neg_id = np.random.randint(low=0, high=self.n_items,size=1)[0]\n"," if neg_id not in self.train_items[u] and neg_id not in neg_items:\n"," neg_items.append(neg_id)\n"," return neg_items\n","\n"," def sample_neg_items_for_u_from_pools(u, num):\n"," neg_items = list(set(self.neg_pools[u]) - set(self.train_items[u]))\n"," return rd.sample(neg_items, num)\n","\n"," pos_items, neg_items = [], []\n"," for u in users:\n"," pos_items += sample_pos_items_for_u(u, 1)\n"," neg_items += sample_neg_items_for_u(u, 1)\n","\n"," return users, pos_items, neg_items\n","\n"," def get_num_users_items(self):\n"," return self.n_users, self.n_items\n","\n"," def print_statistics(self):\n"," print('n_users=%d, n_items=%d' % (self.n_users, self.n_items))\n"," print('n_interactions=%d' % (self.n_train + self.n_test))\n"," print('n_train=%d, n_test=%d, sparsity=%.5f' % (self.n_train, self.n_test, (self.n_train + self.n_test)/(self.n_users * self.n_items)))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"J2RxhIxmSYvl"},"source":["### NGCF Model"]},{"cell_type":"markdown","metadata":{"id":"P4vz1IOwTvED"},"source":["#### Weight initialization\n","\n","We then create tensors for the user embeddings and item embeddings with the proper dimensions. The weights are initialized using [Xavier uniform initialization](https://pytorch.org/docs/stable/nn.init.html).\n","\n","For each layer, the weight matrices and corresponding biases are initialized using the same procedure."]},{"cell_type":"markdown","metadata":{"id":"wo9vgNvJUWvR"},"source":["#### Embedding Layer\n","\n","The initial user and item embeddings are concatenated in an embedding lookup table as shown in the figure below. This embedding table is initialized using the user and item embeddings and will be optimized in an end-to-end fashion by the network."]},{"cell_type":"markdown","metadata":{"id":"ntRUCJGMUeNf"},"source":["![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASEAAABICAYAAAC5kUM9AAAgAElEQVR4Ae2daVhUV7rvz5f74T733A99zj3dT/p0x3S0p6QTTxJN2k6cYsc4xChx1hijxiHGeR4jzjOKAwqIoOIMIiAKIiLzXMxTMRbFUExFUUVBjft3n12AIuJAGoLEzfPUQ9Xae7/rXb937f9ea+219/o3pD+JgERAItCDBP6tB/OWspYISAQkAkgiJFUCiYBEoEcJSCLUo/ilzCUCEgFJhKQ6IBGQCPQoAUmEehS/lLlEQCIgiZBUByQCEoEeJSCJUI/ilzKXCEgEJBGS6oBEQCLQowQkEepR/FLmEgGJgCRCUh2QCEgEepSAJEI9il/KXCIgEZBESKoDEgGJQI8SkESoR/FLmUsEJAKSCEl1QCIgEehRApII9Sh+KXOJgERAEiGpDkgEJAI9SuAXK0KNufe45O7CqVOncLkWhUJrwNKjqKXMJQJdRECoIyPoEmdPn+D4cSe84srRGoQuMv7zm/mZRchCScRFju/ZyqaNG9mwYUObz2ZOBBZS12TtAgoCNV7LGPb5fA55R5BRUkOj2UrvDVMXIJFM/IIImNBVK8mJvcLG8R/xtWM8Sk3vvcT+zCIk0FirIGjbaN7u898MXX2Fe1EyZDF3cPp+BKM236ao1tAFYmGl6sI3/M+Uw0QV1GISJPn5BZ2BUlFsBAQsBhVXFw5g8qFoitXmXsvlZxYhkZOZ9MNj+MOv/p0hOxNQ1pmwmBqpTjrGUns/FNX6LhOhAXPOkFWh6wJ7vTa+kuO/aAJNBK7+mGkOkgh1MswWMh2+4A+/+r8M3S2jrN5MVbQ/IVmFZBepMZq7ojvW3BIaOOcM2S8qQnoFMT5nObpvHw6ufsjKGjB1hSudpNM1u+tRxN7E/eg+9jm44isrQ2d8iQujLyH2pnsLe1+SSnW8zO52FCN9SRy+7o7s3+eAq28SpVoj3U+8icA1kgh1FI/npLWK0P/hrck7OHLKma3TZ7A3XIna2HG3yVKVSfjd2/j7+eHX9nM7jkKdgScbop0TIaEugTPLJjJ1zRn8A1xZMXYQw+eeIq5C14Ht5xSvpzcLdSScWc6kqWtw9buF68ov+MfwOTjFlqM19bRzT+YvsndbMZlpa1zwu+XKyi8+Zvick8SUa7G5K9SSGXqPpJJ6ml7KYQ+BusSzrJwyjdXOvvi7rmLcx8OZcyKa0vpm4II6i9B7SSjqm7r45ogkQk/WqBdKaRWhf+e9+c5c9fPn0MxJbL+voPYpItSU/4Cr587gfPo0p9t8nD1Dye8wsJ0RITM5Z2cz8I13mLT7An53b2L/xRv812tjOJRYhrbLKr6AWuaFs1sgWbX6x8VNUCPzcsYtMIta/eOSKqhleDm7EZhVS7tNHdI257gz56M/8O7EXZz3DeLm9nG8+evXGH0wntL6LitMh3l3PtFMjvtcPnrzXSbuPIdv0E22j3uTX782moNxSmzuWvLxd9jHxdgKNJ29AySoSfZ2wS0wk5qGtlwF1MneuLgFklnT8EQsHh3zAqptzsFj3iD6vjuRHR43Cbq5gy/7/obfjtpPbInGJjqWglsc2X+RmHINnS3Cs5lKIvRsPk/d+kiEBttHkVdeR+E9b+7laZ56klkbNVRXqqioqHj8UyVeHTu669UJEbIquTD7bX7z6w+Zt/8kbp4XcD68k207zxJWWo/BakKjSCejWIPBLIBVTV5mIeomUyeb2xbk5xYxetxGfApqaGzb6LPIObdoNOM2+lBQ0/jYGJZFfo5Fo8ex0aeAmscO6giwlVLPb3nntV/z4dx9nHTz5ILzYXba78QtTEl9kxWTRkF6ZjF1BhMCVtT5mRSpG3um62ktxXPOO7z2m4HM3XsCtwvN7O13uvFAqaHJrEOVl0Cw/30yVA2d76JZ5JxfPJYvN3qTV92WqwX5+cWM/XIj3nnVT8Si9Rh51fPHJ62lF5nb/7f8ZuAc9hw/w4ULzhzeac9OtweUaJow61TkJQbjfz8Dla6ru2iSCHV0FrxAWqsItY4JWbE0atEZm8gODCBRpcPQrjOtT77EznXL+X7RIha1+Xy//gzRKi1PNqA6IUKWPE7a9eX//WoIW++mkl9eSWVlJZVVdehNGrJDLnNgwWi+PRZnG78y557l+0XHiFLWdZDvs4ovoK/IJS2tgOrGdgIm6KnITSOtoJrGdgNRgr6C3LQ0CqpfRCgs5DlN5I//9SsGbwkiJb+8uSyVVdTpjWiyQ7h8cCFjvj1GbKkGkzkX9x++51hkCXVPQnxWYbpmmyUPp0l/5L9+NZgtgSnktWUvTqkwqUg5u5yxM/YTolB3vhUhNKKSp5FeUNWOq0CjSk5aegFVT8Tiacd0XGRL/ikm//nX/Mcnm7kty6NcrDuVlVTV6TFbBUyqVNxXfMHM/cFddOe3rR+SCLWl0YnvFjIcxrYZmG5WHGtVMNumreZKfhUNbVsJgKW+FHlWBmlpaY9/shSoDeYOWiSdECFBS+imwbz+q9cY/uNt5GoDVksNyQ/iUGh1aCqC2frZxyzyzKRSb0bpOZdPF54jraIBq9BASYKMwoam5vELkYK5gqRb1wlIVKI1dF/3x6xK4tb1ABJLtDzKRkAbupkhff6D14ZtISCnFoPVQk1yGHEKLVpNBfd+HMkniy6QXtGAufQi8/65CI/UcnRiGAxKQo+vYpH9NdKqmrsp5qo0Lu3bgZNPDIU64zPHNMwqWQc+PaNqCFoebBnKG//xGsO23CKntpl9Slg8Cm0jZque5MPjGTT3DLIyHRYE9MpEkgt1ND7sKZlRyQLwCkigpL4bJ6SaVcgCvAhIUFD/CDiC9gFbh/+B/3xtKJv9sqlpsmKpSSEsXkG93oxVn4LDhI+Z6yoOVjfXB0GdzbWdi9jokUh5y7gR+kwubV3L3nP3yalrQmx0P//vSRFqUsYT/CCNcq2hg/Pi+RZ7Yo+f+Ra9hZJIT7aO/yO/+t//i//+eCaLlq5izeplzB33AW8MXIF/UU3nr3hPkOuECCGgzbnBjxPfp2/f9/nn5DksXLiY9c5hKOsNmEvO8c2grzmRVIZGncjRyX9nylFxfEVLScJ17L9Zy9XCR8JpVfmy5rMP6D96O8GKzraWnijIUxKsqHzXMvKD/ozefpdStfHhfoI2Bx/7SXzQty/v/XMycxYsZPF6Zx6UiN3JEs7P/pivjydQWqcm0XEKg6YeJU6paR4XsTaQcHg2Qz4azKLzMlQNFoSmbFx2OxKWVYL2mRM+m336fEB/RtvfRak2PPTp6V8EtLk+2E/+gH793mPE5DkssLEPRaExYLWWcXn+YGY4xlKiMdsEyGvHt6y7kodK13KWWivxWzeKAf1HYx9UhLprB10eum6t9GPdqIH8z6htBBbWPqqjgpbcm9uZMqAf/d4bwaRvF7Bg8XpOhxbbJt5ay66wYMgMjkYrqGsdlmqIwnHqQN6Z5oSsTIsVPTlBB/j673bsCcmkwtDREMNDV9p8aS9CZjJd5jDi0yVcyFI9cTFvc+BL9fVnFiFxsmIpeWnxxERHEZuUSnpGJpmZ6aQkxhKTUkSdwfzYmMhPo9UsQuI8ocwXuUVvaUCVn0b0XT9u3n5AnCwNeXk9RouALnA1g8bYE5KfTbS3PXZ/G4n9nTRKGo00qNM4NusHzudVoG05JwRjFTmRV9iw7BARpW0q608ryFOOEjBW5RB1ZSPLDkVQUdv2hLfQoMonPeYufjdv8yBWRpq8nHqjBUEXxJp/jMU+WE52jDfbv3qHz+0DSFXosYgtIaEaP6eTeDnNZ/QXq7iYUklDfQjHHO9SUq1/zpW1xaerok/hlD/m01OKISbb2KcT08I+tg17oe4Oq4ZMYGdAMkU6C0a9mvQTs1lyPpey+pY+u2CkKieKq5uWcyi8hJpuEiExrrlRV9m0/BDhJY9fKC0NKgrSYwj28+X2g1hkaXLKNQbMgoDmzmqGTdjJLVkhWovYlgNDsidO+xYwcNBKvHJUVGffw+/WQSZ/8gOXMspo1ddnUGvZ1H6ekIBOEUfI/ZTubRU+37FO7fEzi1CnfPsXdhZFaBb9Jx62XeVftFNkNRtoMpiwPJxhLT7+sYj3PhjDoq0n8b64hqF/Gcqc/TdIVYvdhVLc5y/DM/+RCCHoyA2+yMXAdFQN4uBv9/wJulyCL10kMF1Fg6mDXKxmjE0GTBbhoQ9CjTffvz+AMYu2csL7ImuH/ZVhc/bhnVJLo9j+b4zE5fR9FMVpeG+wY/yaSySGHOHAZTmVL3BnTfTp3qVLT/fpGSisZmM79mBKOcjYd4cxa7snceLAtGCl7NwiVnjKH4kQAjr5PS5fCiStQtd9A+yCDvm9y1wKTKOiwwFmK2ZjEwaThYfVBxOph8fRf/gs7C/E2o4TMCC74MqdJFe+HTSenX4BXPF+QEbYASZ+tYt7eVWPWlnP4GXbJNRwdf4AJh+MejRjWrBgElusHVSJ55nrqe2/WBGqvraYQQNHMH3ZNhw8Iyj+iQ+wmmpyiQmPJiVHQXVlNtERiWQVVqIzWRE6EiGMaKsqqNYam1sX3RVZo5aqimq0RstzWihtHDDVII+JIDolh+KqSrJjIkjKLKRSJwovmDPdOXGjgGqtCV2BL5snjmfO9DnsDC2l9kVaGD/Fpzbutf8qaOTERCSQkd/SksNKqUd7EQKjtoqKai1GW3OuvZWu+v1T4ipQL48lIiGDfFvL2gpNSZx3CaSgIpkTUwcyct5eriSVUnNvM6O/c0bWclv/mV6LD7AGenJy1xLGDXyLiYfjUda96KX2mZZ7ZOMvVITAVFNAqkxGSmYuBUrxSv+i/ez2cRCwmC1YxUuLYMEsTgmwXWWs1GRdZ+Wno1h5ORalvrXVI/w8VyHhUQunvcfP+i1YzFis4rFtygXoy1Lw2z6PZacjya8zYrU2UOi/lQkjFnMuQ9U8cP0sw+K2n+jTU80+xhusNdl4rf6MMSsvEqVooLUBKPwsl/2fFtdHvIGmClKubmLFofvkVdcStnsu6zyikCvyuL97PAMm7+eeXP0CUxFMaCuVFMkzSE5KIlucYf5iI9lPRd2TG36xItT9UAWM9eXkpqSQW6a2zVfq/jy7LwdLYx0V8kxyleKkyGbBtjaUkibLpUrf1fNbflo5BKOWcnkqKbllqJs60QL8adl1/VHWRupK88gtUdNosqArK0BZ14jJ2EBVQTpJGcXUiHfUuj7nl9qiJEIvdXh+TufE5p3Y2ns8T8HaLuHxzdKvLiLQQr+LrPUuM5II9a54Sd5KBH5xBCQR+sWFVCqQRKB3EZBEqHfFq+e9lXpnP1MMXh3Qr7wImYrCCYwvRftyvifiZ6rwL5aNtUZGUFgu6i6a/2QqjiAoXkl9Y++9vfxi5Dq5l7WG5Lth5NbqOvl8YifzeUl2f+VFSNAreHB2P/tdbyEr7c0vMuvGGmWppzDyMg67jnMzpcw2R6orchP0JYS5H+CAqz9Jyt73IrOuYPC4DQv1RVFcObKb4z4ySrXtHnR+fOdfzK9XXoTAgrYslUDXbSyYNoVZi1azdY8DTm6eXPP2wdfPhwvOLpy/4YuPpzPO573wPneKo04e3PQ6h7PzeXy9z3H6qBMe17045+zMeS8vzp06ipPHdbzONR/j1XKMj+8NLnleITDoHuExSaTLlagbzbzYTSgTuqpictMSiA67z51rl7ly0x8fz9M2f3xa/fF6cX+uu59sPtbm1yX8fH3xu+mD95ULnDlxmN2bVzB36iRmLtvHhfuZVDR05Ykh3qZOJfCMPQunN7Pfsvvwk+y9fbnh6YLLeS+8zp3G0ckDsawuLufxFcvq6ITHNZG9SzP70444eVyzsReP8b3hiYvrJe4EhRKdlI5cqX7xeWNmHVUKOWkJ0YQHXcfNzZMbvjfwdOk4r1b/rrmffMxPW12x+XmTG5c8uXTDl5u+vvh4X+HCmRMc3r2FFfOmMunrZew9H0JGuXhBfDW6ZJII2a4nZhoqC8mIDuDC8Z2sWTiLSWM/Y9jHA3nv3bfo26cPb/T7I/3eeJ3X3+hL3z6/47e/60O/vn14/fU3bP9//9vf0efNvvT5/eu80bcvfX73W37X50369vn948f060fft/rx57/+hbfefof+7w1k0NBRTJq7mr1ng8hQ6R9/9EDQUxJ/E+ddK5ht9xlDBg3gvf5/4+2//pU/9fsbffv9mX5v/P4n+/Nma1lEv97ux1/efpv+H3zIJ8NGMm7ybBav38upq0HEZiiofWGx7MxF2kxDVREZ0bfxPL6TtYu+YdIXIvsPeb+Vfd9+/xL7fn3f4PXX/8Cf/vxX3n6nP+8NHMSQUZOYu3oPboHpTwiroFcS7+vC7pXfYjdyCIMGvEf/v73NW39+kz593qBvv3688fqz4/yQ6xN1pB99+75Fvz+9xdvvvs+Hnwxj5LjJzF68nj1OVwiKSae4Vnwk6NUQILGmSCLU9nyxGtDWlFOcl01GiozE+FiioyKJiIggPDy8yz4RURGEh90nJDiI277XOO96kqMHdrJ55QJmfbsap/sFaAxWrJp0vPYsZd53i1mzdRcHjp7E7aIXfrcDCQ4JJSw8iojwrvNN9CsiMpKomFjiE2WkZuSQr6iwjQF1+znRyj4/p9vYP7gfQnDQbXyvncfV6SgHdm5m5YJZfLvaiXv5dbYJp5p0b/Yum8f8xavZsusAR0+64Xndj9uBwYSEhnVNHYiIIiIiksioGGLjE5GlZpCTr6CiVnxx26sjPq2nniRCrSR+7v+CgNVqwWxqQq/VUFtVjiI/i+RIP9wOOHA9IQGv4w64XA0kSpZFQUkFVbUadHoDJvExklewsv6rIRIEK1aLGVOTHq2mlqpyBflZyUT6n+WgwzXi4704ccSFq3cikWUVUFJeRa1Gh95gsj2uIyH/VyPQ8fGSCHXMpedSzQ1UFSUSdOM6vuGvXtO8J8Cb9dUUJQVxw8uX8LQiarql29kTJesdeUoi9DLGSTBSX1XViQHrl7EQvcsnwVhPVZU4YN38zp/e5X3v9lYSod4dP8l7iUCvJyCJUK8PoVQAiUDvJiCJUO+On+S9RKDXE5BEqNeHUCqARKB3E5BEqHfHT/JeItDrCUgi1OtDKBVAItC7CUgi9JLGz1BTTHr0fRIKdTS1rlf1Qr4K6CvzSI4IJ6Ws4bGFETtOfyGjv+CdDNQWpxMdmkChtunxdemfV2pBT1VeChHhKZTq2i68KKCvyiNFjEGprk0Mnmfw1dwuidBLGXcBbUEQe7/+kg3eBVTrOzOV30pt1jW2TJrBvhBxhdrWY9unWykLOYXDNRmV2keLJ3YLDms5IaePcC2pkuasftpL47vFN0FH4d39fDN+A9fzKukc6lqyr29l8sy9BD+2aKeV2uzrbJ08k73B4uTHMu47H+Vakor6bkbdLYy62agkQt0M+Keat+jlOE39mO89c1C1Xxf7OUZN9bHsGTuaLXcKqH4oQtA+XZt+m5tRxWi6+30+gpb0275EFWlotJjJvb6NfdezqWxZFvk5xenmzRb0eaeYPngxFzLLO7HwoOiWCW3cPsaN2UKAvIqmVr0Xt2jj2DduDFsC5FQ11ZNxRyx/HfpOtWq7uegviflXRoQsdUpyszLJF59SN6lR5maRma9CLy5WJ1aamgJkMdHEpRVT27K+vaBXkZucSHpxLU1mK1ZTPaVZOSjraynOzLItVWwWTNQUyIiJjiOtuBajuf1aCQJ6VS7JieIjGE2YrVYMagUZuWXotdUUpSWSodRgMDWgypWRkKFsXi3VquL87KEsORdNVGQ8KYU1Nh+a6017my21SWigLDOemJibbPxsJJtbRaij9CYj9cUyotNKaWjUU1eSSW6ZnobqYtLiE5FXtnma31BNXlI00SlysiNvcMEnltL6JgwPmRVR0/T0VSIEUz0KWTRppQ00akrwWjmYLzddJTiuCLV4nFWPKjeZxPRiam2/DagVGTZ/tNVFpCVmoNQYMDWoyJUlkFFSj6GjJW6EdnawYqgrIbMta3E57IesS6g3mLGoLjBn+BI8xIdK41MorGls8yYDkbWclKR0imxPt7dEoKGcrPgYYm5u4vNRmx+JkNBAeVZzDDZ9PorNAXIqtfUokmNIU+poMhoestZVF5OekIhcXNix9b1uImtZjI11VtQNPH1iUWqaMNQWkhwbTVzqs1m/JLrSKTdeHRGqkuGydDxL3NJQaSpIPrMcu6VnSVfpMFWGcNj+OL7BAZxcu4sbRTXUFd/F+fgFbgf7c3L9chzvF5KfcJFNU6byw/ZdbFq6BMdQJfKAQ9gf9yX4lhNrd3mjqm5oEwAzirsunLgQQLD/SdYvdySkMIdI15VMmLmGw8dcOOu4iunTV7D32GnczzqwfNo8jkWUomlScf6bAYz94QDO513ZtmgBu29mU6M3oAhutenEhhWOhBSpaTIUE+R0kNNeQYT4HmLa/wxl7a18qnUdp1dVZOD14wymbg+gIDsKj7UTmbHqEMdOueNsP5dJK86RotJhacjg0p5DXLr3AP9jy/lu6yku306lqiiQwzuOc/PuLZzW7ca7oOopa5MJqDO92DZzKttvFaIqknHq2/cZsfgY5/xklNblEehyggsBd/F32sAKx3vkZ0fiusqOmWsOcczlLI6rZjBjxV4cT7tz9sgKpn/nSFhJHY/1bMwlBLuefMxOYU0x8W6rsft6NYdaWM+YsYK9jiLrI6yY/h2OYSXUllzg2w+/4Id9pzjnas+ihbvxyaymwWKi5J4rJy8EcNffiY0rHblXUIu26C6nDp3meuA9fA/N4P2ha/ETW0JGBXdPHeL09UDu+R5ixvtDWeuXS26SF/ZfT2O7fwGViiTOr5/EzFUHcTzljov9PCav8EBWrsXckMHlfYe5GByK//EVzN96iosBKVQWBXJk5wl8gvxxWr8H7/xKtO2vdW1qXW/7+sqIkGCqxWfJh0w+kkippola32X8fepRZKUajFknmPz5Qk4GJpMSFkuupoLAHRP5av4W9h89wvoJHzB0gz/5BeHs/eJDZh0JJDohjcKaBtKOTebzhScJlKUQFpuLvvHRqSFo7rNz0kTmb9nHkSMbsPtgKOv9Msnx28Cwf67iYmQahfJLfP+PiewKiCNbkcuZbwYx3yOLCm0F57/5mDnHH5BWpCT51ByGzTxKRIYP20Sbm1tsDhBt5lFwZyfTFh/jQVYp6spwto8axeY7eRQE7uogXVxltQ7ZQTuGr/Imt6SU0G0j+XSZB2EpBRRH72PCP9fhK69Cm3SESVN3EpRdSnnoNsZN3MXtzAp0mSeZOnoRJ+4kkSwyq2986qtITRoZh+w+ZZV3Dqq6GgLWDGbS3hDSiuqovLeDSRPns3nfEY5ssGPAsPX4ZmTjv/FTPlvlSXhqAfLLi/lk0i78Y7JQ5Lrx7ccLcM8oQ/uw+yOgCd3NlEnz2by3jR25EuXdTYwYuQrP8FQK5JdZPHgSu/xjyFLk4vbtxyxwz0CZf545g+dy7H4KhcoUTs/9lK+PhFNcfJfdUyYxf9NeW/y+GjiMdT4JXP9xBj843idDqaYyYgdjRostoUoqH+xhxg+O3M9Qoq6MYMeY0baWUKlKxuGJI1h1PZsKdQ1h20fzz2XuhMryKY7ez1cj1+GTU0m9zJGp03dyJ7OE8gfbGT95F7fSy9BlOjF97CKOBySSHB5Hrkb/VNa9TYBEf18ZEQIDwWs/ZrqjjDKNBcP9DQyecZyUsnosulx8j6zju8ni1fcsMco4DowfwkKXMGQ5hchTY4jLrqSpSY7TlBGsuSGnqlE8AwR0ub4cWfcdk+1msvpsNNW6RyJkznRkwpCFuDxIIqdQTmpsHNmVehri9zB6wi5Ci2oxGh6wedg0jiQo0VgaubX0I75xzqBMU9HcHbvUPCbUFL6F4SM34ee/lTFDF+Lc1qaqntRjExmy5BLZqgYESxaOdl/wY2AOkYc7ShfHiiwUu85g5LqbyKt0pDlM4IttQRTUGLCWuTNr+HKu5qjQlnixxO57XMJTkXmuYPLKi6RU6DDrcvE7uoH5U75i5mo3osq0Tz8xLApcZ37Oupu5VDY28WDzcGY6JqGsM5B57CuGLXQmNDGbQnkqsXHZqPQNxO8di92u+xTWGDGEbeHT6UeIU6gxNwWw/B+zOZ2qpP5ha8BM5rGJDFt4mvtt7TQYaUrcxxd2uwgpqMFoCGPLiOkciVOgNjcRsPwfzD6dSon8vK075pkljgk1EfHjCEZtukXWg4NMHL6I0yEJZLfGrzSJIxOHscQzg/IGAUv2cSaN+5Hb8nJSTk5h2BJP21sRBUs2xyeN48fbcqr0xZz5ejTrbohlM5N+9Cu+3HaHvKomrGXnmD1iOVcyy6kv8Wb5pMU4hyYj81zF1FWeyMq0mHVybjlubGF9hsjS+qez7oUq9EqJ0P0Ng5l4MBZlnRl94Cr+McWBRGUdalk44ZkZJEVcZt34Cey4Hc6hiR/wxfYgCtQGrIIVo9GMYM7j5OSRbPDLp1ochRS0yMLDycxIIuLyOsbb7aBEWfuwGlgUZ5k5YBzbg/JRG8S164225XqNCbsZNWE3YcW1mAwP2CSKUHwJdZZG/JcM5JtT6ZRpytuIkJXK64sYtfgcyXHHmD5wHNsD86h9aNNMifssPpx6hBilBrMplUPjRrHplpzk0193kC76b6HIZTqfrfVBXllP6uHxjNt2t1mESt2ZNXQpl7JV6KpvsXPdIc6JL1PzDyIyR0WDyUy9LIKIzHQSI66wfsJX7LhTSE3bkdmHFABLES4zRrL2RqsIDWOqQxyKOiMK0e9x9tzJq8VgFbAajZgFIwl7xmC3O/ShCA2f5kBssRpz4y2W/X02TilKNA9FyGIr/0fj7Lktr2ljR8CYuIexX+0itFWEPp2OQ2wxanMjt5b9ndlOKSjaipC1Eq8fxrDYI5mSTDe++ehL7G/nUtPK2lCCx+xBTHWIpLjOjCnVgfFjNuEnryDv3GwGTaDKfY0AAAZUSURBVHUgsrjOFgOH8WPY5CensqHQJsJrvUURMpJ2xI7x2wLJqxIF/xyzP13KxcxytNUB7Nl4CA/P6/j5BxKZXUGD0Ux9cmQz68grbPhqIjsCWupfW8a9+PsrJEIWiq8uZtT4H9hx5DSu2yfz3oAp7LiRSvGVTSze7cHVq8dZOn0p7okKMm7aM33UBOZvPcgJFw/8EgvJeXCcOe//kU9/OM29fC0GSx2+Py5mt8dVrh5fxvSlZylVaR9VB6OCW9tnMNpuPlsPnsDFw4/EwixCDkzh7b+NZ5tvCmn3dmH3l/eY5nCXjJQANn3+Jh/OOkFIfg6uc4Zgt+Io5y66sn/zDs6EF1GnLeLWjmabW1psJpRq0RX4sHHyBOZtPsBJF0cWDH6X0SvdeRDlyZqJT6ZHJIZy8rsP+eOnS3C+fo3dU9/lrbFbuJFSgMxzCZ/0G8zCM+EoCq+wZtYCVq9Zy4YtOzns4kO8UkulzzZ+2OPOlSsnWDZjKWcTytB2eOfHQnm8Kws++hMjlrgRpqgj8fAEhtgtZqtTMJmp3mybORq7+Vs4cMIFD78ECrNCODj1Hf725Y/cTE7j3u6v+Ov7UzkUmEZywBZG9/uQWceCydc8Wp7aqAhg58wxLXac8fBNoLQmn/DD03jnnS/50UdG6r3dTHzrfaYeCiQtOYAto/vx4axj3I08wewhX7HCwQNP1wNs2XGGsAI1TU0Kbu/8mjF237H5wHGcPXxJUNaQc2MTU+zmsWn/SVwcFzKk/2hWuEVQmn6dzVPsmLdpvy0GC4f0Z/SKM/j4HmXu3//MiB/OEBwVxMGv3+PtsZvwkuUju7iMIX8awgKXBxQVXGXd7IWsXt2GdUk9lTe3s3TPWS5fOcHymctwi1NSb3rYF31U33rpt1dIhMBUW4AsKpKE9FwKcuIJj0gmp6yexopckpNTbJ+U9Hyq9CZMunJyEsIJDY8hOauAUrWeevEuSUwk8akFqLQmLIKZKnkyySniJ4X0/CqMpoeXZ7EJgK48h4TwUMJjkskqKEXdoKFCnkx0VCLZZXWoK3KRRccgy1OhqS0lKzGa2OR8Khu0FKbEIpMlkZqZTXZuScs7npttJra1Kb6Ey1yPMjOB6LhksvPzkUU9ID5TQW2DGkXGk+k1tRXkpcYSGZ9GgUJBtiyaqIQsyup01BRnEB8ZR2pRNeoUd/YccOeK9y1u+3vjcWAZ83b4I89ORfYYMyOWDs8LAX11AamxkcSnFdnY6pRJhIXHk55Xga6pnrKcRMJDw4lJzqKgVE2DpgJ5cjTRidmU1ampyE0mOkZGnkpDbWk2SdGxJOdX0tByZ9N27ll0VOQkEvHQTi2NRi2VeclERz9indyGdXZSM2tVbT7JsTJkSSlkZmeTW1Lb8iJ8C7qKHBIjWuKXX2qLgbG+lMyEaOKSs8jLlxEVFk+GeGfUoKG0JQZZeWIMwojPKKJUmUuKrfyFqKrLyE2JsbEurdM2s46KI7WwCnWqB/sOunP5MdZ+5GSl2uqXWEfF+lnZ8DTWvVOFXikREkMkWMRXdYqT5SxY2uiFYDFiMIjC0uZMEiyYDEbbS8dtqYJ4XOunNeACFqMBg3hCtDm0dWtrngajuKJG8w6PbLTaavuf5jwQsFqsNn/F2/7tbYvlaGuzOT8Bi7k5H4ul7cu5Okh/WI62ebf/biL52BxWuMWTX16PTquhLPooq/YGoqhuxCr60J5Z24K3fH+svLY0K2bx5WEPeQlYTGJXtbWc7f1o95vm309m1d5OC8unlrV1uxWLtflY8QXzD91q9d9isnXHW+NnSxYsmG3+WrDY6lOrN+1YPzXvdmUSTKQc/46VZ2KRl7WwjnFkzd47FFXqX5h1qxe96f8rJ0K9KTg976uAKsSBlUs3sOekB5e9buDjd4eonEoaH2vx9bynvd8DAdX9o6xetoHdJ9y5fL2FdbY4l63N1bL3F/SJEkgi9AQSKaEtAUuDivysLHILiilRllJWWffia3a1NSR9fy4BkXVBtsi66JViLYnQc6uGtINEQCLQnQQkEepOupJtiYBE4LkEJBF6LiJpB4mARKA7CUgi1J10JdsSAYnAcwlIIvRcRNIOEgGJQHcSkESoO+lKtiUCEoHnEpBE6LmIpB0kAhKB7iQgiVB30pVsSwQkAs8l8P8BQIzxTVPvPikAAAAASUVORK5CYII=)"]},{"cell_type":"markdown","metadata":{"id":"HKqah7XFUhan"},"source":["#### Embedding propagation\n","\n","The embedding table is propagated through the network using the formula shown in the figure below."]},{"cell_type":"markdown","metadata":{"id":"ZvrR6lvUUuIm"},"source":["![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAXUAAAApCAYAAADKxrnwAAAgAElEQVR4Ae2dd3iUVdr/33/23d/qirvrvjZcxV1R0bXRV6SDFOlSVYr0IiBShFBCCb2ELtJbSEJIMKRDAmlAAiG99zLpk2QyM5k+z+d3PZOZZBKSEAUbO3Ndc81TznPK99zP99znvu9z5n+wfWwI2BCwIWBD4LFB4H8em5bYGmJDwIaADQEbAthI3SYENgRsCNgQeIwQsJH6Y9SZtqbYELAhYEPARuo2GbAhYEPAhsBjhICN1B+jzrQ1xYaADQEbAjZSt8mADQEbAjYEHiMEbKT+GHWmrSk2BGwI2BCwkbpNBmwI2BCwIfAYIWAj9ceoM21NsSFgQ8CGgI3UH1MZEBRJ3LpXgk5nfExbaGuWDYH/PgSM0hgikirR6IRmG28j9Wah+R3fEGREnT/EpXgpGkPznf87buFvt+qGMlJTi9DqDaCWkp0QQVBYEpqaXBKTS9HobYPsb7fzfqWaGcpJSy1Co6uVmZzESK6HJaFW5pCUXIpavG7+CKoMvI6e56ZEjrYZUXpoUjeUpZBSpMNgBHV5NgkRQYQlV6DOjSepVGu6bqmQ7feXQUCTeJLVmz1Jq9BgfJw4vY4wRWGzEGYyWq3+JwCrpyAxmVKNnmbejZ+Qp5KEi4dxjiqhRicgVKdzafUYhix1QyorIeb8AS5ElaLS/5Y7xUB5aipFWh0G1EhzEom8Hk6SVM1PqbZekkRyqZpfbcLYFGGGJ6HW/DSZkSQlU6rWPVKZSbx0BJe7RSh0RoTqDDzWjuOTpa6UVhQTe+EQzneLUVo0c0FDYeAu1h6NoFihoylJejhSV8bjcuAC98o0GASB6rSLrBw5iGUehShK73F2nxPRUvHeT3g/bI/8NAQEKQGrp7HOLw+59nECvp4wRVIUCdPNbjRDll6iQq5pJVZ6KgsKkekNCBipijnPgQv3KFPpm3w5WplpXTJ18hm+XX2G6OIaEwEK2kLOftGBARtvIlVoUaScY+U3R7lbXnu/7sHf0IEywY3vnKMoVuoQhGoy3Nfw6dCluOXIULdSnPRVEgplepNCZ6yKxengBaJKarDw0i/XXCWJbkdwiRJJsZYw3deO5ZOlFymtUreyGnqqJIXIdAaMoszEOnHowl1KRHxamUNLyTQp57Bbe4YoicIsM0Wcn/oOH28IoaRaiyLVCbtlR4ksUdThZ5TdZNtsO9zTK1A1oZE8BKmrSTq5lJVn4yhTiy+JgFZymomv9cMhUopKJyf5zHIWH42mSlM/fWipgbZ7D4+AUeLKrGHfcCVPzuPE6SbCtDtDjDVhfv4mAzbdpEL5IK1LQJl7k0v7lzLpS0duiaQKGBUpnPt2CUejpI9Ae1YStmkC888kU6Eyy7vmNut6fsRS3wKqxc7Q53N+3qes9c3/bQ646hTOrlzDmXtFKEW1XNBSeG4ybw/cSFiR3IRZSxIqKHO55X6AZZ9NxzG8rFa7NCpIOb+KpUfvUqZ8NINnS3WwvqdOOcuqNWe4V6i0Isx/8/GGUErlrZGZW3gcXM7n0x0JM5OqKDNOdks5eqe0Xnu2LvRHHSsJ3/IZX52Op6zGIjORbOjbm6VeOVRpRJkp4MJX41nnbT4X8xfkhK77lJknYihT3s+tP53UFSHYj5nL2fTqOtuO5tZqundfytXiWhD1eWeZNXINAcW/xij9o9B9TBIbkVyYyn+mnCJDpn6EU8RfGx4FYRvHmwizUm1WTUTC/KiHiTDlD1QBBdRlOST5rqBnj2+5WixHZ2qSnvzzcxmz1o8ChfbhNC91OGv7jsUxphzLe6ZP3c/wLjNwyqiittp6Mg98Sj+7q5TLtb82qPeVrwx3YOKC0yRKVdRShYaI9b3pudSHPNmD8RHUZeQk+fFt756s8C+k2txEfb4T88euxTevGpGnfpmPkvDNk1hwOpHyOsKMYEOfXvWE2WJFzDLjt5K+vVbgVyAzK0l6Ci4sYNw6H3JlmoeTGc1N1g8cz56oEixjjD7tEKO7z+RsitSshevJPDSegXZ+FFdZZqQClT/Mp8/n33GvVGnuq/rG/GRSV4euoseovSTWkYeelL1D+GC6M7lyswDoMtg7ohd2QeXUPGhgrK/TL3sk6FCU5ZEYcYvkMu3v21QklHFpxrsM2BxZL8h1aBqpTIwmQ6H7ddtoKCQuOhelTpzdtfKjDmdNn09xjJVieT9FwhzWeQZOmVVompiCNs7ZaDCgy9jH8D6rCCyxkDroMw8wpp+dFdE3frJ158aCY4zrOAvnvCqzmcJIyYWpdBx7kHhpjfnFE1D+MIf3JxyhtFLVuox/sVRqbq7rz7g90fXanz6VAyO7MvN8OhWWwbSl+hgNGHSZ7B/Zj1VXi+pIHX0mB8f2xy5AJPpW93pLJT34nvom9v3HsedeKQqzMqtPO8iobjM5l2ohzJazMclM5gFG97cjQCLDMgzrMw8xbqCdFdG3nE9zd42SE0zqOhunLCkqEyxGSlym03XcfqJLFPUyc2U+XSYdRlKmrMvKkHWI0Z2+4GhcWd07YbnZNKkbCrjpcoQ9Wzfj4ODQ8Lv1NGElNWR/N4p3Zlystb2JuRmLOf/Zu4w5lILM8pYJSjxmvM247zORP/QQLaBMucrpgzvZtnUrW3cdxPlmAaqHjSZQZRJ0eAmjeo1l5+1KLDNnC0At/grVJPmf4uCubWxx2MTWPYc4cngfjoddCc+SoX+Al1JQpnD1zCF2bdvK1q27OOh8k3yVxXFnpDTqEkf2bGfr1q3s3H+Sq6k1aA1gLLvH5WP72LFtKzv2X+ROsdmJpQ5mRae3+PJCHtUN8BZQJLqw64A/mXJtixq8TlZASnQU8VllKPXG1hNvU0Dp8wi7cATH7dtMbdj93WWiSsqIPr+DAwFZKEzuewMFty7yveNWNjeWNYetnAopQpH9PWM7zsQ5X2bW9ETCnELHTw+SUKEy2TpLo/3xcHPB2dnZ6uvCpdDMuugBQ9Z+RjQidUF5mdnvj+dIagWK3DDOHdrFdlG+dh7ENUKCojCKH47vZ6elj9wiKVTpMBrLiblykgM7t7P74BkCA7cx+P25uBVY6qjg6pKufLwpnBJFvUaj8VnAu6P2UVJR0xRij+aaXkZBSjT34jMpVepMznKD5DZuRx3Ztvn+d3rrqRAk8iyOje/MrAs5VJqN58YSZ6Z1HsuB2NrZh7E0hoDLbrg0wNgZl0uhZCi1tSRkyOLgqH7YWZO6oMRzbkcmfJeMtKYVI/ADUTAgibjEsb3b2LK5ET85bOFksAR51jEmdJmFU3Zl/SDr/CVdxu0nxqTdGimNCeBykzKTgcpsMhbJc0wjUheUnszrPJHDicWkhDrx3Z7tJk7aedCV2wVyCu95cvJAPU9djJCg1BoxSmPxPnWQXdt3c+B0EMlxexjRdR6uORXmOioIXPYhQzaGIKmut9lrfBfRcYwjklJFHTKC8ipLunRinmt2XX9ZbjZN6oKC4hQPlvdpz8sv9+dbt2vcCAsj0H030z4axMbwMhJ3DeadOZdNQmPKTO7Pwo792RQhtSJGDV5zOjBifxrVrRnpLbVq8ldAV5lPwqVl9HnjHcbvvEZcvvyBxNlkVtYX9dVIbm6k3/NdWRn8I2cUgpbKvHhcv+nNG50msz/wDrEJUfjtm8vIT1fhnt582JFYBUFXSX6CO8v7vsm743dyLS4feR2RCqhK0/G3H0KHt0ayyS+KzEq96QUVVGVk3TvDvL5j2eQTQ76y9rohfT9DX/6INWEiIVs1siae44sXsfdGDtXNhSEISlI9d7JyzT5cvLxw3raAmfbupMg1903vrHJu+dAopyjJjaV93uTVfw5j49V4JDU6FFme2M9ehVtKNRqjgKI4hR++7c+b7V6m/3IX/K+HERbozp4ZvRmyIZii+D0M6ziXSxLL9F3B1a+78vHGm5SYGmqkLD4Ib8/LeHh4WH0vc+V2Tm2oGNAUqaPxZv47o9iXVIGyuog412/o3+FVXum3Ep80KRplGVmhe5jU8TVe6fAZ+25nUyX2kaCmPN2Zxf1HseZSJBnZznzZ8QtOZFbWal26KDb17cESj2jSS8RBwNTjSM9PpuscV0pllql0yxD+uLsCytQr7F61ln3OV/By3s7CWfZcSpKhqi4m1XMVA01tW4aTbxBhYYF4OM6k75D13JDE4TiyM3Mv5lFlJnXFtW/oPqjenm4siyfIx5PLDTD24PKVW2TXtEDqaPD56n1G741HqmyB1I1qqgpSiIqMIjmvApVB9NQ19RFQlqTiafcxb736Cv2WOuEbJMqMB44z+zJ0/XUK4hwZ2WUuF3MtpK7g2tIPGbwxhEITYRopT7iOT1Mycysblag9mWTmflJH48vCjmPYG1dKsSQet2UDefuf7ei74grJZWqUZVmE7f2cLq+3o8PEPYRlVKAzCgjqctJdlvDx6NW43kqnvNSN2d2mcDy1HNNYp7vHloG9WOJ2h9RiS+SgQMWFaXw415lCqdXsTp/K3qHt6W8fbGqPNUpNk7qYQh/Dph7P8Kc/9WN3ehU1ej0aeQHBm6Zi51uO5PwU3vnsNPlmW6Tu7np6dl+CZ3wWpRZiEqScnfgBs9wkKB7RtEuftpsBz73JbA8xBKjpLrduYGuOjSXHGf3Sh9iF/khSN2WuJ2lHf55/bzG+JUrEYKeazMOMefU1Jh1Pr5+1NFcRfRp7Pn6BDrPdKbwvREmg6OhoXnz1c87myxvaI7VRbBw+mwt1miGoAxbx9osj2J9useGKhRrIdZnHoGmHiS1XNWt6UUYd5IsP+zLn5G3SC4tI/P5LBs89RUyl+qeTuli8NhL77v/Hk20+4XC22TRhrCTUfgTDN1xFYooi0BO7pTfPPfkEfXckUa7QodfKkYRuYcZqH0rynJje8QtOZpmftxDm5WgyTIQpoKuRUVkhRSq1/laYImOMQq2cGDL2MqzXylpTi1l0BOl5vugyG9fcWg3bUO7D4k7P8OQLoziSWGGyhQs1iewa/CJt2nRi5Y0SLIq3LmE3I4evIyhPjk6fzfGJg/jWbGIwFp/ls/admbjxLOESlTmkT0nIqmFMP51MVVNhC83JSGuvK+9xeMpH9Jt9nPBUCUWJR5kxdC4n74kECfq4rfR74c882Wc78aXV6PRa5JJQts1cg3dRLk4zOzP5RAYVJluAjqjN/floiTtRaSXoDEYEXQ2yyopGGEupqJCjFknLJG4Z7B/Rh5UBknpTi1CB05TuzHHJuk+rrG2agYo4D/Y7bGTr7gMcOXqEg7s3s9Z+Lz/El6FuMpZST9y2/rz41JP02RpHSbUoMwokYduYtdabwhwnZnWZzPF0aUPCvHSX1GIxBLslmVFTLzMHGNVvJf51NnUQKi4w9T9zcc6sQCUYKPdbQtdnn+KFEYeJKxdNbQKqpD0Me/kvPN1xBYGSarMPR0ei4xhGrbtKdpUWwZDDyS8+YaVvnskpaiw5z+QOXZm4/hSh+TVmX6WS0DUjmXkqAanF9iiCJlThPO113px2hqwKVYPBrwVSj2Nzz2f40xMD2JstR2Mo57Z3MOlZcaSU6tBmfM/Y/t8SWCaGZxkpPjWOdh9MZLPTbYo0Zi+34gbLB00zOVMtFpnWymdz6Qy5Bxn6wjss9Ba9z02n0pUlcsPTBScXbyJyquu0eWN1DhG+bpw7c54rkfkozIOPsfQEYyykrsoj0qSNuHP5WhzZCTfw8ryMu7s7/jElaPQGSmIC8LzsgYdvFBK1hvR9g3mx03KulSlrIwRkrkx+5QWGH0hDVjdD0VGWeIMrLk64eEeQU12rXWPI5dCwF3l3oTclivsbJD0znrbtp+NS1CiaRZ/IzgmL8CixkL2RohOf8tI/J3OuTqMFdEkcGNmJ8Yfjm7eLCpX4Le3CC69O4FSGzKQ91+THEpUkQS6GgjUNM+gz8DjhTaaiBW1ed5f1H4qkPpwjuZawOAHFrbX0776QS9nVaAU98dv68tyTT9J/dxpVKgPlEb6EpGUSm1KKVpPJMTNhik5RY/EZJr3WiYmbRMJU0xoLnLEkFq9d43mz3UCWnQkj12zmUoas5JPpZ0ipNDuWhQr8v+7E3//cltHfJ2JyzBqzOTi8LW3++BTd1gabZwc6EhxHM3J9MBK5OFXWke/xDZPXX0MimrhUGQR7BnArPotytRgOB0J1MJtm2OFpwrg5UMXrejIun8QnQ476/uCGZh4UqPRfzn9e+icTTqSa+lqoKSDuXpJpKi9aAvUJ2xnwwlM82W8nyVIl+vJI/ELTyIhNoVSjIfP4ZwxZ6U+B6BQ1FnPu8zfoMnEDZ8IKUDU3w7OujbGEWO/dTOjwKgOXniI0u6Z2MFOGYDd8BqeTrGfx9Q+qUy/isNYRJ79wohJSycrOIi0xhoiAs2z6dhte6bImcNCTsGMgbds8ST9REVDqkUb6EZqWYZIZjSaTE58PYaVfPjKtgLHkHF+8KRLmaRNhtqo5pXH47JnI2/8cyDcnQ8gWTVmAMnQ1I2eeJqGs1lciVATwTdfnaNN2FIfjyk02bmPOYUa/8hf+31NdsAuSUC2+2rpE9o8bw4agPFOdQEfBD8uZtiGAPJkGoyqTkCtXuRWXSbnKUDsrrw5h8+w1XE6taISBhoAl7/OPwVuJqrO/12L6YFL/03t8efA058468MXnO4mSmTU3XR6XFk3CPkhcTGFElXGdy/63SMiWmkgBBKqv2zN11RWyFc0E6xvKSA4L4lpAAAENvkHESDRNLlxqmdRF27ErDvb78bx1lzBXe6ZMXMeVLAWa6miOLV7Ooev3uBd8gqXjZ3AgotykBTQgdW0RXqsmMnnVYTxuZ1OWF8GpxYPp0ncBZ+LL0RiMyDPdWfXZXPb4JFCu1ZJhTeqaMqKOTeXDj2ZzOq4Sk2lOUJB40QH7/Z7cuhuGq/1UJq3zJEuhxfiQpH65jtR1JGzrw/Md5uBhNQAYso8y+rV+bIy436FS91opA1n6wTM83WMj9ypqBdVYay+oS9LkgeY6qyZv47ZIEE0mEAW5KVIHQfYDc97pzNdeRch1FlL/E+9O3ceJs2fZPGUyOyOlZjNSPWEWiphZEabUFE7bXOH11wVlKZmxYQT4BXMntZBqkwmlmuCN07HzzLCaURmR+i2m49+fou2YoyRVqtEXujD9o7d45Yk/8HS3dYSUKNHrEnAcPYL1N8SBr3bYM8oScd22Ddf4StR6IzrRGVw3IqpJubSD3W6xlDwwLl7DjdVT2X6rDPn943x9oxocKQla3pln/9KD9ZFltc5BoxhZXf+xkPoT70xhz7EznN08lSk7IigzTz10BZdZOmUDVwvEiDYVmcFXCLgVR1a5usl3sT5n85GgpDQzjvAAf4LvpFAoE+35AtUhDsxc/QNploHT+kFjKX6blrLLK44i0YxodU/QlJPotpbFjqFUKixuSksCC6k/wTuT93DszFk2T53CjtulZsx0FFxextQNV8mvFmVGJMwAbsVl1RGmJafmfk0yExfOVf9g7qSIseqi2a2akM2zWP1D7cBpetZYgf+SLjzXpi2jv4unvEZPoessev+7HU/84Wm6rg4yDay6xP2MG72BwDxLJA0YZUlc3LEd11jRSmBE30hmUj12scctxoRNnSiZCtUSvvo/vNR9mcmJay0mrSD1Tsw/484VDwdGf7KBiEpLuJORqvgLbN5ykaRq0f6jQ2e9ykidzMWtO3EzEWHD6tSBqE4n8NxJjn5/hCNHrL5HnQjJVTWpgbVI6oYcLsztxZBFR/EOvUWY1waGtGvPl075VBdeZev8bfgVKtDUZHJwZHsGbLmFVBwR6zT1YnLCT7N563H8Y7IpU2gx6BTk+3xDt3/0xD60DJUetAkH+Xq1OwlF4oIBA5n7BvNC+09Yum0DS6YMY8DYZRy7mkS5pjbCw5DjzNzeQ1j0vRcht8Lw2jiUdq9/iZMYS67/6Zr6jgmLsCb1iLXdeO69RfiKMbVmkNXXvubf7SZwrIXFI7q4rfR9rg2vzbps1jrreqjJA72inJLCQiQ5LswZvoIr8enkSYoorlLVTVvrHmyG1NHfY1PPVxi2J4Eqldasqf+JjnNP4vaDB5vHjmBDeFmdqUMkTBcLYRoaE2Zdac0fCIKpboIgYPoC6pRL7NjtRlypyhTHbHnYKPVl4Qd/589tx3A0qZws5xl8PHsziz78P576W3fWhZRQFePIqBGiLVpetyhENHVVZUcSHNOEZqsvJDb0LtktLcbTKygvKaJQkoPLvJF86xlLWq6EouJKDA8aZPXxbB/wIn9pP4NLVqRhaZP4W0fqHWdzzPUyHlvGMXJDGCWWkcMoI9F1O9td46gQZzI6MVKqmXfXOuO6YwFBNMNYMBYfVafgvnM3brHF1DRlRlFeZdXcfUSJcfCmsOwwzh+7xJ1CpUnLN1YEsGb+Xqoq5HWl1B7Uk/oHs47hctmDLeNGsiG0uFYrFmM3ZIlc3L4d1zhpE4TZKLumTpuSmVQPdu1xI6bIegGZkQq/r+nybBvajv6O+NJMXGYPYY7DQno8/zTPdFtNUKGUmH1jGb0+kFzR9FJXngFZ9h1CYvKpMdvx627pi4gLizINqvfPRrVEOfThH+/NwyOvqoFp9sGk/kRftsVJKJUm88MZX7JEz7+lVEMVWRE3iJZo7iNgfWEMIXezqdS2MH03KCgryCcvN5dc629eYe300VKO1W9LpC5UXuLL1//F8DWncHFzx93tJHvs13EiXIq6pozMu8F4XzzDiZPfs6jXc7y1wMu0MsxE6m3fZvSir/j044k4BOYhtxJAQR7NjsGv02OFL4WKKiIOrOVIZIlZSGtJ/cV3ZnAiJBTvHWPo8OZYDkVbpksClZem88Zrw1l90hk3d3fcTu7Bft0JwsvV6B9A6pXnJvDSa9O4UNjI/KKLZvOk5fiWWq5rCV7+Ps92Wc51ixkIqHGfTrt/TeW8tUnGCk8wUnzuM9r9tR0TTltrrA0SWZ3UkODuyNaN67FfM5Ue/+7PzOWrWGu/gT0/JFPTeKFZc6RuSGHXwJfos/42FUqNmdSfoM+We+QVS0m+cg7fTEXtTMdUuoGqrFrCVLdm7mxV46YP9RTGhnI3u+L+/XGMUny++sBkgvn0+xsc+WIA810TCN8ymJfa/I3u9kEEOQxjxPob9w+Cgha5vInBzahCLvpM6t/m+6qlSvBg37ZNrLdfw9Qe79B/xjJWrbVnw54fkCtbXgFpLHHii389Q7txJ+pNSY1KsJD6k70duJNdhDTFi/N+GcitbDwGWTZ3QmIoUInbBDz8R18US1hUFlKzgtM4R9GnMXfhBQoqakxEVxO+g7EDp3Ag0uzs1yfhOGcd1VJZo0ctpP4kvTZFklUkJcXLCb8GJqsWCLNRbq071VMUF0ZUltQ0E7N+RlQEFnd+ljZtR3M48DumDFqAc2woWz9px1+f6caagKs4jBzF+sBcqhr7Fy0y0zhaziwzxsbXTQXrSdg5iJc7TMe5LnqmtkatIHWzTV3QI6+UodGkEuAba5pailkIWjnVKnFaYt1EMNbITY7MRpcbJqqJwXnLGlauWM7y5VbfFQ64xstN4XsNH4CmSV1LVlQMhXnfMeKFfzB6byTpuRIKCwspLCxCqtRjKL7B7vkL2HIxlHsJsRwe346353maQjJrNfXOzDt6lMX93uOjOSe4Z61RCRqyTk3i3+9N59ztizhsdCOt0rL9gZnUzTZ1VclNdoz6gB4LXUgVp30iaR4bzYsvj8YxMp0ciVinQgqLRNOCEaEZ84s26x4xxVqqL8+hfdtRHGzg/BTZOoBlU3bVmUtAS8iKD3i28zKCSuvNIZqAhbz1ykRONLvwQ8X1ZR/w/PuzcM2oX0jWGPf6cz0VOQnExUQTHXGILwbM51RQOHeiY0jMq0JvPVsTH2qO1HXROPT8B0N3JVBZp6lbbOpG9IpKZBoNadf8ia3Q1GrSFuFvLGz1lfsRR0ZUcnHpdVMSakTq/RXv//3PtB0wlv7dZuCSUYksbjdDXmrD37p/zuc9+rPGyvTyIwpuNqm+IofEuBiioyM4PPlj5p8MJOxONDGJuejEDcJa+KiCV9C57QfMvJBKZTMOrDpSN9vUjXoFlTI16rRAAuKkqE1mJAGtXI7K2IIy1kI9Gt8yqkQeuJ8fLOmEmhusmbWD2+LaAQEM5ancDr1DhrnPhUpvls3ZT2VFfThf7bP1pF5rUzfLjFpNWmAAcVJV7QzKIjNNEqOlFq39NcuMxSls/ZioCCzqxLNt2tL/0/50m3GBlPJK4h2H0+6vz9Bt0iR6DFhDYG7VI1rprSd+x8f8460ZuORYInxqK9QCqcc2dJSaZN9IWeAaPv3GA3mNZYJv3bIfeawvJ+3ubW6GhxNu/b15h3Rp0wuBDBmOfPxcB+Z5ltQt09UVXDHtj5BQHMCyLs/z5tiD3DHtRyM6p3LJKlZR6DKLdzvPxjlDtHNX4jL5Vd6c7W7aFMdYeJRRbbuxMjCHZP+tjOnUhfHbAimoqV/WbCi+woLOr/PR8Cms9co1CWptaw2k7hrAC+8txk+MfjFtuGPPkA8G8K1nlildTchKuj7/JmMPRFJq3lJBnptFscaA0ZDB3kEv8NbcH+o36NEVcMVuGUcTZSgzTzHh9bcYd/iueTsGsVQ9xf5rmbstpH6dAFoi1nbn2XcX4mNlfhHjbIe/2p/Nd+sX7lj3kiALYXWfbkw+eIuiB9p6rZ8ENP4sGbuJ8HJFCzb1O2ZH6TC+q3OUAgpv5r3diYWeEuRaMZLB2lFaOxc0lgdhP2EZ7s3OMhrV5xGeGsu9mP/+Mzz59F958/OzpFWpMaoT2DXkJdr85Rme77aMawXWphdL4ea9Qkz7y1iu/dhfDQFLx7MprKR1NnWhmtC1/fnP5P2EFdSu5m6qRH38dvpbOUpNKBvLub5hEssv5SNr7eYuTWX+U68ZKwjdtRC7CzGUNpY/oYp7Rxbz9fd3qGwQoysWJpKataPUIjM32PjZcsCxo3EAAAqgSURBVC7VLQL7qRX7sc+JisBCOj37FE//9Q0+O51sMu2qEx0Z1u6v/OVvz9P1G/9GphexDBVFSbe4ERROYnGtyal1Jeu4u6k3L703n8u5teYXfWks/p7XaJrUDfncdFrLsFee4A9/eJVBi+xYu34961cvZmL3V+m07Doy0bj8i34ElGmBnF49hFeefJp/j7dj+5597N2xlrkjO9FhwnFyZGXEn/uawZ06M3jaCjbt3M3u/ecIzVcivbqSD9/4iOlbDnDk++9ZN/o1nusyhc2n3XHdPJY327xA76WXSSrPI8xxLO926MUXdke5nmO27RurCFn5H/45wJ4gy4sjyEm5eoqVg17mz3/rxLTtF7hdqEZXk8fV7V8wcNh8HL0TKS+P5dySIXTuPIhpKzaxY/du9p8LJbc0kaunVzO03Z95+t/jsNu+h317d7B27kg6dZjAsexqNNpibn63mPEjxzNn9S6+O3GCI/u247DlMD4p1qYDPYk7+vH8m7MbOEoFdSK7h77LxKPJtZEcpj7TUp6ZREp6Itf2LmDG6nPckVjbCM0da9TfZ1Zr0OX6VDydrlNQ08yCJlGOLqxl6Ct/5n//+Abjt7gQWVq7UEp9ex29us7FLbOSrJvOrB/xT5763z/Q7uMFrFxjz/r1q1k86UP+1fkbrlqtAG1Q/s95YizHd3Fn/q9NeyafT6NK1H4FNQn7R9Du6b/R7dtrFJiiXiyVEPeXMe8VMmNv3f4ylrs/7ldP2pUL3MhXtrhaVlueRVJKOonX9rFw5mrORhSYZn8NyzKi1xsxFNzGZcNI/tXmj/yh3UDmf7sa+/XrWb14Ej1e68ISq2X9DZ//uc8EavJucNLBjvUH3bgek0VpRSk5iaG47V3DCoeThIgRUnU2X7E+RiQRrmwc1Z6n//gH2g2Yx4rVFpnpwWtdluBntQL0526BJX9juR/fdHuOv7T/nDPJtZttie/fwTGv8bdnurLcL9cc9WJ+QlAQd2E/x/3CCQ84zIr5W/BuPCO3ZH7fr5Ywu24mR+lVicwUSl16ZQWDu3drhtQFBUVp9wj198bLy5/rN28TERFJxK0Qrvn6EJpSef80+75CH/UFAV1F7da+Pt5e+AdHEh0TS0zUba77euJzJ99k59LLcokOvsJFZzeuBN7krujtVhswyDK57eOOu28wkbFJJNz24ryLL7eT00i5G4K/jzeBkVlU6Axoy+K57uNL4M148mTm0EOMFHz/JZ8fiqFMZV7iLmiRZscTEeSDt5c/oXeTKZSL6QXUJUmE+XoTGF2AQq9FlhtDsNdFnN2uEHjzLnFZ5ajU5WQnRBLk4423fzCR0THExtzj9nVfPH3ukKcWowEEtBXZxIZfw8/fD/9rwdyMvEt0SlFdSGYt0gJlp8fz0iufcybfskhHtI9pyDw3jb5TTpIsapvokVzbyayR/ejdqwfvt+9ArylrOODkTUhUIpkSKQpFJXkJN3D5/hLRYvRHUxYKsVBBg6xSURcyel+PCzIkSXe44evFlSt+BN9JpbhGxKeGmxtGMGbTNXIVWuTF6USHBuDj5YX/9XBuR0QQGXGLkGu++IQkU9GMPfa+8h7pBSOy1Ks4O/sRZx6IxIguTWEkHucvEpJRhbaBqUlAXZpNos9yevVY8ZDbDghoZJW1Zotm2qQvDGT3nNEM6N2LHh+8Todek7Hbdx7vkLskZEqQyhVU5icS7HKUS9EV1FQVkR4TRoCPN15+1wm/HUFEZAS3Qq/h6xNCstRs4mqmvJ/1sqCmLO0OgZfPc/zwXnbu2MneQ0c54+bPrVQxOKEBo4uCh6I4nZiwAEQu8LOSmVCzzEgtYdU/a8UbZW6UkXbNBWc/McLJ7HsUNBTd+QGni8Gki2s+rN8lQw6u367k5L0CyktDsR/Uk28u18asN8q5iVM1fovf4R9DtxNTKm4pIKDKv4v3kQXNkHoTWfwmLgnG2giGRpURzNfrLgsGNDVKasSOtQJRMGioqRHjmsUd6HSoVaKJxxINYfkVc2nkwRcvGSV4bNiAR4a4CrKuJATR9igWIuZjOra6p9egEonZXAdT+coaNOLWsaZkAqITxLqOtZfN1+uzMmknek0NSqWqEZnUJ1KJkS4vfsLelMoGW3IayiM5OG8mu0KLqdHKyY0OJcjfCw/nExzYsoZlC6bz2eih9OvZk979BjJo8ED69h3KbMcgcpsLR60vtoUjCya1eIphkmK79blurJi9Ec8GM40Wsvm1bokyoqldZl9fBQMaVe1W0/XXao+Mer1pfxlxK4JrdZuGNU71KM4FFHnRhAX5433ZmZMHtrJ22QJmfDaGof160rN3XwYOGszAvn0ZOnsPgdktr2x+FDV6+DyMaGTF5GYkExcbR1JaDkVVFr/Vw+f+S+Ug6NRoGvsQDBrT/u33v+cK8pNSKVLq0MuusWzAaHaEiiG+raitIMVpcnve+vJs3eIjQZHBdeezvzNSb0VbH30SNaneh9i8ahFLD4VRaGVnf/RlPVyOhpzDDH+5OytvlDbcJkDQUhbjzoE9rtyTKlCIWoQYrqVTUV1eSHZKHHfDb+Dnfp7jB3axedNW9p2+wp3catPy5oerVaOnDRKuH9/P2esZVGrNM55GSX7Pp4asA4zs+/OTul4t/vGEOGjqUMnLKcxOJe7uTW74eeB0/AC7tjiwde8pPO/kmPYCt9Jtfs/wPp5115cSfnQNdnu9SRLXBLSms/SJ7Br0Lz7edJNi8xaPgqaSgpxCG6k/WErUJHvswsHxPGE58kdPcg+uQKtTCKowVnVtz+SzuY029BJ3C5BTkJZtip23mmg0yNs0k6iuRFouRaZqOfyuwYM/5kRQIEnPQVw01Fw9fkx2v7W0vwypt9BqwYC2Rk6VtBypTIwAaQ1DtJCf7dbPi4CxjIjz+znkFEC06Hi3TOsfVKrSl0UfdOGrS1lUWjbvE60FYCP1B2EnOmWUZXnkFovOmt/4CyJIuTL3A/rYh5n+kODBbbOleNQIGMTtfXuv5GpRU5Exj7o0W36/bwS0ZF3ewZZj/kQXKFCl+eJzT0ZN4zj2JhqpT3Hkk87TOJlo3gzMKk3T0S9WCWyHvycEjJS4Taf7xKOkmpyiv6e6//7raiyJw2f3BDpY9pcRncK//2bZWvBzIaCJZMcn3en96XTmL1rMrFFjsL9W1DBCpsmyBcouzqTnlGPEPso/yWiyLNvFXx0BY8kVvho8j4s5ln29f/Uq/ddUQFCWkBETip/vdSLFvUIsu5X+1yBga+iPQsBQTNz1QIJuBBMcEkyQ3w0Syyw7eraQkyDDf+kI5p2z+hs8q+Q2Td0KjMfi0FhJuMM0ll7Koto6TOexaNxvvBFN7BXyG6+xrXq/KgKNIt/MNvEHVUmQBrBm5np80ysbROJZnrORugWJx+ZXQJXlwcZ154h/2L3QHxtMbA2xIfC4IKAj230zDs4xFDfzR942Un9c+tq6HeIiiIATnLtRwKPZ/Mo6c9uxDQEbAr8WAsaK27ie9iHWvItlU/WwkXpTqDwG14yKArKLlKZ/eHkMmmNrgg0BGwLiskh1MTmSarTNLvO2hTTaBMWGgA0BGwKPFQI2Tf2x6k5bY2wI2BD4b0fARur/7RJga78NARsCjxUC/x+3ILT73sRliQAAAABJRU5ErkJggg==)"]},{"cell_type":"markdown","metadata":{"id":"Co9D_oNXUlgj"},"source":["The components of the formula are as follows,\n","\n","- **E⁽ˡ⁾**: the embedding table after l steps of embedding propagation, where E⁽⁰⁾ is the initial embedding table,\n","- **LeakyReLU**: the rectified linear unit used as activation function,\n","- **W**: the weights trained by the network,\n","- **I**: an identity matrix,\n","- **L**: the Laplacian matrix for the user-item graph, which is formulated as"]},{"cell_type":"markdown","metadata":{"id":"SLrIvZowUyyI"},"source":["![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOkAAAA6CAYAAABVnX5iAAAVDklEQVR4Ae2dd1hU176Gz1/35mhEjLElGsWSeKPx5CRGb2JNMWJLjp4cjSZqComJxxKjRlEsUQRF7EFRYjkWUETpKggoUqSpiBVp0oQBBAZmmGGG2e99hjqUGRgFIbnb59mPM7utb31rvXu13x7+gvhPdEB0oE078Jc2rU4UJzogOoAIqVgJRAfauAMipG28gER5ogMipGIdEB1o4w6IkLbxAhLliQ6IkIp1QHSgjTsgQtrGC0iUJzogQirWAdGBNu6ACGkbL6D/F/IU0RxatQLrfW74+gUSlVSIqkz4w2VdKE7hWshlArycsF62HMfgHGSlT58PEdI/XFX4EwqWnsZi8EgWHI/kblIa2YWlaJ6+bj9zowSVFElmOqnx7vw8YhBzDiXzuOTpMyJC+syLUkywngNSF2b3G8WqoCyKVPWO/vF2qKLZMHoAM50SRUj/eKUnKm7QgSeEVC2Xo9QIPH1b1aCqmp3qRPwPbMdmrRWrVqxi3aYt2G3ZjN1vJ/C/lU2JWlNzrvaTCGltP8RvfwIHjIRUkD3AZ8dqLNdasWShJQ7+CRSWlrWcEZpCUq8fY+GI/vTsac4azyDCIv3YN38iYz7+ip1XHiFT6TwqREhbrizEO7eSA8ZAKhQQtmM2Iz+wYO/5AJyXT2ToB//mP7fyUbQgp5RGsn5Edzp0nIRDYh5yjYL0kxYM7tKVt5b6kiktrWnRRUifsCIJctJjYnmoUNGSZWm0uraqy+iMPMUFRkCqyfZgwdsv0Xf679x7XEJh0C8Mf6kPE+2jyJG1YMmqYtg4qjsmVZAKUHrFkmHdOmL2tSupBUoR0qeoAkAJ6TFn2Pi1JR7ZxSh1eiZPd9+nvVqr6yzWX6/CI7uoDel62nwZeb0RkCqClvN2144MXRlMVpGaslRH/tGzEy9PP0xSvqIGFCMlNHp6XUhLJYRYm9P7xVeZ6RRHXonOA0JsSRu1s4ET1BRlRWI71YLjmYUo2gyklbqmWXA8oy3pasDCltzVZEgF8o7Poo9pB0bbxJIrK0PIPcrM3qaYvL+FuBwZdaZwmk91FaTtBjJ5/nwsvpjCuwMGYf7LYcLTZKh0ExYhfULfNens/2Iezo/aGAxaXV/Ow7lNPTye0OMnvcwISAtc5mDWyYRR1tfLu7eajAP8s5cppuN2cC9P3vKQdhjBspOn2DbnTTo/34Px2yKR1O1mi5A+YU3QpOHYJiFNEyFtMqSgjFrPqO6mDPnJj0ypitLoXxnVvSMDf/AkvVBn8uYJq4ney6paUu2YNCGb9IjtTB3QmZdHreJ8ahG1AotESPXaaOCAhsf33Fg08mNW+twmt7Ss5cYuBlTUP6Th8X03Fo8azwqfW89IlwalNIukm+FcSypqG+F3RkAqSK+yaXxveo63IyqniMTfP6PvS0NZeDaRQqVun7O+20+1Rwte5cTRb9rZXWUm51eM5uUuA5i2M5xHcnVNnWpTkKoLSb93g+u3ksnRimyWsZ6a1LCTHNhpx2ZbW2y32LNtmz3b9zjh7BtBcoFuyJgGyQ0/3N1cOXXqlM7mytnQJJSqisF8ZmYGweec2WOzGQcXH2JuxqFWqxsuM8Vdzmzdwsnrj1GodTNkQJdPBEkFpdSEm1bo8tCjS1FLl0ulLm9i4m7p19WwWuP3Co+55W3PDxPG8MORJAoVjVRsrR/2dpy89piSWn4Yn7TeK4yAFKGEh4Hb+Xr8JL5bu5bvJnzI9NXOXM8uoaXkoU4j7IQVk/qY8NxzA/ls4wmuZskpiD/L6smDMHtjIt9v9qSgWFmRxbYBqYAs3pttq9ay+5Q3Pi52LPp+Ax7xxZQ2UuZ6C6r6gIaizFucWjKG18zMmGDlQXBYOEEeh7BZMIOpXy5n36UUZOUjdQ05Ny/i7XGWM2fO6Gxn8QxPQVk5mg8MDGTZsmUsXLSQhQsXsmPHDuRyeXWKNR8E8gNW88GAPgxb4kVasW73qVLXzzW6Lmt1eeroCkqhuFpXgB5dydUPj4CAgFq6du3apUdXjcKn/iQoyEs8zfy/9WDKrgc8lus+iOreXSA/0IoPXzNj2E8epBbp+lH33Kf4bgykgKDM4X7EJfz8zuPrd4Wb6VJULRnsqyki/W4Ul3y98PQ8T3BMPFkyNZqyYtJuXMb7lCveYQ9QVAVUtAlIZdfYO3sE73//O2HxGWTGOTL343kcji1AqTMT/eTFVkrk2v/lxfYdmLQ3mUJFGcqiLB5EOrPso0EMHjmXPWHZ5S2dSlbA47xccnN1tzzypAo0lU37o0cZ+Ls4st1uE1uc3Im5qafF0mTg9u+PeaPXc7Qb8A0nkwprjzWo0NWlfQcmOiRRUKkrQatrnFbXHHaHZpWHiTVNV3ptXXG3W74l1RaKKoYNI3sxdU8C+YYCwLV+LBjPkFe0fnyNS2IBLdKjNBLSinoloFYqy+E09Jh58jpY+0pBbzdRQKMqLR82VOtofUgF8i8sZVgPM2YcTigfBwjyVK5H3iK9SNVMby+oiF7/Ll3amzBlfyqFVWsmgoxE528Z3LkLA2cdIK5Q0bTZPFkov20/w+27l9j9sw1+WbI68FUUiDreiW/m/MqOb4fwgokZnx++T0GtWqnV9V6FLseH9XS9odU1cz838xVNC5iQheFQpWupDRceFTeoq6q6aIoeEnXhDM7HXPCJTKVIpUEbuVoiiScq8CxeERkUZd8nzNeVYye8iEwr1lkaUJN7Jwh3l2Mcd9nBnMHd+GS3YUjV8b/z7dxf2f7t3+jc0YwZh+6S31j3uEqsMf8/EaTGJPCMz219SGUELv07nTuOYIN2nKJtOTUag7CU5d4j7FIAF/398dfZLgbFkqFUN3CtHki1SeW48tWrprTrMoW9dwtoUp1RpRMbm4FCFsnG6YtxT5c2EDggJ8rucywcr5McZMXobib0/Mc+4vJ1HwR6IK3U9fVrnWjfZQoOt/MpaUq3X6vrZoUu6+k/6dFVUcGE4lgOL7Nkb0AUUUEHWT7rexyuSpCryyiM98Nu7ije/8oKuy27+f3QXlbN/ADzFWdIkSoRBBl33WxYZXMUv8hoQs6swbx3ZybtMgSpnOits7DYd42kwDWM7dGRXp86cPNxif4HkCaX++GXCbxYu5z9LwYRm6HQeWDUgUaEtI4htb8a/6qaOo7N73fDpL8F7pnF6MYV1751zTfFgwCOH3Ziv6MjjjrbgROXSZGrjIIU1XU2jupC+78OYfnlHGR65n9qUtd+EhA0Um5772P3sVDSZPXTFPIvsuKjCSw95MPFcw7MGWRKu27j2RZT+SAqv6F+SFFrdXWl/V/fYFmQhGIjdN3xdizXlVpcX1dVPjS5AWxZYItvmpQSWQIOUwcy3iYUiUyNqigd1+9ep9c4S9yCb5CYlkr0ZnN6j1lHiKSY0rTTLPpgKlY+d5Eo1CiyfVj81kt8aqAl1fqxctxElh70xl/rxxudaN/tY+yjc5HrG9IoEghyPozT/trl7HjgOJeTZVQN2aryVP2/CGm1FQ19MBpSTfYxZvU2pff0wyRIlQ0AVj+ZsiIJaQ9TSElJJjm5ZktJz6OkTNtlq/tPf0uqhdRaC2m74ayPytNfYWrdUkGCvzOufpHcu5NAWnGZzkys9kQNGad/wPyTRWzcuRfH/XvY8MXfebHdi4zZFF4+c11xOwOQqm5gPborz7cbzrrIHOqub9eSU/1FQcJFFwO6qk8snyxJiLqMr9sxDh9x4qexPRg035OM8smcUoJ/eRuzf+0jIb+k3E+Z61f0HbaU85mF5Hj+wKBXZ3MkuaAi2ko7Jh1laEyqIcNtPhM+XcSGnQ6VfrxFl3YvMtq64sFQo0znU1kxkrRUUlJSapVzckp6edic3rkdEVIdE+t/NBrSkkvL+Hv3v2Fx8gHSJk7lltx0xW69FZYrV7JSZ1tl40qcVNlA90k/pJosZ2b360jHNxbhm9G0llyT7c3KWbNZsm4L1kvt8Ko7JlXHc+CLqaxwi+L2gySSkpJIDLdn8ismdB5uRVC2jIqGUT+kWl1z+pvScfBCfNKKmtTD0Ei8sZw1hyVrN1fr0hdXrJEEs3vxYmxPXiIq9hp7P+/L4B/dySh/+6IC0n6zDpJaUBG/WuL2Df3eWYJvZiGZR2bwSp+ZHExpIqTqeJy+nMaK05Hciq/xY0rvjnQevprALD2+l8ThZv8rayxrl/NKSxtOxRag0Ne7eGJI1TwMPMhOO1s2bdpUZ7Nlt9ddZC36akx9oMr3tOqYVJByZfVYhn2xh7BHTV+XUufcJzIshODg4Frblav3yWkwsEBF1LqKiaPJjjUTR2XSBHw3TKRfz7eYuzeC7JKmBSUI8mQigsOJjo4hIuQGqSW642AB6VUbPpm2gcsZxTVrbapEnKb3peMLb7LEN53i8n69iqjKiaPJ+2omjjTluibRX6vL4SpZugvbespRu1urK7JKV2iFroZ7khqy3X7krWHfcfxeLoqyAly/6s/r37tWQxq09E3MPncipRLS4lNzMXtrEV4ZUgrD1jKi11AWeCRTWCqgyfVh4Zvd+dD6WgNvjmj9sOUf/9zApXSdh40qkd9n9Mf0hTf5ySeVolohNpWZVOcQHxVGSJ1yDr5ylXsSJXXfja625okh1SBNiSXqahihoaEE2M1hrq03F4JCCQ0NI/pBHiq9iVan3vwfDEGqlnDTz4uAhl4W16OkSS1paW4Sd+8ncCdgNwu+seQ/ERnI660ca1A3iyFq0kJPYDXhFZ7/r/9mwKeW2Npvx36rLWt+XsD8BcvYdMSfOzmKOl1WPTk0uFtDRtgRVk8dhNno+ewLSEJeuc4pueaO9dR+dHiuPWYTlrLTPYbYoKNYTazRZVNX12F/bkuaQ1dd0QLSAEtGvj6ab21/44CTE+v/+So9hs/BxjmCm0GH+GlMd0wHzcD+/AMKU0JxshjKC12HY7E3gMTsaxxe8DHvmX/LKtsd7Nm/A4t3XmLIVEuORjzSCVKo8MNq2mD6jv6RvRcTq9ejJdfd2TStPyZ/bY+Z+c/sOHOdXKVOlE1dycZ8bwBSddJFft+5mfXaX0NY8yu2W+3ZamfPXueAco+rq5ogoF0e0W7Sk/P44Vg6eTJN5T5jRBhzrhq5XFm9xFfvSgOQaiRe/GL+Lu/O2MFVSVUPrd4dau1oFFJ1ZgDb5k1l3PvvM/rt13h9zFesdXDhXMg17iRl8rhYRn7abYJPOXFWu05aD95a6TXhiwZpxl2ignzwdHfHJzCcyOhooiIjCAkOIUob3aRdSK4/kG3CveueIlCccYswf298gyK5kyZFXX5jAXl2AjdDL+Dl6YGnXwjXHmQhSb2joyusWldocAiRzaqrrk7QFCYQ5uXKaZ8gwq/FERvqwVFnb0LvZpKdcoNQP088fAKJSc5HWZDKjSvn8fI4x+XYhxSolDxOjOSi51m8AsKIjrtLuO9pfC9HcC9LVplnbZo1fpwLiuR2WmFlkECVH37VfsTEZ+uZT6ivvdE9DUCqKUgh5ugC3uvfk57ma/AMCuXqeQfmTxqL+Te7Cc2S1xtSFJVDmtFIgEajagycICBP8GXXmtWsX7uUxVZ7uZhQWH9CzACkQslDIjwc+OHbzYRJimjKTzo1AqlA8cMYLvv74HH6GAd2bGTVz/P5ZtY0Jn00lrEfjMN80iQmfPQRk+ftIPCh7rqcgbz+IQ7VfgroXct+hnkRypTI5dqlDAEEFYoSJWXlrUhVy1HRomjjM8sDOapamUqN2uu1UTHaVket0q5p186j4azUPteoSw3fGBqAVHtJaeQ63uv2PCaT95KUX4JGkYbLN6/TpevbLDv3qF6Xu6UhFQrC2TV3NB9a/Iav/wmWTXyHjxYc5VZ+nWUpA5AiFJMQ5MJ/vK6TXaJqYNK0vlmNQqoqKamI6tCUIpfmkJ54hxsRwVz0duWI4062bFiP9baDuEekIC1fXK+fiLhHdMCgA3ogVcVsYGT3DnSshFSglCsr36GriRlfn06jsM4sW8tCqiHbcyFDX+7LdKc75MkLCfplOC/3mYR9pKT2bL5BSJU8Tk0iPV9JWVPW0uEp/j6pUIZSJiU/N4fcQm3Xo/aT1mChiAdFB3QdaCKkpZIQrM178+KrM3GK012/rrhZy0KqIGj523QzHcqKy48oUpeR6jiVXp16Mv1wEvlVUXFaKYYg1a7ZG4lKIy2prpPiZ9GBFnKgEUjbDZzCggXfMfvTdxkwaDzLDoY1GJCizrjBjbSS+mPE5pAt5HH8iz50MhnNphs5FJcJ5B6dRe9OJozdEkeuTKdZNAip8WJESI33TLyiuR1oBNLnRyzj5MmtfDnkBdr3MGd7VE4Tg1iaUahQgMscM14wGcXGaxKKyzRkOH3GK6amjNt+l1y5CGkzui3eqs050Aik5WNSSRpXt35C/849GWN1gbTipk26NF9elUSuH0kP0yEs9stAqiolZsNoepgOZJ5HWvn6c3VaYktabYX44c/igD5Io3+tmDiapJ3dlaPI8GX5qJfo8upn7L6a1cBafUsaIiANt2Z8n16Y20UhkSZxcHo/Xhr6b9wS6rzCJ0LakgUh3rtVHGgAUnVaOM5rJtHH5Dme+5/p2LhEIJEXcN/NkkmvmzFk8o/Yed1H1jwvMDcp24I8hYv2X2E+eR7r1s9j4of/wvJYDFkldYI6REib5Kd40h/JgQYg1UjTuRMRiLeHO+6+l4i+r205NZQVpXI9yJOTJz0Jjc+ltOY3a55BjgWUOfcID7zAhXM+nA+OJa1Q92dzKiWIkD6DshCTeLYONABp+euF+pYqBA2qUlV5IMezFVqZmqBGqTQQDCJC2irFIibakg40CGlLJtjC9xYhbWGDxds/ewdESA16Lq6TGrRHPPhMHNBCavY60zae4nxQKLGpbeT3gI3MvCBL51bUVUIubGfGwF78a7/4R4SNtFA8vc06oHpIqPtZzofe4Pa9B6TmKShrntecnmmWhdLHpCc/IP72VS6ccePyg+Jm+fVMsSV9psUoJtawAwJlahUqlXZT67w61/DZbXavoKnJh1rd5AD6xvIjQtqYQ+Jx0YFWdkCEtJULQExedKAxB0RIG3NIPC460MoOiJC2cgGIyYsONOaACGljDonHRQda2QER0lYuADF50YHGHBAhbcwh8bjoQCs7IELaygUgJi860JgDIqSNOSQeFx1oZQdESFu5AMTkRQcac0CEtDGHxOOiA63swP8B060KAVwSYHwAAAAASUVORK5CYII=)"]},{"cell_type":"markdown","metadata":{"id":"TcNXtDMFVLii"},"source":["#### Architecture"]},{"cell_type":"markdown","metadata":{"id":"kyKDfxNfBWl-"},"source":[""]},{"cell_type":"code","metadata":{"id":"4i1YYbJB4oGQ"},"source":["class NGCF(nn.Module):\n"," def __init__(self, n_users, n_items, emb_dim, layers, reg, node_dropout, mess_dropout,\n"," adj_mtx):\n"," super().__init__()\n","\n"," # initialize Class attributes\n"," self.n_users = n_users\n"," self.n_items = n_items\n"," self.emb_dim = emb_dim\n"," self.adj_mtx = adj_mtx\n"," self.laplacian = adj_mtx - sp.eye(adj_mtx.shape[0])\n"," self.reg = reg\n"," self.layers = layers\n"," self.n_layers = len(self.layers)\n"," self.node_dropout = node_dropout\n"," self.mess_dropout = mess_dropout\n","\n"," #self.u_g_embeddings = nn.Parameter(torch.empty(n_users, emb_dim+np.sum(self.layers)))\n"," #self.i_g_embeddings = nn.Parameter(torch.empty(n_items, emb_dim+np.sum(self.layers)))\n","\n"," # Initialize weights\n"," self.weight_dict = self._init_weights()\n"," print(\"Weights initialized.\")\n","\n"," # Create Matrix 'A', PyTorch sparse tensor of SP adjacency_mtx\n"," self.A = self._convert_sp_mat_to_sp_tensor(self.adj_mtx)\n"," self.L = self._convert_sp_mat_to_sp_tensor(self.laplacian)\n","\n"," # initialize weights\n"," def _init_weights(self):\n"," print(\"Initializing weights...\")\n"," weight_dict = nn.ParameterDict()\n","\n"," initializer = torch.nn.init.xavier_uniform_\n"," \n"," weight_dict['user_embedding'] = nn.Parameter(initializer(torch.empty(self.n_users, self.emb_dim).to(device)))\n"," weight_dict['item_embedding'] = nn.Parameter(initializer(torch.empty(self.n_items, self.emb_dim).to(device)))\n","\n"," weight_size_list = [self.emb_dim] + self.layers\n","\n"," for k in range(self.n_layers):\n"," weight_dict['W_gc_%d' %k] = nn.Parameter(initializer(torch.empty(weight_size_list[k], weight_size_list[k+1]).to(device)))\n"," weight_dict['b_gc_%d' %k] = nn.Parameter(initializer(torch.empty(1, weight_size_list[k+1]).to(device)))\n"," \n"," weight_dict['W_bi_%d' %k] = nn.Parameter(initializer(torch.empty(weight_size_list[k], weight_size_list[k+1]).to(device)))\n"," weight_dict['b_bi_%d' %k] = nn.Parameter(initializer(torch.empty(1, weight_size_list[k+1]).to(device)))\n"," \n"," return weight_dict\n","\n"," # convert sparse matrix into sparse PyTorch tensor\n"," def _convert_sp_mat_to_sp_tensor(self, X):\n"," \"\"\"\n"," Convert scipy sparse matrix to PyTorch sparse matrix\n"," Arguments:\n"," ----------\n"," X = Adjacency matrix, scipy sparse matrix\n"," \"\"\"\n"," coo = X.tocoo().astype(np.float32)\n"," i = torch.LongTensor(np.mat([coo.row, coo.col]))\n"," v = torch.FloatTensor(coo.data)\n"," res = torch.sparse.FloatTensor(i, v, coo.shape).to(device)\n"," return res\n","\n"," # apply node_dropout\n"," def _droupout_sparse(self, X):\n"," \"\"\"\n"," Drop individual locations in X\n"," \n"," Arguments:\n"," ---------\n"," X = adjacency matrix (PyTorch sparse tensor)\n"," dropout = fraction of nodes to drop\n"," noise_shape = number of non non-zero entries of X\n"," \"\"\"\n"," \n"," node_dropout_mask = ((self.node_dropout) + torch.rand(X._nnz())).floor().bool().to(device)\n"," i = X.coalesce().indices()\n"," v = X.coalesce()._values()\n"," i[:,node_dropout_mask] = 0\n"," v[node_dropout_mask] = 0\n"," X_dropout = torch.sparse.FloatTensor(i, v, X.shape).to(X.device)\n","\n"," return X_dropout.mul(1/(1-self.node_dropout))\n","\n"," def forward(self, u, i, j):\n"," \"\"\"\n"," Computes the forward pass\n"," \n"," Arguments:\n"," ---------\n"," u = user\n"," i = positive item (user interacted with item)\n"," j = negative item (user did not interact with item)\n"," \"\"\"\n"," # apply drop-out mask\n"," A_hat = self._droupout_sparse(self.A) if self.node_dropout > 0 else self.A\n"," L_hat = self._droupout_sparse(self.L) if self.node_dropout > 0 else self.L\n","\n"," ego_embeddings = torch.cat([self.weight_dict['user_embedding'], self.weight_dict['item_embedding']], 0)\n","\n"," all_embeddings = [ego_embeddings]\n","\n"," # forward pass for 'n' propagation layers\n"," for k in range(self.n_layers):\n","\n"," # weighted sum messages of neighbours\n"," side_embeddings = torch.sparse.mm(A_hat, ego_embeddings)\n"," side_L_embeddings = torch.sparse.mm(L_hat, ego_embeddings)\n","\n"," # transformed sum weighted sum messages of neighbours\n"," sum_embeddings = torch.matmul(side_embeddings, self.weight_dict['W_gc_%d' % k]) + self.weight_dict['b_gc_%d' % k]\n","\n"," # bi messages of neighbours\n"," bi_embeddings = torch.mul(ego_embeddings, side_L_embeddings)\n"," # transformed bi messages of neighbours\n"," bi_embeddings = torch.matmul(bi_embeddings, self.weight_dict['W_bi_%d' % k]) + self.weight_dict['b_bi_%d' % k]\n","\n"," # non-linear activation \n"," ego_embeddings = F.leaky_relu(sum_embeddings + bi_embeddings)\n"," # + message dropout\n"," mess_dropout_mask = nn.Dropout(self.mess_dropout)\n"," ego_embeddings = mess_dropout_mask(ego_embeddings)\n","\n"," # normalize activation\n"," norm_embeddings = F.normalize(ego_embeddings, p=2, dim=1)\n","\n"," all_embeddings.append(norm_embeddings)\n","\n"," all_embeddings = torch.cat(all_embeddings, 1)\n"," \n"," # back to user/item dimension\n"," u_g_embeddings, i_g_embeddings = all_embeddings.split([self.n_users, self.n_items], 0)\n","\n"," self.u_g_embeddings = nn.Parameter(u_g_embeddings)\n"," self.i_g_embeddings = nn.Parameter(i_g_embeddings)\n"," \n"," u_emb = u_g_embeddings[u] # user embeddings\n"," p_emb = i_g_embeddings[i] # positive item embeddings\n"," n_emb = i_g_embeddings[j] # negative item embeddings\n","\n"," y_ui = torch.mul(u_emb, p_emb).sum(dim=1)\n"," y_uj = torch.mul(u_emb, n_emb).sum(dim=1)\n"," log_prob = (torch.log(torch.sigmoid(y_ui-y_uj))).mean()\n","\n"," # compute bpr-loss\n"," bpr_loss = -log_prob\n"," if self.reg > 0.:\n"," l2norm = (torch.sum(u_emb**2)/2. + torch.sum(p_emb**2)/2. + torch.sum(n_emb**2)/2.) / u_emb.shape[0]\n"," l2reg = self.reg*l2norm\n"," bpr_loss = -log_prob + l2reg\n","\n"," return bpr_loss"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"5xbcqHLUSowG"},"source":["### Training and Evaluation"]},{"cell_type":"markdown","metadata":{"id":"N6S7uZU3Tpht"},"source":["Training is done using the standard PyTorch method. If you are already familiar with PyTorch, the following code should look familiar.\n","\n","One of the most useful functions of PyTorch is the torch.nn.Sequential() function, that takes existing and custom torch.nn modules. This makes it very easy to build and train complete networks. However, due to the nature of NCGF model structure, usage of torch.nn.Sequential() is not possible and the forward pass of the network has to be implemented ‘manually’. Using the Bayesian personalized ranking (BPR) pairwise loss, the forward pass is implemented as follows:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"LEMcstCz4vSm","outputId":"4e06cd7c-8e69-4f6b-d4b8-054d373f01cb"},"source":["# read parsed arguments\n","args = parse_args()\n","data_dir = args.data_dir\n","dataset = args.dataset\n","batch_size = args.batch_size\n","layers = eval(args.layers)\n","emb_dim = args.emb_dim\n","lr = args.lr\n","reg = args.reg\n","mess_dropout = args.mess_dropout\n","node_dropout = args.node_dropout\n","k = args.k\n","\n","# generate the NGCF-adjacency matrix\n","data_generator = Data(path=data_dir + dataset, batch_size=batch_size)\n","adj_mtx = data_generator.get_adj_mat()\n","\n","# create model name and save\n","modelname = \"NGCF\" + \\\n"," \"_bs_\" + str(batch_size) + \\\n"," \"_nemb_\" + str(emb_dim) + \\\n"," \"_layers_\" + str(layers) + \\\n"," \"_nodedr_\" + str(node_dropout) + \\\n"," \"_messdr_\" + str(mess_dropout) + \\\n"," \"_reg_\" + str(reg) + \\\n"," \"_lr_\" + str(lr)\n","\n","# create NGCF model\n","model = NGCF(data_generator.n_users, \n"," data_generator.n_items,\n"," emb_dim,\n"," layers,\n"," reg,\n"," node_dropout,\n"," mess_dropout,\n"," adj_mtx)\n","if use_cuda:\n"," model = model.cuda()\n","\n","# current best metric\n","cur_best_metric = 0\n","\n","# Adam optimizer\n","optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)\n","\n","# Set values for early stopping\n","cur_best_loss, stopping_step, should_stop = 1e3, 0, False\n","today = datetime.now()\n","\n","print(\"Start at \" + str(today))\n","print(\"Using \" + str(device) + \" for computations\")\n","print(\"Params on CUDA: \" + str(next(model.parameters()).is_cuda))\n","\n","results = {\"Epoch\": [],\n"," \"Loss\": [],\n"," \"Recall\": [],\n"," \"NDCG\": [],\n"," \"Training Time\": []}\n","\n","for epoch in range(args.n_epochs):\n","\n"," t1 = time()\n"," loss = train(model, data_generator, optimizer)\n"," training_time = time()-t1\n"," print(\"Epoch: {}, Training time: {:.2f}s, Loss: {:.4f}\".\n"," format(epoch, training_time, loss))\n","\n"," # print test evaluation metrics every N epochs (provided by args.eval_N)\n"," if epoch % args.eval_N == (args.eval_N - 1):\n"," with torch.no_grad():\n"," t2 = time()\n"," recall, ndcg = eval_model(model.u_g_embeddings.detach(),\n"," model.i_g_embeddings.detach(),\n"," data_generator.R_train,\n"," data_generator.R_test,\n"," k)\n"," print(\n"," \"Evaluate current model:\\n\",\n"," \"Epoch: {}, Validation time: {:.2f}s\".format(epoch, time()-t2),\"\\n\",\n"," \"Loss: {:.4f}:\".format(loss), \"\\n\",\n"," \"Recall@{}: {:.4f}\".format(k, recall), \"\\n\",\n"," \"NDCG@{}: {:.4f}\".format(k, ndcg)\n"," )\n","\n"," cur_best_metric, stopping_step, should_stop = \\\n"," early_stopping(recall, cur_best_metric, stopping_step, flag_step=5)\n","\n"," # save results in dict\n"," results['Epoch'].append(epoch)\n"," results['Loss'].append(loss)\n"," results['Recall'].append(recall.item())\n"," results['NDCG'].append(ndcg.item())\n"," results['Training Time'].append(training_time)\n"," else:\n"," # save results in dict\n"," results['Epoch'].append(epoch)\n"," results['Loss'].append(loss)\n"," results['Recall'].append(None)\n"," results['NDCG'].append(None)\n"," results['Training Time'].append(training_time)\n","\n"," if should_stop == True: break\n","\n","# save\n","if args.save_results:\n"," date = today.strftime(\"%d%m%Y_%H%M\")\n","\n"," # save model as .pt file\n"," if os.path.isdir(\"./models\"):\n"," torch.save(model.state_dict(), \"./models/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".pt\")\n"," else:\n"," os.mkdir(\"./models\")\n"," torch.save(model.state_dict(), \"./models/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".pt\")\n","\n"," # save results as pandas dataframe\n"," results_df = pd.DataFrame(results)\n"," results_df.set_index('Epoch', inplace=True)\n"," if os.path.isdir(\"./results\"):\n"," results_df.to_csv(\"./results/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".csv\")\n"," else:\n"," os.mkdir(\"./results\")\n"," results_df.to_csv(\"./results/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".csv\")\n"," # plot loss\n"," results_df['Loss'].plot(figsize=(12,8), title='Loss')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["n_users=943, n_items=1682\n","n_interactions=100000\n","n_train=80000, n_test=20000, sparsity=0.06305\n","Creating interaction matrices R_train and R_test...\n","Complete. Interaction matrices R_train and R_test created in 1.4850668907165527 sec\n","Loaded adjacency-matrix (shape: (2625, 2625) ) in 0.018111467361450195 sec.\n","Initializing weights...\n","Weights initialized.\n","Start at 2021-07-12 09:57:58.311285\n","Using cuda for computations\n","Params on CUDA: True\n","Epoch: 0, Training time: 9.11s, Loss: 107.9355\n","Epoch: 1, Training time: 8.88s, Loss: 101.6095\n","Epoch: 2, Training time: 8.75s, Loss: 80.7764\n","Epoch: 3, Training time: 8.76s, Loss: 76.1915\n","Epoch: 4, Training time: 8.58s, Loss: 73.0698\n","Evaluate current model:\n"," Epoch: 4, Validation time: 1.51s \n"," Loss: 73.0698: \n"," Recall@20: 0.0623 \n"," NDCG@20: 0.2352\n","Epoch: 5, Training time: 8.84s, Loss: 69.3378\n","Epoch: 6, Training time: 8.71s, Loss: 64.4498\n","Epoch: 7, Training time: 8.67s, Loss: 60.1440\n","Epoch: 8, Training time: 8.76s, Loss: 56.8538\n","Epoch: 9, Training time: 8.78s, Loss: 52.3951\n","Evaluate current model:\n"," Epoch: 9, Validation time: 1.54s \n"," Loss: 52.3951: \n"," Recall@20: 0.0837 \n"," NDCG@20: 0.2559\n","Epoch: 10, Training time: 8.72s, Loss: 50.5261\n","Epoch: 11, Training time: 8.73s, Loss: 49.2488\n","Epoch: 12, Training time: 8.72s, Loss: 48.5012\n","Epoch: 13, Training time: 8.75s, Loss: 47.5585\n","Epoch: 14, Training time: 8.82s, Loss: 47.0483\n","Evaluate current model:\n"," Epoch: 14, Validation time: 1.51s \n"," Loss: 47.0483: \n"," Recall@20: 0.0926 \n"," NDCG@20: 0.2676\n","Epoch: 15, Training time: 8.84s, Loss: 46.4847\n","Epoch: 16, Training time: 8.98s, Loss: 46.2644\n","Epoch: 17, Training time: 8.99s, Loss: 45.5963\n","Epoch: 18, Training time: 8.78s, Loss: 45.0955\n","Epoch: 19, Training time: 8.84s, Loss: 44.9321\n","Evaluate current model:\n"," Epoch: 19, Validation time: 1.55s \n"," Loss: 44.9321: \n"," Recall@20: 0.1102 \n"," NDCG@20: 0.2934\n","Epoch: 20, Training time: 8.61s, Loss: 44.4621\n","Epoch: 21, Training time: 9.02s, Loss: 44.1910\n","Epoch: 22, Training time: 8.94s, Loss: 43.7996\n","Epoch: 23, Training time: 8.83s, Loss: 43.1078\n","Epoch: 24, Training time: 9.01s, Loss: 43.1549\n","Evaluate current model:\n"," Epoch: 24, Validation time: 1.54s \n"," Loss: 43.1549: \n"," Recall@20: 0.1217 \n"," NDCG@20: 0.3255\n","Epoch: 25, Training time: 9.08s, Loss: 42.8759\n","Epoch: 26, Training time: 8.92s, Loss: 42.4126\n","Epoch: 27, Training time: 8.82s, Loss: 42.0810\n","Epoch: 28, Training time: 8.97s, Loss: 41.7865\n","Epoch: 29, Training time: 8.89s, Loss: 41.3096\n","Evaluate current model:\n"," Epoch: 29, Validation time: 1.57s \n"," Loss: 41.3096: \n"," Recall@20: 0.1257 \n"," NDCG@20: 0.3217\n","Epoch: 30, Training time: 9.15s, Loss: 40.9893\n","Epoch: 31, Training time: 9.11s, Loss: 40.8605\n","Epoch: 32, Training time: 9.06s, Loss: 40.3089\n","Epoch: 33, Training time: 8.87s, Loss: 40.1379\n","Epoch: 34, Training time: 8.89s, Loss: 39.6859\n","Evaluate current model:\n"," Epoch: 34, Validation time: 1.51s \n"," Loss: 39.6859: \n"," Recall@20: 0.1293 \n"," NDCG@20: 0.3432\n","Epoch: 35, Training time: 9.12s, Loss: 39.9238\n","Epoch: 36, Training time: 9.12s, Loss: 39.4329\n","Epoch: 37, Training time: 9.20s, Loss: 38.9671\n","Epoch: 38, Training time: 8.79s, Loss: 38.7849\n","Epoch: 39, Training time: 8.78s, Loss: 38.3410\n","Evaluate current model:\n"," Epoch: 39, Validation time: 1.54s \n"," Loss: 38.3410: \n"," Recall@20: 0.1365 \n"," NDCG@20: 0.3411\n","Epoch: 40, Training time: 8.85s, Loss: 38.6723\n","Epoch: 41, Training time: 8.78s, Loss: 37.9243\n","Epoch: 42, Training time: 9.07s, Loss: 37.8358\n","Epoch: 43, Training time: 8.85s, Loss: 37.2368\n","Epoch: 44, Training time: 8.97s, Loss: 37.4086\n","Evaluate current model:\n"," Epoch: 44, Validation time: 1.51s \n"," Loss: 37.4086: \n"," Recall@20: 0.1383 \n"," NDCG@20: 0.3554\n","Epoch: 45, Training time: 8.94s, Loss: 37.1695\n","Epoch: 46, Training time: 9.05s, Loss: 36.9502\n","Epoch: 47, Training time: 8.75s, Loss: 36.5551\n","Epoch: 48, Training time: 9.08s, Loss: 36.4953\n","Epoch: 49, Training time: 9.13s, Loss: 35.9976\n","Evaluate current model:\n"," Epoch: 49, Validation time: 1.54s \n"," Loss: 35.9976: \n"," Recall@20: 0.1397 \n"," NDCG@20: 0.3541\n","Epoch: 50, Training time: 8.79s, Loss: 35.8774\n","Epoch: 51, Training time: 9.03s, Loss: 36.0130\n","Epoch: 52, Training time: 9.00s, Loss: 35.4460\n","Epoch: 53, Training time: 8.76s, Loss: 35.2867\n","Epoch: 54, Training time: 9.11s, Loss: 35.4907\n","Evaluate current model:\n"," Epoch: 54, Validation time: 1.53s \n"," Loss: 35.4907: \n"," Recall@20: 0.1435 \n"," NDCG@20: 0.3563\n","Epoch: 55, Training time: 8.97s, Loss: 35.1628\n","Epoch: 56, Training time: 8.86s, Loss: 34.5842\n","Epoch: 57, Training time: 8.83s, Loss: 34.1935\n","Epoch: 58, Training time: 8.88s, Loss: 34.3039\n","Epoch: 59, Training time: 8.79s, Loss: 34.2499\n","Evaluate current model:\n"," Epoch: 59, Validation time: 1.49s \n"," Loss: 34.2499: \n"," Recall@20: 0.1495 \n"," NDCG@20: 0.3704\n","Epoch: 60, Training time: 8.84s, Loss: 33.9897\n","Epoch: 61, Training time: 8.66s, Loss: 33.2779\n","Epoch: 62, Training time: 8.86s, Loss: 33.2062\n","Epoch: 63, Training time: 8.78s, Loss: 32.9654\n","Epoch: 64, Training time: 9.03s, Loss: 32.2721\n","Evaluate current model:\n"," Epoch: 64, Validation time: 1.51s \n"," Loss: 32.2721: \n"," Recall@20: 0.1497 \n"," NDCG@20: 0.3725\n","Epoch: 65, Training time: 8.90s, Loss: 32.5445\n","Epoch: 66, Training time: 8.85s, Loss: 32.1805\n","Epoch: 67, Training time: 8.81s, Loss: 32.1525\n","Epoch: 68, Training time: 8.80s, Loss: 31.7560\n","Epoch: 69, Training time: 8.81s, Loss: 31.3688\n","Evaluate current model:\n"," Epoch: 69, Validation time: 1.51s \n"," Loss: 31.3688: \n"," Recall@20: 0.1536 \n"," NDCG@20: 0.3816\n","Epoch: 70, Training time: 8.55s, Loss: 31.3098\n","Epoch: 71, Training time: 8.87s, Loss: 31.3700\n","Epoch: 72, Training time: 8.72s, Loss: 31.1579\n","Epoch: 73, Training time: 8.76s, Loss: 30.1733\n","Epoch: 74, Training time: 8.76s, Loss: 30.5201\n","Evaluate current model:\n"," Epoch: 74, Validation time: 1.50s \n"," Loss: 30.5201: \n"," Recall@20: 0.1581 \n"," NDCG@20: 0.3809\n","Epoch: 75, Training time: 8.70s, Loss: 30.2994\n","Epoch: 76, Training time: 8.76s, Loss: 29.8949\n","Epoch: 77, Training time: 8.77s, Loss: 29.7122\n","Epoch: 78, Training time: 8.74s, Loss: 29.7030\n","Epoch: 79, Training time: 8.64s, Loss: 29.6655\n","Evaluate current model:\n"," Epoch: 79, Validation time: 1.49s \n"," Loss: 29.6655: \n"," Recall@20: 0.1609 \n"," NDCG@20: 0.3873\n","Epoch: 80, Training time: 8.94s, Loss: 29.6567\n","Epoch: 81, Training time: 8.87s, Loss: 29.5109\n","Epoch: 82, Training time: 8.91s, Loss: 29.1704\n","Epoch: 83, Training time: 8.82s, Loss: 28.6625\n","Epoch: 84, Training time: 8.79s, Loss: 28.7304\n","Evaluate current model:\n"," Epoch: 84, Validation time: 1.48s \n"," Loss: 28.7304: \n"," Recall@20: 0.1613 \n"," NDCG@20: 0.3908\n","Epoch: 85, Training time: 8.85s, Loss: 29.0495\n","Epoch: 86, Training time: 8.76s, Loss: 28.4390\n","Epoch: 87, Training time: 8.81s, Loss: 28.5633\n","Epoch: 88, Training time: 8.83s, Loss: 28.3275\n","Epoch: 89, Training time: 8.96s, Loss: 27.8343\n","Evaluate current model:\n"," Epoch: 89, Validation time: 1.52s \n"," Loss: 27.8343: \n"," Recall@20: 0.1591 \n"," NDCG@20: 0.3895\n","Epoch: 90, Training time: 8.92s, Loss: 28.3271\n","Epoch: 91, Training time: 8.85s, Loss: 28.0346\n","Epoch: 92, Training time: 8.69s, Loss: 27.7937\n","Epoch: 93, Training time: 8.93s, Loss: 27.5649\n","Epoch: 94, Training time: 9.08s, Loss: 27.9189\n","Evaluate current model:\n"," Epoch: 94, Validation time: 1.50s \n"," Loss: 27.9189: \n"," Recall@20: 0.1611 \n"," NDCG@20: 0.3912\n","Epoch: 95, Training time: 8.86s, Loss: 27.9343\n","Epoch: 96, Training time: 8.83s, Loss: 27.2735\n","Epoch: 97, Training time: 8.92s, Loss: 27.3794\n","Epoch: 98, Training time: 8.84s, Loss: 27.2788\n","Epoch: 99, Training time: 8.86s, Loss: 27.4216\n","Evaluate current model:\n"," Epoch: 99, Validation time: 1.50s \n"," Loss: 27.4216: \n"," Recall@20: 0.1656 \n"," NDCG@20: 0.3922\n","Epoch: 100, Training time: 8.71s, Loss: 26.6066\n","Epoch: 101, Training time: 8.88s, Loss: 27.1389\n","Epoch: 102, Training time: 9.04s, Loss: 26.6459\n","Epoch: 103, Training time: 8.71s, Loss: 26.8171\n","Epoch: 104, Training time: 8.91s, Loss: 26.7730\n","Evaluate current model:\n"," Epoch: 104, Validation time: 1.49s \n"," Loss: 26.7730: \n"," Recall@20: 0.1627 \n"," NDCG@20: 0.3926\n","Epoch: 105, Training time: 9.06s, Loss: 26.4580\n","Epoch: 106, Training time: 9.12s, Loss: 25.9192\n","Epoch: 107, Training time: 8.93s, Loss: 26.4427\n","Epoch: 108, Training time: 8.77s, Loss: 26.3804\n","Epoch: 109, Training time: 8.86s, Loss: 26.1349\n","Evaluate current model:\n"," Epoch: 109, Validation time: 1.52s \n"," Loss: 26.1349: \n"," Recall@20: 0.1691 \n"," NDCG@20: 0.3950\n","Epoch: 110, Training time: 8.81s, Loss: 25.8410\n","Epoch: 111, Training time: 8.84s, Loss: 25.9275\n","Epoch: 112, Training time: 8.77s, Loss: 25.9278\n","Epoch: 113, Training time: 8.92s, Loss: 26.2235\n","Epoch: 114, Training time: 8.90s, Loss: 25.4737\n","Evaluate current model:\n"," Epoch: 114, Validation time: 1.50s \n"," Loss: 25.4737: \n"," Recall@20: 0.1673 \n"," NDCG@20: 0.3995\n","Epoch: 115, Training time: 8.78s, Loss: 25.7582\n","Epoch: 116, Training time: 8.77s, Loss: 25.3173\n","Epoch: 117, Training time: 8.63s, Loss: 25.4568\n","Epoch: 118, Training time: 8.63s, Loss: 25.3934\n","Epoch: 119, Training time: 8.63s, Loss: 25.2544\n","Evaluate current model:\n"," Epoch: 119, Validation time: 1.50s \n"," Loss: 25.2544: \n"," Recall@20: 0.1689 \n"," NDCG@20: 0.4028\n","Epoch: 120, Training time: 8.77s, Loss: 24.9747\n","Epoch: 121, Training time: 8.93s, Loss: 24.7825\n","Epoch: 122, Training time: 8.92s, Loss: 25.2147\n","Epoch: 123, Training time: 8.79s, Loss: 24.5176\n","Epoch: 124, Training time: 8.72s, Loss: 24.7453\n","Evaluate current model:\n"," Epoch: 124, Validation time: 1.48s \n"," Loss: 24.7453: \n"," Recall@20: 0.1682 \n"," NDCG@20: 0.3954\n","Epoch: 125, Training time: 8.78s, Loss: 24.9444\n","Epoch: 126, Training time: 8.81s, Loss: 24.9258\n","Epoch: 127, Training time: 8.77s, Loss: 24.5360\n","Epoch: 128, Training time: 8.70s, Loss: 24.4527\n","Epoch: 129, Training time: 8.65s, Loss: 24.5864\n","Evaluate current model:\n"," Epoch: 129, Validation time: 1.48s \n"," Loss: 24.5864: \n"," Recall@20: 0.1689 \n"," NDCG@20: 0.3977\n","Epoch: 130, Training time: 8.66s, Loss: 24.2351\n","Epoch: 131, Training time: 8.84s, Loss: 24.4298\n","Epoch: 132, Training time: 8.57s, Loss: 24.3624\n","Epoch: 133, Training time: 8.74s, Loss: 24.1980\n","Epoch: 134, Training time: 8.84s, Loss: 24.0672\n","Evaluate current model:\n"," Epoch: 134, Validation time: 1.47s \n"," Loss: 24.0672: \n"," Recall@20: 0.1735 \n"," NDCG@20: 0.4069\n","Epoch: 135, Training time: 8.75s, Loss: 24.4691\n","Epoch: 136, Training time: 8.67s, Loss: 23.9019\n","Epoch: 137, Training time: 8.77s, Loss: 24.1378\n","Epoch: 138, Training time: 8.68s, Loss: 23.8090\n","Epoch: 139, Training time: 8.81s, Loss: 23.9487\n","Evaluate current model:\n"," Epoch: 139, Validation time: 1.48s \n"," Loss: 23.9487: \n"," Recall@20: 0.1687 \n"," NDCG@20: 0.4037\n","Epoch: 140, Training time: 8.64s, Loss: 23.8015\n","Epoch: 141, Training time: 8.57s, Loss: 24.0985\n","Epoch: 142, Training time: 8.70s, Loss: 23.8640\n","Epoch: 143, Training time: 8.77s, Loss: 23.5799\n","Epoch: 144, Training time: 8.77s, Loss: 23.7568\n","Evaluate current model:\n"," Epoch: 144, Validation time: 1.48s \n"," Loss: 23.7568: \n"," Recall@20: 0.1708 \n"," NDCG@20: 0.4068\n","Epoch: 145, Training time: 8.75s, Loss: 23.6537\n","Epoch: 146, Training time: 8.77s, Loss: 23.8114\n","Epoch: 147, Training time: 8.64s, Loss: 23.5442\n","Epoch: 148, Training time: 8.51s, Loss: 23.2413\n","Epoch: 149, Training time: 8.77s, Loss: 23.5159\n","Evaluate current model:\n"," Epoch: 149, Validation time: 1.49s \n"," Loss: 23.5159: \n"," Recall@20: 0.1698 \n"," NDCG@20: 0.4052\n","Epoch: 150, Training time: 8.67s, Loss: 23.4435\n","Epoch: 151, Training time: 8.54s, Loss: 23.5388\n","Epoch: 152, Training time: 8.54s, Loss: 23.2494\n","Epoch: 153, Training time: 8.60s, Loss: 23.1259\n","Epoch: 154, Training time: 8.68s, Loss: 23.1326\n","Evaluate current model:\n"," Epoch: 154, Validation time: 1.49s \n"," Loss: 23.1326: \n"," Recall@20: 0.1709 \n"," NDCG@20: 0.4059\n","Epoch: 155, Training time: 8.69s, Loss: 22.8828\n","Epoch: 156, Training time: 8.60s, Loss: 23.0292\n","Epoch: 157, Training time: 8.54s, Loss: 22.9355\n","Epoch: 158, Training time: 8.61s, Loss: 22.7000\n","Epoch: 159, Training time: 8.83s, Loss: 22.9723\n","Evaluate current model:\n"," Epoch: 159, Validation time: 1.47s \n"," Loss: 22.9723: \n"," Recall@20: 0.1731 \n"," NDCG@20: 0.4118\n","Early stopping at step: 5 log:0.1731223464012146\n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAsUAAAHwCAYAAABOlBKbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3zd5X33//d1ls7R3tOSlzzkiY0xZhMgCZvQ7EFIQ0Lvlmb3kdk2d9M0bXqnd0hzN00hJCG/JCQpGSYJGZhA2QYbG+8hW7b23jo60hnX7w/JRhgZrPk90vf1fDz8sM/3DH30fQjz5uJzfS5jrRUAAADgZh6nCwAAAACcRigGAACA6xGKAQAA4HqEYgAAALgeoRgAAACuRygGAACA6xGKAQAA4HqEYgBIEsaYE8aYa5yuAwDciFAMAAAA1yMUA0ASM8akGGPuNsY0jv662xiTMvpcvjHmN8aYbmNMpzHmSWOMZ/S5zxhjGowxfcaYw8aYq539TgAgufmcLgAA8Jq+IGmLpPMkWUlbJf2tpL+T9ClJ9ZIKRl+7RZI1xqyQ9NeSLrDWNhpjFknyzm7ZADC3sFIMAMntvZK+ZK1ttda2SfoHSbeNPheVVCJpobU2aq190lprJcUlpUhaZYzxW2tPWGuPOVI9AMwRhGIASG6lkk6OeXxy9Jok/R9J1ZL+aIw5boz5rCRZa6slfVzS/5bUaoz5iTGmVACAsyIUA0Bya5S0cMzjitFrstb2WWs/Za1dIulmSZ881Ttsrf2xtfbS0fdaSV+d3bIBYG4hFANAcvEbY4Knfkl6QNLfGmMKjDH5kv5e0g8lyRhzozGm0hhjJPVopG0iYYxZYYy5anRDXkTSoKSEM98OAMwNhGIASC4PayTEnvoVlLRD0h5JeyW9KOnLo69dJmmbpH5Jz0r6lrX2MY30E/+LpHZJzZIKJX1u9r4FAJh7zMieDAAAAMC9WCkGAACA6xGKAQAA4HqEYgAAALgeoRgAAACuRygGAACA6/mcLkCS8vPz7aJFi5wuAwAAAPPczp072621BWdeT4pQvGjRIu3YscPpMgAAADDPGWNOjned9gkAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6hGIAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6hGIAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6hGIAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6rg3F1lp1h4c1FIs7XQoAAAAc5tpQ/HR1h8770iPaXdvtdCkAAABwmGtD8YKckCSprmvQ4UoAAADgNNeG4tLskIyR6jrDTpcCAAAAh7k2FAd8HpVmhQjFAAAAcG8olkZaKOq6CMUAAABu5+pQXJ6bqlpWigEAAFzP1aG4IjdVLb1DikQZywYAAOBmrg7F5bkjEygauplAAQAA4GauDsUVuamSRAsFAACAy7k6FJfnjITiekIxAACAq7k6FBdkpCjF52GlGAAAwOVcHYqNMSrPTVVdJz3FAAAAbubqUCxJ5cwqBgAAcD1CMbOKAQAAXM/1obgiN1V9kZh6wlGnSwEAAIBDXB+KF+Qwlg0AAMDtXB+KT80qpq8YAADAvVwfik+dalfHSjEAAIBruT4UZwT9yk710z4BAADgYq4PxdJIC0VdF7OKAQAA3IpQrJHjnmmfAAAAcC9CsaQFuSE1dA0qkbBOlwIAAAAHEIo10j4xHE+opS/idCkAAABwAKFYI+0TklTbQQsFAACAGxGKNXZWMZvtAAAA3IhQLKk0OyRjONUOAADArQjFkgI+j0oyg6onFAMAALgSoXhUeW4qK8UAAAAuRSgeVZ6bqrouQjEAAIAbEYpHleekqqV3SJFo3OlSAAAAMMsIxaMKMlIkSV3hYYcrAQAAwGwjFI9KDXglSYPDrBQDAAC4DaF4VNA/GoppnwAAAHAdQvGo0OhKMT3FAAAA7vO6odgY811jTKsxZt+Ya7nGmEeMMUdHf88ZvW6MMf9ujKk2xuwxxmycyeKnU+jUSvFwwuFKAAAAMNvOZaX4+5KuPePaZyU9aq1dJunR0ceSdJ2kZaO/7pT0n9NT5swL0T4BAADgWq8biq21T0jqPOPyLZLuH/3z/ZLeMub6D+yI5yRlG2NKpqvYmRQKjNwKQjEAAID7TLanuMha2zT652ZJRaN/LpNUN+Z19aPXXsUYc6cxZocxZkdbW9sky5g+oYBPkjQ4HHO4EgAAAMy2KW+0s9ZaSXYS77vHWrvJWrupoKBgqmVM2cs9xawUAwAAuM1kQ3HLqbaI0d9bR683SCof87oFo9eS3ss9xWy0AwAAcJvJhuKHJN0++ufbJW0dc/39o1MotkjqGdNmkdRSfPQUAwAAuJXv9V5gjHlA0pWS8o0x9ZK+KOlfJP3MGHOHpJOS3jH68oclXS+pWlJY0p/PQM0zwuMxCvo9zCkGAABwodcNxdbad5/lqavHea2VdNdUi3JKyO+lpxgAAMCFONFujJDfS/sEAACACxGKxwgFWCkGAABwI0LxGKEAK8UAAABuRCgeg55iAAAAdyIUjxGkpxgAAMCVCMVjhPxeRrIBAAC4EKF4DHqKAQAA3IlQPAY9xQAAAO5EKB6DkWwAAADuRCgeg8M7AAAA3IlQPEbI71UsYRWNJ5wuBQAAALOIUDxGKOCVJFaLAQAAXIZQPEbQPxKKI/QVAwAAuAqheIyQn5ViAAAANyIUj0H7BAAAgDsRisc4HYppnwAAAHAVQvEYp9snCMUAAACuQigeg55iAAAAdyIUj0FPMQAAgDsRisegfQIAAMCdCMVjnJ5TzEoxAACAqxCKx6B9AgAAwJ0IxWO83D6RcLgSAAAAzCZC8Rhej1HA51E4GnO6FAAAAMwiQvEZQn6vImy0AwAAcBVC8RlCfi89xQAAAC5DKD5DKODVYJSeYgAAADchFJ8h6PcypxgAAMBlCMVnSA14mVMMAADgMoTiM9BTDAAA4D6E4jME/V6FaZ8AAABwFULxGUK0TwAAALgOofgMIb+HjXYAAAAuQyg+Az3FAAAA7kMoPkMwQCgGAABwG0LxGVL9Pg3HEoonrNOlAAAAYJYQis8QCozcEjbbAQAAuAeh+Awhv1eSaKEAAABwEULxGYKnQjETKAAAAFyDUHyGUICVYgAAALchFJ8hxEoxAACA6xCKz0BPMQAAgPsQis9A+wQAAID7EIrPcCoUR2ifAAAAcA1C8RlonwAAAHAfQvEZToXiMCvFAAAArkEoPkPwVPsEK8UAAACuQSg+AyPZAAAA3IdQfAa/1yOfx9BTDAAA4CKE4nGEAl5CMQAAgIsQiscR8nvpKQYAAHARQvE4QgEvPcUAAAAuQigeR8jvZSQbAACAixCKxxH001MMAADgJoTicdBTDAAA4C6E4nEwfQIAAMBdCMXjYKMdAACAuxCKxzHSPpFwugwAAADMEkLxOEJstAMAAHAVQvE4aJ8AAABwF0LxOE6NZEskrNOlAAAAYBYQiscR8nslSUMx+ooBAADcgFA8jpB/5LbQVwwAAOAOhOJxpAZ8kgjFAAAAbkEoHkcwMNI+wWY7AAAAdyAUj+NUTzFHPQMAALgDoXgcp0Ix7RMAAADuQCgeRygwclvCtE8AAAC4AqF4HEE/PcUAAABuQigeBz3FAAAA7kIoHgcj2QAAANyFUDyOEO0TAAAArkIoHkcwwIl2AAAAbkIoHkfA65HH0FMMAADgFoTicRhjFPJ7GckGAADgEoTiswgFvLRPAAAAuASh+CxCAa8irBQDAAC4AqH4LEJ+VooBAADcglB8FoRiAAAA9yAUn0XQ72VOMQAAgEsQis8iFPAykg0AAMAlCMVnwUg2AAAA9yAUnwU9xQAAAO5BKD6L1BSv+odiTpcBAACAWTClUGyM+YQxZr8xZp8x5gFjTNAYs9gYs90YU22M+akxJjBdxc6mRXlp6g5H1dE/5HQpAAAAmGGTDsXGmDJJH5W0yVq7RpJX0rskfVXS1621lZK6JN0xHYXOtpXFmZKkw819DlcCAACAmTbV9gmfpJAxxicpVVKTpKskPTj6/P2S3jLFr+GIlSUZkqSDhGIAAIB5b9Kh2FrbIOlrkmo1EoZ7JO2U1G2tPdWMWy+pbLz3G2PuNMbsMMbsaGtrm2wZMyY/PUX56Sk61NTrdCkAAACYYVNpn8iRdIukxZJKJaVJuvZc32+tvcdau8lau6mgoGCyZcyoqpIMHWwmFAMAAMx3U2mfuEZSjbW2zVoblfQLSZdIyh5tp5CkBZIaplijY6pKMnWkpV+xeMLpUgAAADCDphKKayVtMcakGmOMpKslHZD0mKS3jb7mdklbp1aic1YWZ2g4ltCJjgGnSwEAAMAMmkpP8XaNbKh7UdLe0c+6R9JnJH3SGFMtKU/SfdNQpyNOTaA42MRmOwAAgPnM9/ovOTtr7RclffGMy8clbZ7K5yaLpYVp8nmMDjX36qb1pU6XAwAAgBnCiXavIcXn1dKCdB1ipRgAAGBeIxS/jpUlGTrErGIAAIB5jVD8OlYWZ6qhe1A9g1GnSwEAAMAMIRS/jlMn23GIBwAAwPxFKH4dq0pGJlDQQgEAADB/EYpfR2FGinJS/TrEyXYAAADzFqH4dRhjtLI4k1nFAAAA8xih+BysLMnQ4eY+JRLW6VIAAAAwAwjF56CqOFOD0bhqO8NOlwIAAIAZQCg+B6cnUNBXDAAAMC8Ris/BssIMeYzoKwYAAJinCMXnIBTwalF+mg4wqxgAAGBeIhSfo/PKs7XjRCeb7QAAAOYhQvE5urQyX13hqA7SVwwAADDvEIrP0SWV+ZKkp6vbHa4EAAAA041QfI6KMoOqLEzXU9UdTpcCAACAaUYonoBLK/P1Qk2nhmJxp0sBAADANCIUT8AllfkajMa1q7bb6VIAAAAwjQjFE3Dhklx5DH3FAAAA8w2heAIyg36tL88mFAMAAMwzhOIJurQyXy/V96g3EnW6FAAAAEwTQvEEXbw0X/GE1fbjnU6XAgAAgGlCKJ6gjQuzFfR7aKEAAACYRwjFE5Ti82rz4jxCMQAAwDxCKJ6ES5bm6Whrv1p6I06XAgAAgGlAKJ6EU0c+P3OM1WIAAID5gFA8CatKMpWd6tezxzjyGQAAYD4gFE+Cx2O0sSJHL3KyHQAAwLxAKJ6kjRXZqm7tV0+YecUAAABzHaF4kjZU5EiSdtezWgwAADDXEYonaX15tjxGevFkl9OlAAAAYIoIxZOUnuLT8qIM7apjpRgAAGCuIxRPwYaKHO2q7VIiYZ0uBQAAAFNAKJ6CjRXZ6ovEdKyt3+lSAAAAMAWE4ik4tdluF6PZAAAA5jRC8RQsyU9TVsivF2vZbAcAADCXEYqnwOMx2lCRzUoxAADAHEconqIN5Tk60tqn3giHeAAAAMxVhOIp2rgwW9ZKLzGaDQAAYM4iFE/R+vJsGcNmOwAAgLmMUDxFmUG/lhWms9kOAABgDiMUT4ONFTnaVdvNIR4AAABzFKF4GmyoyFbPYFQ1HQNOlwIAAIBJIBRPg40c4gEAADCnEYqnwZKCdKX4PDrc3Ot0KQAAAJgEQvE08HqMlhak60hLv9OlAAAAYBIIxdNkeVG6jrb0OV0GAAAAJoFQPE2WF2eosSfCyXYAAABzEKF4miwvzJAkHaWFAgAAYM4hFE+T5UWnQjEtFAAAAHMNoXiaLMgJKeT3stkOAABgDiIUTxOPx6iyMF1HW1kpBgAAmGsIxdNoeVGGDjcTigEAAOYaQvE0Wl6Urta+IfWEmUABAAAwlxCKp9GpzXZHaKEAAACYUwjF02hZUbok6QgTKAAAAOYUQvE0KssOKS3g1RH6igEAAOYUQvE0MsaosiiDsWwAAABzDKF4mq0oYiwbAADAXEMonmbLizLU3j+szoFhp0sBAADAOSIUT7NlpyZQsNkOAABgziAUT7PlTKAAAACYcwjF06w4M6iMFB+hGAAAYA4hFE8zY4yWFzOBAgAAYC4hFM+A5UXpOtrSJ2ut06UAAADgHBCKZ8Cywgx1haNq72cCBQAAwFxAKJ4BK4pHJlAcau51uBIAAACcC0LxDFhVkilJOtBIKAYAAJgLCMUzICctoNKsoPYTigEAAOYEQvEMWVWapf2NPU6XAQAAgHNAKJ4hq0szdbx9QOHhmNOlAAAA4HUQimfImrIsWSsdbKKFAgAAINkRimfI6tKRzXb0FQMAACQ/QvEMKckKKifVr/0NhGIAAIBkRyieIcYYrS7N0v4mNtsBAAAkO0LxDFpdmqkjzf2KxhNOlwIAAIDXQCieQatKMzUcT+hoS7/TpQAAAOA1EIpn0OrSLEliXjEAAECSIxTPoMX5aQr5vUygAAAASHKE4hnk9RhVlWToAKEYAAAgqRGKZ9jq0iwdaOpVImGdLgUAAABnQSieYWvKMtU/FFNtZ9jpUgAAAHAWhOIZdmqz3T422wEAACStKYViY0y2MeZBY8whY8xBY8xFxphcY8wjxpijo7/nTFexc9GyonT5PIbNdgAAAElsqivF35D0e2vtSknrJR2U9FlJj1prl0l6dPSxa6X4vFpWlEEoBgAASGKTDsXGmCxJl0u6T5KstcPW2m5Jt0i6f/Rl90t6y1SLnOtWl2bqQGOPrGWzHQAAQDKaykrxYkltkr5njNlljPmOMSZNUpG1tmn0Nc2SiqZa5Fy3tixL7f3DaugedLoUAAAAjGMqodgnaaOk/7TWbpA0oDNaJezI0ui4y6PGmDuNMTuMMTva2tqmUEby27w4V5K0/Xinw5UAAABgPFMJxfWS6q2120cfP6iRkNxijCmRpNHfW8d7s7X2HmvtJmvtpoKCgimUkfxWFGUoJ9WvZ493OF0KAAAAxjHpUGytbZZUZ4xZMXrpakkHJD0k6fbRa7dL2jqlCucBj8fowsV5eo5QDAAAkJR8U3z/RyT9yBgTkHRc0p9rJGj/zBhzh6STkt4xxa8xL2xZkqvf729WXWdY5bmpTpcDAACAMaYUiq21uyVtGuepq6fyufPRlqV5kqTtNZ2EYgAAgCTDiXazZHlhhnLTArRQAAAAJCFC8SwZ6SvO1bPHCMUAAADJhlA8i7YsyVND96DqOsNOlwIAAIAxCMWzaMuSkb5iWigAAACSC6F4Fi0rTB/tK+YQDwAAgGRCKJ5FHo/RliW5eu54h0YO+wMAAEAyIBTPslN9xfVdg06XAgAAgFGE4ll2qq+YI58BAACSB6F4li0rTFce84oBAACSCqF4lhljdOGSXD1fw2Y7AACAZEEodsDasmzVdw2qJxx1uhQAAACIUOyIVaWZkqQDTb0OVwIAAACJUOyIqpIMSdJBQjEAAEBSIBQ7oDAjqPz0FEIxAABAkiAUO6SqJIP2CQAAgCRBKHbIqpJMHW3pVzSecLoUAAAA1yMUO2RVaaaG4wkda+t3uhQAAADXIxQ7pKpkZAIFfcUAAADOIxQ7ZEl+mgI+jw429TldCgAAgOsRih3i83q0oihDBxpZKQYAAHAaodhBVSUZOtjUK2ut06UAAAC4GqHYQatKMtUxMKy2viGnSwEAAHA1QrGDTm22289mOwAAAEcRih1UVcoECgAAgGRAKHZQZtCvBTkhNtsBAAA4jFDssKqSTFaKAQAAHEYodtiqkkzVtA9ocDjudCkAAACuRSh2WFVJphJWOtzCIR4AAABOIRQ7bDWb7QAAABxHKHbYgpyQMlJ8bLYDAABwEKHYYcYYrS/P1uNHWhVPcLIdAACAEwjFSeC9F1aornNQjxxocboUAAAAVyIUJ4E3rS7WgpyQvvtUjdOlAAAAuBKhOAl4PUYfuHiRnj/Rqb31PU6XAwAA4DqE4iTxzgvKlZ7i031PHXe6FAAAANchFCeJjKBf79hUrt/saVJzT8TpcgAAAFyFUJxE/vySRUpYqx88e8LpUgAAAFyFUJxEynNT9aZVxfrR9lqFh2NOlwMAAOAahOIkc8dli9UzGNXPXqhzuhQAAADXIBQnmU0Lc3TRkjz92yNH1NpLbzEAAMBsIBQnGWOMvvJnazUcS+jvt+53uhwAAABXIBQnocX5afrEG5fr9/ub9bu9TU6XAwAAMO8RipPUhy5drDVlmfq7rfvVE446XQ4AAMC8RihOUj6vR1996zp1hYf15d8ecLocAACAeY1QnMRWl2bpLy5fov/eWa+tuxucLgcAAGDeIhQnuY9evUybF+Xq4z/drfufOeF0OQAAAPMSoTjJBf1e/eCOzbqmqkhffGi/vvaHw7LWOl0WAADAvEIongOCfq/+870b9e7N5fp/j1Xrsz/fq3iCYAwAADBdfE4XgHPj83r0lVvXKj89Rd/8U7VCAa++eNMqGWOcLg0AAGDOIxTPIcYYfepNKxSJxnXvkzVakBPShy5b4nRZAAAAcx6heA763HVVauge1Jd/e1AlWSHdsK7E6ZIAAADmNHqK5yCPx+j/vuM8bVqYo0/8bLdeONHpdEkAAABzGqF4jgr6vbr3/Zu0IDukD37vBf1o+0kl2HwHAAAwKYTiOSwnLaAf3LFZq0oz9YVf7tPbvv2MDjb1Ol0WAADAnEMonuMW5KTqJ3du0b+9fb1OdIR14zef0j8/fFCRaNzp0gAAAOYMQvE8YIzRW89foEc/eYXetnGB/uuJ47rpm09pb32P06UBAADMCYTieSQnLaCvvm2d7v/gZvVGorr1W0/r7m1HFI0nnC4NAAAgqZlkODJ406ZNdseOHU6XMa/0hKP64kP79KvdjcpLC+iSynxdWpmvS5flqzQ75HR5AAAAjjDG7LTWbjrzOnOK56msVL/uftcG3XxeqX79UpOeqm7XQy81SpL+bGOZvnLrWgX9XoerBAAASA6E4nnuqpVFumplkay1OtLSr1/uatC3/+eYjrb0679uO59VYwAAANFT7BrGGK0oztBnr1upe9+/STXtA7r5/z3FwR8AAAAiFLvSG1cV6Vd3XayMoF/vufc53b3tCCPcAACAqxGKXaqyMEO/uusSXbemRHdvO6o33/2EHjvc6nRZAAAAjmD6BPR0dbv+bus+HW8b0DVVhbpqZZHWlmVpeXG6UnxsxgMAAPPH2aZPEIohSRqOJXTvk8d175PH1R2OSpL8XqMN5Tn61JuW68IleQ5XCAAAMHWEYpwTa63qOge1r7FHext6tHVXgxp7IrpuTbE+d12VKvJSnS4RAABg0gjFmJTB4bi+8+RxfevxY4onrN61uVw3ry/VxooceTzG6fIAAAAmhFCMKWnuiehrfzysh3Y3ajieUFFmiq5dXay3nV+utQuynC4PAADgnBCKMS36IlH96VCrHt7bpMcPt2koltBly/L1l1cu1UVL8mQMq8cAACB5EYox7XojUf3ouVrd91SN2vuHdF55tr50y2qtW5DtdGkAAADjOlsoZk4xJi0z6NdfXrlUT33mDfrHt6xRc09Eb//2s/r1S41OlwYAADAhhGJMWdDv1W1bFuq3H71U6xZk6SMP7NLd244oGf4vBAAAwLnwOV0A5o+89BT98EMX6vO/2Ke7tx3V4eY+XbAoV/1DMfUPxWSt1RXLC7VlSa58Xv57DAAAJA96ijHtrLW654nj+pffH9KpH6+Q36u4tRqOJZSXFtC1a4r1ZxsX6PyFOc4WCwAAXIWNdph13eFhWSulB33yez2KRON6/HCrfr2nSX862KrBaFzXVBXpc9ev1NKCdKfLBQAALkAoRlIJD8d0/zMn9R+PVSsSjet9Wxbqo1cvU25awOnSAADAPEYoRlJq7x/S1x85ogeer5Xf69EN60r03gsrtLEih5nHAABg2hGKkdSqW/v0vadPaOvuRvUPxbSiKEPvuKBcN60vUWFG0OnyAADAPEEoxpwwMBTTQy816oHna7WnvkceI11Sma9bzivTjetKFPR7nS4RAADMYYRizDnVrX3aurtRv9rdoLrOQZVlh/SFG6p03ZpiWisAAMCkEIoxZ1lr9VR1u/7ptwd1qLlPFy7O1RdvWq1VpZlOlwYAAOYYQjHmvFg8oZ+8UKd/++NhdQ9GddmyAr3rgnJdU1WkgI/DQAAAwOsjFGPe6AlHdd/TNfrvHXVq6okoNy2gG9eVaE1plpYWpmlpQbqyUxntBgAAXo1QjHknnrB68mibfvpCnR491KrhWOL0c5WF6frKrWu1eXGugxUCAIBkM2Oh2BjjlbRDUoO19kZjzGJJP5GUJ2mnpNustcOv9RmEYkxVPGFV3xXWsbZ+Vbf264fP1aquK6wPXLxIn37zSoUCTK0AAABnD8XT0Yj5MUkHxzz+qqSvW2srJXVJumMavgbwmrweo4V5abpqZZHuvHypfvexy/T+LQv1vadP6NpvPKHf7GlUc0/E6TIBAECSmtJKsTFmgaT7Jf2TpE9KuklSm6Ria23MGHORpP9trX3za30OK8WYKc8e69Cnf/6S6joHJUmFGSlatyBbN60v0U3rSuXxMNoNAAA3OdtKsW+Kn3u3pE9Lyhh9nCep21obG31cL6lsil8DmLSLluZp2yev0L6GXu2p79ae+h7tONmpbT9p0XeerNHnrl+pi5fmS5K6Bob1xNE2HWzq01s3lmlZUcbrfDoAAJgvJh2KjTE3Smq11u40xlw5ifffKelOSaqoqJhsGcDrSvF5df7CHJ2/MEeSlEhY/Wp3g772h8N6z73bdWllvsLDMe2u61Zi9H+c3Pvkcd22ZaE+cc1yZaX6HaweAADMhkm3Txhj/lnSbZJikoKSMiX9UtKbRfsE5oBINK77nzmh7zxVo9KsoK5cUagrVxSoPDdVd287oh9vr1VWyK9PvmmF3n1BuXxeZiEDADDXzehIttGV4r8ZnT7x35J+bq39iTHm25L2WGu/9VrvJxQjGR1s6tU//Hq/njveqZXFGfr7m1adbrUAAABz00xOnzjTZyR90hhTrZEe4/tm4GsAM66qJFMPfHiL/vO9G9UXiek9927XX/5wp4639SsZ5nsDAIDpw+EdwDmIROO694nj+tbjxzQYjasoM0UbK3K0oSJbV60sUmVhutMlAgCAc8CJdsA0aO6J6A/7m/VibZd21XartjMsj5HeunGBPvHG5SrNDkmS+odiemh3o363r0m3X7RI16wqcrhyAAAgEYqBGdHSG9G9T2+gTucAAB9TSURBVBzXD549KRnp9osWqn8orod2N2hgOK70FJ/CwzF95da1etdmpqwAAOC0mZpTDLhaUWZQf3vjKn3gkkX6+iNH9Z2napTi8+imdaV694UVWlGUob/60Yv67C/2qr1/SHe9oVLGcGAIAADJhpViYBo190SUmuJVZvDl2cbReEKfeXCPfrGrQe+9sEK3bihTcVZQhRlBBXyMeQMAYDaxUgzMguKs4Kuu+b0efe3t65WfkaJ7njiuH22vPf3ckoI0feXWtdqyJG82ywQAAGdgpRiYRcfb+lXbGVZzT0TNvRFt3d2okx0D+us3VOqjVy/jgBAAAGYYK8VAElhSkK4lBS+Pb/vwZUv0xYf269//VK2nj3XoI1dVqrq1X7vrurW3oUchv1eXLcvXFcsLtWlRjoJ+r4PVAwAwf7FSDCSBrbsb9IVf7lP/UEySVJYd0tqyLPVGotpxokvD8YSCfo+2LMnT5csKdPnyAi0tSGPTHgAAE8RKMZDEbjmvTJsX5+pwc59Wl2apICPl9HPh4ZieO96hJ46064kjbfrS4QOSRoLzuzeX67aLFikr5D/bRwMAgHPASjEwx9R1hvXE0Tb9fl+znjzarowUn95/8UJ98JLFyktPef0PAADAxTi8A5iH9jX06FuPV+t3+5rlMUY5qX5lhfzKSQ1oUX6aPnb1MpXnpjpdJgAASYNQDMxj1a39emh3g9oHhtUdHlbXQFQv1XcrnrD6qysr9RdXLGGTHgAAIhQDrtPUM6gv//agfrunSQvzUnXXlZVaWZKhRflpygz6Za1Vc29ER1v6dbJjQBdX5mvpmMkYAADMR4RiwKWeOtquv39on463DZy+lpcW0HAsob7RaReSlOLz6DPXrtQHLl4kj4epFgCA+YlQDLhYLJ7QsbYB1bQP6ETHgGraBhTwebS8KF2VhRkqyAjonx8+pEcPteqiJXn62jvWqyw7NO5nWWtlrQjOAIA5iVAM4DVZa/WzHXX60q8PyBijSyrztLo0S2vKMlWSFdJLdd167niHttd0qmcwqpvXl+rdmyu0bkEW85IBAHMGoRjAOanrDOvrjxzRrrpu1bQPvOK5/PSALlycp6Dfq4f3NmkwGldVSaY+fNli3bqhjHAMAEh6hGIAE9YXiepgU58auwe1pizrFafo9UWi2rq7UT/aXquDTb26pqpI//LWtcpnVjIAIIkRigHMiETC6rtP1+hf/3BYmUGf/vVt63TVyiKnywIAYFxnC8UeJ4oBMH94PEYfumyJfv3Xlyo/PUUf/P4OffSBXTrc3Od0aQAAnDNWigFMm6FYXN98tFrffbpG4eG4rqkq1F9euVR5aSmq7QyrtjOspp5BBX1eZaX6lRn0qzAjRRsX5nC4CABgVtA+AWDWdA0M6/5nT+j7z5xQdzj6iuc8Rkqc8ddOWsCrK1cU6k2ri3TVykJlBP2zVywAwFUIxQBm3cBQTL/d0yRjpIrcVFXkpaooI6hoIqHewZh6I1HVdoT1yMEWPXKgRW19Q8pO9evb7ztfW5bkOV0+AGAeIhQDSGqJhNXO2i597hd7dbJjQP/6tnW6dcMCp8sCAMwzbLQDkNQ8HqMLFuXq5//rYm1amKtP/PQl3b3tiJLhP9wBAPOfz+kCAGCsrFS/7v/gZn3uF3t197aj+tOhVpXnpio/LaC89BStKsnUxZV5Sg3w1xcAYPrwbxUASSfg8+hrb1+nqpIM/XF/iw429qq9f0i9kdjp5y9akqc3rCjQW89fwMY8AMCU0VMMYM6IROPacaJLjx1u1WOHWnW8fUCVhem67/ZNWpiX5nR5AIA5gI12AOadZ6rb9Vc/flFG0n/dtkmbF+eefu5kx4COtvTr0mX5zEAGAJxGKAYwL51oH9AH739BdZ1hff76KoWH43p4b5P2N/ZKknLTAnrfhRV630ULVZgRdLhaAIDTCMUA5q2ecFR3/fhFPVXdLknaWJGt69eWaElBmn68vU6PHmqR3+PRJZV5ygj6FfR7lOLzamVJht66cQEryQDgIoRiAPNaNJ7Q09XtWl6UodLs0Cueq2kf0PefrtH2mk4NxRKKROMKD8fVMxhVfnqK7rh0sd63pYINewDgAoRiABjDWqvtNZ36j8eq9eTRdmUEfbr9okX680sWKS89xenyAAAzhFAMAGext75H33q8Wr/f36wUn0fvuqBCd16+5FUrzgCAuY9QDACvo7q1X9/+n2P61a4GWUklWUFlp/qVkxpQeopPfZGYugeH1TUQVcJanb8wRxcvzdfFS/O0MC9VxhinvwUAwOsgFAPAOWroHtQD22vV0D2o7vCwugej6o/ElB70KSc1oOxUv2Jxq+01HWrpHZIkrS/P1g/v2ExfMgAkubOFYk60A4AzlGWH9DdvXvG6r7PWqqZ9QI8dbtM/P3xQH3lgl77z/k3yeT2zUCUAYDrxNzcATJIxRksK0nXHpYv1pVvW6PHDbfqnhw86XRYAYBJYKQaAafCeCytU3dqv7z5do8rCdL33woVq6Y3ox9tr9d876pSa4tOllfm6tDJfFy7Jpc0CAJIMoRgApskXbqjS8fZ+/f3W/XrsUKseP9ymuLW6fFmBrKSfvFCr7z9zQj6P0U3rS3XXGypVWZjudNkAALHRDgCmVV8kqrd/+1k1dg/qnReU631bFmphXpokKRKN68XaLv1xf4t++kKdIrG4blhbor++qlIrizMdrhwA3IHpEwAwS4ZicUlSiu/sx0e39w/pO0/W6P979oQGhuMqzQpqw8IcbSjPVlVJpgI+jzzGyOsx8nuN0gI+paZ4lRbwnX7OY8QYOACYIEIxACShroFh/Wp3g3ae7NKu2m41dA9O6P0eI5VkhbSyOEMrSzK0sjhTV64ooGcZAM6CUAwAc0BLb0THWvsVt1bxhFXCWg3HEgoPx0d/xTQcSyhhpXjCKpZIqLZzUIeaenW8fUDxhFVqwKu3bCjT+y5cqFWltGUAwFjMKQaAOaAoM6iizOCk3jsUi2tfQ49+8nydfr6zXj/eXqtNC3P0+RuqtLEi53Xf3z8UU4rPIz9zlgG4ECvFADAPdYeH9eDOen3nyRq19EX0rgsq9JlrVyg7NfCK18UTVk8cbdPPXqjTtoMtKsoM6pvv3qAN5xCiAWAuon0CAFyofyimux85ou89c0LZIb/uuGyxEgmrjoFhdQ4M6/maTjX1RJSbFtDN60v1yIEWtfRG9OlrV+hDly6Rx8NGPgDzC6EYAFzsQGOvvvCrvdpV2y1JSk/xKTctoGWF6Xrb+Qt0dVWRAj6PesJRfebne/T7/c26ckWBtizJU03bgI6396uxO6LVpZm6ckWhrlxRoNLskMPfFQBMHKEYAFzOWqu2/iFlBv0K+s8+Ls5aqx8+d1L/+JuDGo4nlJ8e0JL8dBVlBfXiya7TEzKqSjL1qTcu1zWril7x/s6BYX1j2xE1dEd0w7pivWlVsdJS2MICIDkQigEAE9IdHpYxRlmhl8e7WWtV3dqvxw+36Scv1OpY24CuWlmoL960SgtyUvXj52v1tT8cVv9QTIUZKWrqiSjo9+iNq4p1/ZpiXbIsX5mMiwPgIEIxAGBaDccS+v4zNfrGtqOKJqwqclNV3dqvLUty9aVb1qiyIF07a7u0dXeDfrunSV3hqHweo40Lc3TligJdsbxAq0oyX3EAyf7GHt3/zAk9erBVkWhc0YRVLJ5QWU5I33z3Rp1Xnu3gdwxgPiAUAwBmRHNPRP/8u4M61NSnu66q1E3rSl510l40ntCu2m49frhVjx9u04GmXklSYUaKrlheoLULsvSbl5r0/IlOhfxeXbumWLlpAfm8Rj6P0dbdjWrtHdKXb12jd2wqd+LbBDBPEIoBAEmjtTei/znSpsePtOnJI23qjcS0ICek2y9apHdsKldW6itbLLoGhvWRB3bpqep2vf+ihfr89VVq6xtSXVdY9V2DCng9KssJqSw7pKLMoFr7Ijrc3KcjLX3q6B/WBy5ZpJIsNgYCIBQDAJJULJ7QiY6wFuenyfsaI+Bi8YT+9Q+Hdc8Tx1/z84yRxv6rzWOk8txUPfDhLa+amNE5MKzmnggn/wEuQigGAMwLjxxo0Z76bi3ICWlBTqoW5IQUjSdU3zWo+q5BNfdEVJiZohVFGVpelKGajgHdft/zyk7z64EPb9GCnFRZa/WLFxv0j789oO5wVDevL9UXbqia9GmCAOYOQjEAwLVequvWbfdtV2bIr397+3r9x+PH9MSRNm2syNbmxXn67tM18nuMPnbNMl1TVaS9DT3aU9+jvQ09Go4llJ7iU3qKTxlBny5dlq83ry5+zbF2AJIXoRgA4Gp763v0vvu2q2cwqtSAV59+8wrddtEieT1GJzsG9KVfH9Cjh1pPvz7F59Gq0kylp/g0MBRT/1BM7f0jJwFmBn26+bxSvXNThdYuyBr361lrFR6OKzXgfdXGQwDOIRQDAFzvQGOvfrajTh+6bLEW5KS+6vknj7apoWtQaxdkaXlRhvxezyueTySsnjveoZ/tqNPv9jVrKJbQLeeV6os3rVZuWuAVX+dzv9ijl+p75PcaZYUCyk3zq7IwXW9YUag3rCxUfnrKpL8Pay1BG5gkQjEAANOoZzCq7z5Vo289Xq3MoF//cMtqXVNVpLu3HdW9Tx5XTqpft21ZpKFYXF3hkRXm3XXdaukdkjHSeeXZunZ1sW4+r/ScJ2McaenTr3Y1aOvuRqX4PLrn/eersjBjhr9TYH4hFAMAMAMONffq0w/u0Z76HmUGfeqNxPTOTeX63PUrlZ0aeMVrrbXa39irPx1q1baDLdpT3yNjpC2L83Tj+hL5PR4190bU3BtRR/+QEmP+FV3fNaiDTb3yeowuW5avfQ29iiUS+t4HLtCGipxZ/q6BuYtQDADADInFE7rvqRo9erBVn3jjcl20NO+c3lfTPqCtuxv0q10NOtERPn09Ny2g/PSAvJ6X2zcygj5dv6ZYN64vVX56ik52DOi2+55XW9+QvvW+jXrDikJVt/bpod2N2nawVatKM/Xxa5aN2yYCuBmhGACAJGWt1bG2fqX4vCrMTFGK79wmW7T1DekD33teh5v7tKQgTUda+k+3Zuxv7JWsdNtFC3XXGyqVGvCqurVfR1v7VN85qPSgT7lpAeWkBlSYmaJFeWlM1IArEIoBAJiH+iJRffYXe9XaG9ENa0t0/doSFWYG1dA9qG9sO6IHd9bL5/UoFk+8oh3jTB4jLcpL07KidK0vz9YNa0u0MC/tNb92JBrXk0fbdUllnlIDvmn+zoCZQSgGAMCFjrb06Ufba5UZ8o8eaJKu8txUhYfj6hwYVnd4WE09ER1t6dORln4daenT8fYBSdLasizduK5EV1cVaWlB2umJF7F4Qg/urNc3Hj2qpp6I1pRl6jvvv0DFWa99+EkkGld7/5ACXo98Xo/8XqP0FB+TNDCrCMUAAOCc1HeF9bu9zfrNnka9VN8jScpJ9ev8hblaVZqpX7/UqJr2AZ1Xnq0b15Xo648cUXrQp/tuv0Bryl6e29w/FNMLNZ3aXtOp52s6tLehR9H4K3PHssJ0/a8rlurm80pfNQIPmAmEYgAAMGF1nWE9e6xDL5zo1I6TXappH9CKogz9zZtX6JqqQhljdKi5V3d8f4c6B4b1tzdWqWtgWE8cbdeLJ7sUS1j5vUZry7K0eXGeFuenKpawisYSGowmtHV3gw4196ksO6Q7L1+iSyrzVJgZVMboCvJQLK4T7WEdbe1TdziqN68uVkHG5Gc8A4RiAAAwZb2RqNIDPnk8r2x5aO2L6M4f7NTuum5J0urSTF2+vECXVuZrY0WOQoHxN/FZa/XY4VZ967Fj2nGy6/T1kN+r7FS/WvuGFB/TDB3wenTDuhLdfvEirSvL0rG2fj1/olM7TnSpOzysnLSA8tICyk1L0eL8VK0py1JZduisLRp1nWH95/8c0976Hn38mmW6uqpoqrcISY5QDAAAZlQkGtfOk11aUZwxqRP79jX06Hj7gFp7I2rpjahjYFhl2SFVFqarsjBdXo/RA9tr9eDOeg0Mx5UW8GpgOC5Jyk9PUXFWiroGouoYGFIkmjj9uTmpfq0py9KKogxVFqZraWG6UgNefe/pE/rlrgZ5jVFxVlC1nWG9c1O5/vbGKmUE/ae/pxdOdCrF59UFi3Im3P88FIurqTuihXmp9E4nCUIxAACYF/oiUf18Z72OtvbrvPJsXbAo91Whc2AopqOt/dpb3629DT3a19CrY239Goq9HJZTfB6958IK/cXlS5WT5tc3th3Vt//nmEqzQ3rvhQv1wolOPXusQ4PRkeC9JD9N79pcrrduXKCskF8nOsI60tKnmvYBFWakqKokU5WF6Qp4Pdpxsku/3NWg3+5pVG8kpoKMFF2xvEBXrijQZcsKlBXyz/p9wwhCMQAAcLV4wqqxe1DVrf1q6Y3o6qqiV/Un7zzZqU/+7CWd7AirIjdVV64YCbLd4ageeL5WL5zokt9rZGQ0HE+86mt4jJQZ8qs7HFXI79W1a4q1sSJb22s69eTRdvUMRuX3Gl2xvEA3rS/VNVVFSg14daIjrN11XdpT36OA16MFOSEtyElVYWaK6jrD2t/Yq/2NvWroGtTmxbm6bk2xNi/OlY/NiRNGKAYAADgHQ7G4OvqHVZIVfFXLw9GWPj34Yr0kaXlhhlYUZ2hxfppaeiM63Nyng819auga1GXL8vXGVUVKS3l5fnMsntBL9d36w/4W/fqlRjX1RBT0exT0e9Udjkoa6aWOW6vh2CsDt9djVFmQrsLMFL1wolORaEK5aQG9aVWRbt1QpgsW5b6iz7tzYFiPHGhWVziqvLSA8jNSlJcWUH8kpsaeiBq7B9XRP6TzF+XqmqpCV82ZJhQDAAAkiUTC6sXaLv1mT5Mi0bjOK8/W+vJsLS/KkJHUPjCk+q5BNfdEVJod0srijNMnDoaHY3riSJt+t69Z2w60aGA4rgU5If3ZhjIVZQX1u73NevZ4xys2KI4n5PdqMBpXasCrN60q0g3rSrWqNFMlmcFXbaScTwjFAAAA80x4OKY/7m/Rz1+s19PV7UpYaVFeqq5fW6Ib1o2cStjRP6T2/mF19A8pPehTaVZIxVlBBbwePX+iU1t3N+rhvU3qGRxZrQ76PVqUl6bS7JAkKWGtElbKTw/oqpWFunx5gTKDL/dEDwzFdKytX0G/VyVZwdObFF9LXyR6Tq+bCYRiAACAeaylN6KewaiWFaZPeNLFcCyhF2u7dKytXzVtAzrePqCW3og8xshjJGOMTnYMqCs80hN94eI8pQa8OtzSp5Md4Vd8VkaKT2U5IV2wKFdXrijQRUtHjgE/dSjMw/uadLIjrO2fv9qRA1sIxQAAAJi0+GjLx7aDLfrTwVbFrVVVcaZWFI8cHz4ct2rqHlRj96BOdIT1fE2nBqNxBXweVeSmqrq1X5K0pixT160p0Z9fssiRXmZCMQAAAGbNqRnPjx9u05GWPl1Sma/r1hRrYV6ao3WdLRS7Z6shAAAAZk3Q79Vly0bmMs8FDLcDAACA6xGKAQAA4HqEYgAAALgeoRgAAACuRygGAACA6xGKAQAA4HqEYgAAALjepEOxMabcGPOYMeaAMWa/MeZjo9dzjTGPGGOOjv6eM33lAgAAANNvKivFMUmfstaukrRF0l3GmFWSPivpUWvtMkmPjj4GAAAAktakQ7G1tsla++Lon/skHZRUJukWSfePvux+SW+ZapEAAADATJqWnmJjzCJJGyRtl1RkrW0afapZUtF0fA0AAABgpkw5FBtj0iX9XNLHrbW9Y5+z1lpJ9izvu9MYs8MYs6OtrW2qZQAAAACTNqVQbIzxayQQ/8ha+4vRyy3GmJLR50sktY73XmvtPdbaTdbaTQUFBVMpAwAAAJiSqUyfMJLuk3TQWvt/xzz1kKTbR/98u6Stky8PAAAAmHm+Kbz3Ekm3SdprjNk9eu3zkv5F0s+MMXdIOinpHVMrEQAAAJhZkw7F1tqnJJmzPH31ZD8XAAAAmG2caAcAAADXIxQDAADA9czI1DSHizCmTSP9x07Il9Tu0Neei7hfE8c9mxju18RxzyaG+zVx3LOJ4X5N3Gzes4XW2leNPkuKUOwkY8wOa+0mp+uYK7hfE8c9mxju18RxzyaG+zVx3LOJ4X5NXDLcM9onAAAA4HqEYgAAALgeoVi6x+kC5hju18RxzyaG+zVx3LOJ4X5NHPdsYrhfE+f4PXN9TzEAAADASjEAAABcz7Wh2BhzrTHmsDGm2hjzWafrSUbGmHJjzGPGmAPGmP3GmI+NXs81xjxijDk6+nuO07UmE2OM1xizyxjzm9HHi40x20d/1n5qjAk4XWMyMcZkG2MeNMYcMsYcNMZcxM/Y2RljPjH6z+M+Y8wDxpggP2OvZIz5rjGm1Rizb8y1cX+mzIh/H713e4wxG52r3DlnuWf/Z/Sfyz3GmF8aY7LHPPe50Xt22BjzZmeqds5492vMc58yxlhjTP7oY37GdPZ7Zoz5yOjP2X5jzL+OuT7rP2OuDMXGGK+k/5B0naRVkt5tjFnlbFVJKSbpU9baVZK2SLpr9D59VtKj1tplkh4dfYyXfUzSwTGPvyrp69baSkldku5wpKrk9Q1Jv7fWrpS0Xv9/e/caYsdZx3H8+yObhKSF3oKxZitbNfVFai+hSvGGjSJtLV3BQiMBqxaEvPDypmoNCIIvRERLvVS0pYkaLFpjDYLSmpYqaBttyKX1mrYh3bAxCZJ4JY3154vniZ2e3bObgMnMZn4fGHbmmTmH5/z3f878z8wzc0rskmPTkLQM+Ahwle1LgXnAapJjg9YD1w60Dcup64DldfoQcNdp6mPXrGdqzB4CLrV9GfBH4HaAuh9YDayoj/la3a/2yXqmxgtJFwHvBPY2mpNjxXoGYibpGmAcuNz2CuALtb2VHOtlUQy8Adht+xnbzwP3Uf4p0WB70va2Ov83SrGyjBKrDXWzDcC72+lh90gaBd4F3F2XBawC7q+bJF4Nks4B3grcA2D7eduHSY7NZARYJGkEWAxMkhx7Cds/B/4y0Dwsp8aBb7l4DDhX0oWnp6fdMV3MbD9o+9918TFgtM6PA/fZPmr7WWA3Zb/aG0NyDOBLwMeB5gVbyTGGxmwt8DnbR+s2B2p7KznW16J4GfBcY3mitsUQksaAK4HHgaW2J+uq/cDSlrrVRXdQPhD/U5cvAA43dizJtZe6GDgI3FuHnNwt6SySY9OyvY9yJGUvpRg+AjxBcuxEDMup7A9OzAeBn9T5xGwaksaBfbZ3DKxKvIa7BHhLHf71qKTX1/ZWYtbXojhOgqSzgR8AH7P91+Y6l9uX5BYmgKQbgAO2n2i7L3PICLASuMv2lcA/GBgqkRx7UR0HO075MvEK4CymOYUbM0tOnRxJ6yjD6Ta23ZeukrQY+BTw6bb7MseMAOdThmjeBnyvnmFtRV+L4n3ARY3l0doWAyTNpxTEG21vqs1/Pn7qp/49MOzxPfMm4EZJeyhDclZRxsueW091Q3Jt0AQwYfvxunw/pUhOjk3vHcCztg/aPgZsouRdcmx2w3Iq+4MZSHo/cAOwxi/ewzUxm+rVlC+rO+o+YBTYJunlJF4zmQA21aElWylnWZfQUsz6WhT/Glher9heQBnMvbnlPnVO/bZ2D/A7219srNoM3FLnbwF+dLr71kW2b7c9anuMklMP214DPALcVDdLvBps7week/Ta2vR24Lckx4bZC1wtaXF9fx6PV3JsdsNyajPwvnqHgKuBI41hFr0m6VrKcLAbbf+zsWozsFrSQkkXUy4g29pGH7vC9i7bL7M9VvcBE8DK+hmXHBvuAeAaAEmXAAuAQ7SVY7Z7OQHXU66mfRpY13Z/ujgBb6acYtwJbK/T9ZRxsluAPwE/A85vu69dm4C3AT+u86+qb+bdwPeBhW33r0sTcAXwm5pnDwDnJcdmjNdngN8DTwLfBhYmx6bE6LuUMdfHKMXJrcNyChDlbkRPA7sod/Zo/TV0JGa7KeM6j3/+f72x/boasz8A17Xd/y7Ea2D9HmBJcmzWHFsAfKd+nm0DVrWZY/lFu4iIiIjovb4On4iIiIiI+J8UxRERERHReymKIyIiIqL3UhRHRERERO+lKI6IiIiI3ktRHBHRMkkvSNremD45+6NO+LnHJD35/3q+iIgz1cjsm0RExCn2L9tXtN2JiIg+y5HiiIiOkrRH0ucl7ZK0VdJravuYpIcl7ZS0RdIra/tSST+UtKNOb6xPNU/SNyU9JelBSYtae1ERER2Vojgion2LBoZP3NxYd8T264CvAHfUti8DG2xfBmwE7qztdwKP2r4cWAk8VduXA1+1vQI4DLznFL+eiIg5J79oFxHRMkl/t332NO17KD97+oyk+cB+2xdIOgRcaPtYbZ+0vUTSQWDU9tHGc4wBD9leXpc/Acy3/dlT/8oiIuaOHCmOiOg2D5k/GUcb8y+Q60kiIqZIURwR0W03N/7+qs7/Elhd59cAv6jzW4C1AJLmSTrndHUyImKuy9GCiIj2LZK0vbH8U9vHb8t2nqSdlKO9761tHwbulXQbcBD4QG3/KPANSbdSjgivBSZPee8jIs4AGVMcEdFRdUzxVbYPtd2XiIgzXYZPRERERETv5UhxRERERPRejhRHRERERO+lKI6IiIiI3ktRHBERERG9l6I4IiIiInovRXFERERE9F6K4oiIiIjovf8CyBMWCmhaCugAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"wBD0wM3JVXol"},"source":["### Appendix"]},{"cell_type":"markdown","metadata":{"id":"Iz0QI1P7VaDP"},"source":["#### References\n","1. [https://medium.com/@yusufnoor_88274/implementing-neural-graph-collaborative-filtering-in-pytorch-4d021dff25f3](https://medium.com/@yusufnoor_88274/implementing-neural-graph-collaborative-filtering-in-pytorch-4d021dff25f3)\n","2. [https://github.com/xiangwang1223/neural_graph_collaborative_filtering](https://github.com/xiangwang1223/neural_graph_collaborative_filtering)\n","3. [https://arxiv.org/pdf/1905.08108.pdf](https://arxiv.org/pdf/1905.08108.pdf)\n","4. [https://github.com/metahexane/ngcf_pytorch_g61](https://github.com/metahexane/ngcf_pytorch_g61)"]},{"cell_type":"markdown","metadata":{"id":"_JywmGguVbNU"},"source":["#### Next\n","\n","Try out this notebook on the following datasets:\n","\n",""]},{"cell_type":"markdown","metadata":{"id":"MppKYNXJVswT"},"source":["Compare out the performance with these baselines:\n","\n","1. MF: This is matrix factorization optimized by the Bayesian\n","personalized ranking (BPR) loss, which exploits the user-item\n","direct interactions only as the target value of interaction function.\n","2. NeuMF: The method is a state-of-the-art neural CF model\n","which uses multiple hidden layers above the element-wise and\n","concatenation of user and item embeddings to capture their nonlinear feature interactions. Especially, we employ two-layered\n","plain architecture, where the dimension of each hidden layer\n","keeps the same.\n","3. CMN: It is a state-of-the-art memory-based model, where\n","the user representation attentively combines the memory slots\n","of neighboring users via the memory layers. Note that the firstorder connections are used to find similar users who interacted\n","with the same items.\n","4. HOP-Rec: This is a state-of-the-art graph-based model,\n","where the high-order neighbors derived from random walks\n","are exploited to enrich the user-item interaction data.\n","5. PinSage: PinSage is designed to employ GraphSAGE\n","on item-item graph. In this work, we apply it on user-item interaction graph. Especially, we employ two graph convolution\n","layers, and the hidden dimension is set equal\n","to the embedding size.\n","6. GC-MC: This model adopts GCN encoder to generate\n","the representations for users and items, where only the first-order\n","neighbors are considered. Hence one graph convolution layer,\n","where the hidden dimension is set as the embedding size, is used."]},{"cell_type":"markdown","metadata":{"id":"e7RRc2UQBuc9"},"source":["## A simple recommender with tensorflow\n","> A tutorial on how to build a simple deep learning based movie recommender using tensorflow library."]},{"cell_type":"code","metadata":{"id":"hLtJPt_5idKN"},"source":["import numpy as np\n","import pandas as pd\n","import tensorflow as tf\n","from tensorflow import keras\n","from tensorflow.keras import layers\n","from tensorflow.keras import models\n","\n","tf.random.set_seed(343)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DNLlAwKUihC1"},"source":["# Clean up the logdir if it exists\n","import shutil\n","shutil.rmtree('logs', ignore_errors=True)\n","\n","# Load TensorBoard extension for notebooks\n","%load_ext tensorboard"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"8IRTF0EVjQuX","outputId":"932eaa43-725c-4fb8-e9d4-dca92ced4cf0"},"source":["movielens_ratings_file = 'https://github.com/sparsh-ai/reco-data/blob/master/MovieLens_100K_ratings.csv?raw=true'\n","df_raw = pd.read_csv(movielens_ratings_file)\n","df_raw.head()"],"execution_count":null,"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
UserIdMovieIdRatingTimestamp
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n","
"],"text/plain":[" UserId MovieId Rating Timestamp\n","0 196 242 3.0 881250949\n","1 186 302 3.0 891717742\n","2 22 377 1.0 878887116\n","3 244 51 2.0 880606923\n","4 166 346 1.0 886397596"]},"execution_count":22,"metadata":{"tags":[]},"output_type":"execute_result"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"El1C8OwWjhxk","outputId":"f8ed06e7-8554-45f2-982d-987f153a5cc7"},"source":["df = df_raw.copy()\n","df.columns = ['userId', 'movieId', 'rating', 'timestamp']\n","user_ids = df['userId'].unique()\n","user_encoding = {x: i for i, x in enumerate(user_ids)} # {user_id: index}\n","movie_ids = df['movieId'].unique()\n","movie_encoding = {x: i for i, x in enumerate(movie_ids)} # {movie_id: index}\n","\n","df['user'] = df['userId'].map(user_encoding) # Map from IDs to indices\n","df['movie'] = df['movieId'].map(movie_encoding)\n","\n","n_users = len(user_ids)\n","n_movies = len(movie_ids)\n","\n","min_rating = min(df['rating'])\n","max_rating = max(df['rating'])\n","\n","print(f'Number of users: {n_users}\\nNumber of movies: {n_movies}\\nMin rating: {min_rating}\\nMax rating: {max_rating}')\n","\n","# Shuffle the data\n","df = df.sample(frac=1, random_state=42)"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["Number of users: 943\n","Number of movies: 1682\n","Min rating: 1.0\n","Max rating: 5.0\n"]}]},{"cell_type":"markdown","metadata":{"id":"1W5V8T-C8Gpv"},"source":["### Scheme of the model\n","\n",""]},{"cell_type":"code","metadata":{"id":"G-iv9rijkaBf"},"source":["class MatrixFactorization(models.Model):\n"," def __init__(self, n_users, n_movies, n_factors, **kwargs):\n"," super(MatrixFactorization, self).__init__(**kwargs)\n"," self.n_users = n_users\n"," self.n_movies = n_movies\n"," self.n_factors = n_factors\n"," \n"," # We specify the size of the matrix,\n"," # the initializer (truncated normal distribution)\n"," # and the regularization type and strength (L2 with lambda = 1e-6)\n"," self.user_emb = layers.Embedding(n_users, \n"," n_factors, \n"," embeddings_initializer='he_normal',\n"," embeddings_regularizer=keras.regularizers.l2(1e-6),\n"," name='user_embedding')\n"," self.movie_emb = layers.Embedding(n_movies, \n"," n_factors, \n"," embeddings_initializer='he_normal',\n"," embeddings_regularizer=keras.regularizers.l2(1e-6),\n"," name='movie_embedding')\n"," \n"," # Embedding returns a 3D tensor with one dimension = 1, so we reshape it to a 2D tensor\n"," self.reshape = layers.Reshape((self.n_factors,))\n"," \n"," # Dot product of the latent vectors\n"," self.dot = layers.Dot(axes=1)\n","\n"," def call(self, inputs):\n"," # Two inputs\n"," user, movie = inputs\n"," u = self.user_emb(user)\n"," u = self.reshape(u)\n"," \n"," m = self.movie_emb(movie)\n"," m = self.reshape(m)\n"," \n"," return self.dot([u, m])\n","\n","n_factors = 50\n","model = MatrixFactorization(n_users, n_movies, n_factors)\n","model.compile(\n"," optimizer=keras.optimizers.Adam(learning_rate=0.001),\n"," loss=keras.losses.MeanSquaredError()\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Bac1w7u49Ddx","outputId":"bb733033-9aba-446b-a56d-f971897221d0"},"source":["try:\n"," model.summary()\n","except ValueError as e:\n"," print(e, type(e))"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["This model has not yet been built. Build the model first by calling `build()` or calling `fit()` with some data, or specify an `input_shape` argument in the first layer(s) for automatic build. \n"]}]},{"cell_type":"markdown","metadata":{"id":"o-JSFnJA-1dz"},"source":["This is why building models via subclassing is a bit annoying - you can run into errors such as this. We'll fix it by calling the model with some fake data so it knows the shapes of the inputs."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7wkIhqmO92Ca","outputId":"6825be87-3d5e-4d25-e276-5043ff3a3bb9"},"source":["_ = model([np.array([1, 2, 3]), np.array([2, 88, 5])])\n","model.summary()"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["Model: \"matrix_factorization_1\"\n","_________________________________________________________________\n","Layer (type) Output Shape Param # \n","=================================================================\n","user_embedding (Embedding) multiple 47150 \n","_________________________________________________________________\n","movie_embedding (Embedding) multiple 84100 \n","_________________________________________________________________\n","reshape_1 (Reshape) multiple 0 \n","_________________________________________________________________\n","dot_1 (Dot) multiple 0 \n","=================================================================\n","Total params: 131,250\n","Trainable params: 131,250\n","Non-trainable params: 0\n","_________________________________________________________________\n"]}]},{"cell_type":"markdown","metadata":{"id":"9Nxdrz7b_HOq"},"source":["We're going to expand our toolbox by introducing callbacks. Callbacks can be used to monitor our training progress, decay the learning rate, periodically save the weights or even stop early in case of detected overfitting. In Keras, they are really easy to use: you just create a list of desired callbacks and pass it to the model.fit method. It's also really easy to define your own by subclassing the Callback class. You can also specify when they will be triggered - the default is at the end of every epoch.\n","\n","We'll use two: an early stopping callback which will monitor our loss and stop the training early if needed and TensorBoard, a utility for visualizing models, monitoring the training progress and much more."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6N_Y7u5o-QpY","outputId":"b4f299bc-25e2-4e38-b349-dc07184fb488"},"source":["callbacks = [\n"," keras.callbacks.EarlyStopping(\n"," # Stop training when `val_loss` is no longer improving\n"," monitor='val_loss',\n"," # \"no longer improving\" being defined as \"no better than 1e-2 less\"\n"," min_delta=1e-2,\n"," # \"no longer improving\" being further defined as \"for at least 2 epochs\"\n"," patience=2,\n"," verbose=1,\n"," ),\n"," keras.callbacks.TensorBoard(log_dir='logs')\n","]\n","\n","history = model.fit(\n"," x=(df['user'].values, df['movie'].values), # The model has two inputs!\n"," y=df['rating'],\n"," batch_size=128,\n"," epochs=20,\n"," verbose=1,\n"," validation_split=0.1,\n"," callbacks=callbacks\n",")"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["Epoch 1/20\n","704/704 [==============================] - 3s 3ms/step - loss: 12.0905 - val_loss: 5.5121\n","Epoch 2/20\n","704/704 [==============================] - 2s 3ms/step - loss: 2.1751 - val_loss: 1.2149\n","Epoch 3/20\n","704/704 [==============================] - 2s 3ms/step - loss: 1.0271 - val_loss: 0.9839\n","Epoch 4/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.9003 - val_loss: 0.9266\n","Epoch 5/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.8470 - val_loss: 0.8996\n","Epoch 6/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.8046 - val_loss: 0.8786\n","Epoch 7/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.7667 - val_loss: 0.8680\n","Epoch 8/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.7329 - val_loss: 0.8618\n","Epoch 9/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.6999 - val_loss: 0.8558\n","Epoch 10/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.6688 - val_loss: 0.8558\n","Epoch 11/20\n","704/704 [==============================] - 2s 3ms/step - loss: 0.6381 - val_loss: 0.8560\n","Epoch 00011: early stopping\n"]}]},{"cell_type":"markdown","metadata":{"id":"5QUGLmtw_eWA"},"source":["We see that we stopped early because the validation loss was not improving. Now, we'll open TensorBoard (it's a separate program called via command-line) to read the written logs and visualize the loss over all epochs. We will also look at how to visualize the model as a computational graph."]},{"cell_type":"code","metadata":{"id":"_J-v9Hua_SV8"},"source":["# Run TensorBoard and specify the log dir\n","%tensorboard --logdir logs"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Ldq0DwgI_lWC"},"source":["We've seen how easy it is to implement a recommender system with Keras and use a few utilities to make it easier to experiment. Note that this model is still quite basic and we could easily improve it: we could try adding a bias for each user and movie or adding non-linearity by using a sigmoid function and then rescaling the output. It could also be extended to use other features of a user or movie."]},{"cell_type":"markdown","metadata":{"id":"-dpCn5hm_nUM"},"source":["Next, we'll try a bigger, more state-of-the-art model: a deep autoencoder."]},{"cell_type":"markdown","metadata":{"id":"zTGdZ0b4_4rl"},"source":["We'll apply a more advanced algorithm to the same dataset as before, taking a different approach. We'll use a deep autoencoder network, which attempts to reconstruct its input and with that gives us ratings for unseen user / movie pairs."]},{"cell_type":"markdown","metadata":{"id":"yIf926SkCOEp"},"source":[""]},{"cell_type":"markdown","metadata":{"id":"rSdY5NKQAYfI"},"source":["Preprocessing will be a bit different due to the difference in our model. Our autoencoder will take a vector of all ratings for a movie and attempt to reconstruct it. However, our input vector will have a lot of zeroes due to the sparsity of our data. We'll modify our loss so our model won't predict zeroes for those combinations - it will actually predict unseen ratings.\n","\n","To facilitate this, we'll use the sparse tensor that TF supports. Note: to make training easier, we'll transform it to dense form, which would not work in larger datasets - we would have to preprocess the data in a different way or stream it into the model."]},{"cell_type":"markdown","metadata":{"id":"HBcKm55rCVkk"},"source":["### Sparse representation and autoencoder reconstruction\n","\n",""]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"OWybs9LyE8bB","outputId":"1e108c11-a007-4942-bf0d-50409e4bbb1d"},"source":["df_raw.head()"],"execution_count":null,"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdratingtimestamp
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n","
"],"text/plain":[" userId movieId rating timestamp\n","0 196 242 3.0 881250949\n","1 186 302 3.0 891717742\n","2 22 377 1.0 878887116\n","3 244 51 2.0 880606923\n","4 166 346 1.0 886397596"]},"execution_count":21,"metadata":{"tags":[]},"output_type":"execute_result"}]},{"cell_type":"code","metadata":{"id":"M9jASOsh_gvU"},"source":["# Create a sparse tensor: at each user, movie location, we have a value, the rest is 0\n","sparse_x = tf.sparse.SparseTensor(indices=df[['movie', 'user']].values, values=df['rating'], dense_shape=(n_movies, n_users))\n","\n","# Transform it to dense form and to float32 (good enough precision)\n","dense_x = tf.cast(tf.sparse.to_dense(tf.sparse.reorder(sparse_x)), tf.float32)\n","\n","# Shuffle the data\n","x = tf.random.shuffle(dense_x, seed=42)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"1j2-lFANEp8t"},"source":["Now, let's create the model. We'll have to specify the input shape. Because we have 9724 movies and only 610 users, we'll prefer to predict ratings for movies instead of users - this way, our dataset is larger."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9s4qXdbuEpuX","outputId":"45ac7925-b8f3-44dd-8e35-53fa7192b538"},"source":["class Encoder(layers.Layer):\n"," def __init__(self, **kwargs):\n"," super(Encoder, self).__init__(**kwargs)\n"," self.dense1 = layers.Dense(28, activation='selu', kernel_initializer='glorot_uniform')\n"," self.dense2 = layers.Dense(56, activation='selu', kernel_initializer='glorot_uniform')\n"," self.dense3 = layers.Dense(56, activation='selu', kernel_initializer='glorot_uniform')\n"," self.dropout = layers.Dropout(0.3)\n"," \n"," def call(self, x):\n"," d1 = self.dense1(x)\n"," d2 = self.dense2(d1)\n"," d3 = self.dense3(d2)\n"," return self.dropout(d3)\n"," \n"," \n","class Decoder(layers.Layer):\n"," def __init__(self, n, **kwargs):\n"," super(Decoder, self).__init__(**kwargs)\n"," self.dense1 = layers.Dense(56, activation='selu', kernel_initializer='glorot_uniform')\n"," self.dense2 = layers.Dense(28, activation='selu', kernel_initializer='glorot_uniform')\n"," self.dense3 = layers.Dense(n, activation='selu', kernel_initializer='glorot_uniform')\n","\n"," def call(self, x):\n"," d1 = self.dense1(x)\n"," d2 = self.dense2(d1)\n"," return self.dense3(d2)\n","\n","n = n_users\n","inputs = layers.Input(shape=(n,))\n","\n","encoder = Encoder()\n","decoder = Decoder(n)\n","\n","enc1 = encoder(inputs)\n","dec1 = decoder(enc1)\n","enc2 = encoder(dec1)\n","dec2 = decoder(enc2)\n","\n","model = models.Model(inputs=inputs, outputs=dec2, name='DeepAutoencoder')\n","model.summary()"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["Model: \"DeepAutoencoder\"\n","__________________________________________________________________________________________________\n","Layer (type) Output Shape Param # Connected to \n","==================================================================================================\n","input_1 (InputLayer) [(None, 943)] 0 \n","__________________________________________________________________________________________________\n","encoder (Encoder) (None, 56) 31248 input_1[0][0] \n"," decoder[0][0] \n","__________________________________________________________________________________________________\n","decoder (Decoder) (None, 943) 32135 encoder[0][0] \n"," encoder[1][0] \n","==================================================================================================\n","Total params: 63,383\n","Trainable params: 63,383\n","Non-trainable params: 0\n","__________________________________________________________________________________________________\n"]}]},{"cell_type":"markdown","metadata":{"id":"aqXWA_TQGMDa"},"source":["Because our inputs are sparse, we'll need to create a modified mean squared error function. We have to look at which ratings are zero in the ground truth and remove them from our loss calculation (if we didn't, our model would quickly learn to predict zeros almost everywhere). We'll use masking - first get a boolean mask of non-zero values and then extract them from the result."]},{"cell_type":"code","metadata":{"id":"G7AyGH8IFXAj"},"source":["def masked_mse(y_true, y_pred):\n"," mask = tf.not_equal(y_true, 0)\n"," se = tf.boolean_mask(tf.square(y_true - y_pred), mask)\n"," return tf.reduce_mean(se)\n","\n","model.compile(\n"," loss=masked_mse,\n"," optimizer=keras.optimizers.Adam()\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"6O-Hqm_FGTmz"},"source":["The model training will be similar as before - we'll use early stopping and TensorBoard. Our batch size will be smaller due to the lower number of examples. Note that we are passing the same array for both x and y, because the autoencoder reconstructs its input."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"OHoZ3IuJGSrL","outputId":"7923e0b0-7bc6-42ba-b3e6-c2bfe025291d"},"source":["callbacks = [\n"," keras.callbacks.EarlyStopping(\n"," monitor='val_loss',\n"," min_delta=1e-2,\n"," patience=5,\n"," verbose=1,\n"," ),\n"," keras.callbacks.TensorBoard(log_dir='logs')\n","]\n","\n","model.fit(\n"," x, \n"," x, \n"," batch_size=16, \n"," epochs=100, \n"," validation_split=0.1,\n"," callbacks=callbacks\n",")"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["WARNING:tensorflow:Model failed to serialize as JSON. Ignoring... Layer Decoder has arguments in `__init__` and therefore must override `get_config`.\n","Epoch 1/100\n","95/95 [==============================] - 2s 7ms/step - loss: 4.6136 - val_loss: 1.1074\n","Epoch 2/100\n","95/95 [==============================] - 0s 4ms/step - loss: 1.1491 - val_loss: 1.0088\n","Epoch 3/100\n","95/95 [==============================] - 0s 5ms/step - loss: 1.0577 - val_loss: 0.9768\n","Epoch 4/100\n","95/95 [==============================] - 0s 4ms/step - loss: 1.0257 - val_loss: 0.9758\n","Epoch 5/100\n","95/95 [==============================] - 0s 4ms/step - loss: 0.9971 - val_loss: 0.9774\n","Epoch 6/100\n","95/95 [==============================] - 0s 4ms/step - loss: 0.9812 - val_loss: 0.9604\n","Epoch 7/100\n","95/95 [==============================] - 0s 5ms/step - loss: 0.9598 - val_loss: 0.9275\n","Epoch 8/100\n","95/95 [==============================] - 0s 5ms/step - loss: 0.9501 - val_loss: 0.9253\n","Epoch 9/100\n","95/95 [==============================] - 0s 5ms/step - loss: 0.9177 - val_loss: 0.9159\n","Epoch 10/100\n","95/95 [==============================] - 0s 5ms/step - loss: 0.9193 - val_loss: 0.9189\n","Epoch 11/100\n","95/95 [==============================] - 0s 4ms/step - loss: 0.9016 - val_loss: 0.9040\n","Epoch 12/100\n","95/95 [==============================] - 0s 4ms/step - loss: 0.9119 - val_loss: 0.9108\n","Epoch 13/100\n","95/95 [==============================] - 0s 5ms/step - loss: 0.8917 - val_loss: 0.9192\n","Epoch 14/100\n","95/95 [==============================] - 0s 5ms/step - loss: 0.8855 - val_loss: 0.9166\n","Epoch 15/100\n","95/95 [==============================] - 0s 4ms/step - loss: 0.8843 - val_loss: 0.9067\n","Epoch 16/100\n","95/95 [==============================] - 0s 4ms/step - loss: 0.8851 - val_loss: 0.9034\n","Epoch 00016: early stopping\n"]},{"data":{"text/plain":[""]},"execution_count":27,"metadata":{"tags":[]},"output_type":"execute_result"}]},{"cell_type":"markdown","metadata":{"id":"kkkhIjHhGhP5"},"source":["Let's visualize our loss and the model itself with TensorBoard."]},{"cell_type":"code","metadata":{"id":"MMVp_HbwGdGQ"},"source":["%tensorboard --logdir logs"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"eSBFppW8Gkih"},"source":["That's it! We've seen how to use TensorFlow to implement recommender systems in a few different ways. I hope this short introduction has been informative and has prepared you to use TF on new problems. Thank you for your attention!"]}]} \ No newline at end of file +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "2022-01-07-ncf.ipynb", + "provenance": [], + "collapsed_sections": [], + "authorship_tag": "ABX9TyO+EzcZLxWCVB+iE5tVY1BN" + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "iVkUysixCyk6" + }, + "source": [ + "# Neural Collaborative Filtering Recommenders" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-xzeGn4mvtvd" + }, + "source": [ + "!pip install -q pytorch-lightning" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "xNgD2QXOkq4g" + }, + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "import torch.optim as optim\n", + "from torch.utils.data import Dataset, DataLoader\n", + "from torch.utils.data import TensorDataset\n", + "from tqdm.notebook import tqdm\n", + "import pytorch_lightning as pl\n", + "\n", + "np.random.seed(123)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8fqqbAgmrxoT" + }, + "source": [ + "## NCF with PyTorch Lightning on ML-25m\n", + "\n", + "In this section, we will build a simple yet accurate model using movielens-25m dataset and pytorch lightning library. This will be a retrieval model where the objective is to maximize recall over precision." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MrlkpJITv8Rw", + "outputId": "4fd97487-1f04-4563-8adf-5aa306c13d2b" + }, + "source": [ + "!wget -q --show-progress https://files.grouplens.org/datasets/movielens/ml-25m.zip\n", + "!unzip ml-25m.zip" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ml-25m.zip.1 100%[===================>] 249.84M 45.7MB/s in 5.9s \n", + "Archive: ml-25m.zip\n", + "replace ml-25m/tags.csv? [y]es, [n]o, [A]ll, [N]one, [r]ename: N\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "5-juuyKOwCmL", + "outputId": "93ff3c63-a959-4e2d-fdff-c5e72a475160" + }, + "source": [ + "ratings = pd.read_csv('ml-25m/ratings.csv', infer_datetime_format=True)\n", + "ratings.head()" + ], + "execution_count": null, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdratingtimestamp
012965.01147880044
113063.51147868817
213075.01147868828
316655.01147878820
418993.51147868510
\n", + "
" + ], + "text/plain": [ + " userId movieId rating timestamp\n", + "0 1 296 5.0 1147880044\n", + "1 1 306 3.5 1147868817\n", + "2 1 307 5.0 1147868828\n", + "3 1 665 5.0 1147878820\n", + "4 1 899 3.5 1147868510" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A2bKYi9WwGyP" + }, + "source": [ + "### Subset\n", + "\n", + "In order to keep memory usage manageable, we will only use data from 20% of the users in this dataset. Let's randomly select 30% of the users and only use data from the selected users." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HsYAMEXqwZoH", + "outputId": "434b4209-8c53-4474-edd8-6c5d109c3184" + }, + "source": [ + "rand_userIds = np.random.choice(ratings['userId'].unique(), \n", + " size=int(len(ratings['userId'].unique())*0.2), \n", + " replace=False)\n", + "\n", + "ratings = ratings.loc[ratings['userId'].isin(rand_userIds)]\n", + "\n", + "print('There are {} rows of data from {} users'.format(len(ratings), len(rand_userIds)))" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "There are 5015129 rows of data from 32508 users\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "w7OS6UqDpXdi" + }, + "source": [ + "### Train/Test Split\n", + "**Chronological Leave-One-Out Split**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZaqAMrH-gn_i" + }, + "source": [ + "Along with the rating, there is also a timestamp column that shows the date and time the review was submitted. Using the timestamp column, we will implement our train-test split strategy using the leave-one-out methodology. For each user, the most recent review is used as the test set (i.e. leave one out), while the rest will be used as training data ." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A4COa9yVguUO" + }, + "source": [ + "> Note: Doing a random split would not be fair, as we could potentially be using a user's recent reviews for training and earlier reviews for testing. This introduces data leakage with a look-ahead bias, and the performance of the trained model would not be generalizable to real-world performance." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WtdtS0FMgTez" + }, + "source": [ + "ratings['rank_latest'] = ratings.groupby(['userId'])['timestamp'] \\\n", + " .rank(method='first', ascending=False)\n", + "\n", + "train_ratings = ratings[ratings['rank_latest'] != 1]\n", + "test_ratings = ratings[ratings['rank_latest'] == 1]\n", + "\n", + "# drop columns that we no longer need\n", + "train_ratings = train_ratings[['userId', 'movieId', 'rating']]\n", + "test_ratings = test_ratings[['userId', 'movieId', 'rating']]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XFoYAbhqpnPD" + }, + "source": [ + "### Implicit Conversion" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HwYvXqJ6hz2u" + }, + "source": [ + "We will train a recommender system using implicit feedback. However, the MovieLens dataset that we're using is based on explicit feedback. To convert this dataset into an implicit feedback dataset, we'll simply binarize the ratings such that they are are '1' (i.e. positive class). The value of '1' represents that the user has interacted with the item.\n", + "\n", + "> Note: Using implicit feedback reframes the problem that our recommender is trying to solve. Instead of trying to predict movie ratings (when using explicit feedback), we are trying to predict whether the user will interact (i.e. click/buy/watch) with each movie, with the aim of presenting to users the movies with the highest interaction likelihood.\n", + "\n", + "> Tip: This setting is suitable at retrieval stage where the objective is to maximize recall by identifying items that user will at least interact with." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "sQYuW1Otg_Cg", + "outputId": "096aec13-d21e-4690-e88d-258fe9a681a0" + }, + "source": [ + "train_ratings.loc[:, 'rating'] = 1\n", + "\n", + "train_ratings.sample(5)" + ], + "execution_count": null, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdrating
98655406404320191
1764897511439826711
190457581235279861
312501220593864871
45403492980345711
\n", + "
" + ], + "text/plain": [ + " userId movieId rating\n", + "9865540 64043 2019 1\n", + "17648975 114398 2671 1\n", + "19045758 123527 986 1\n", + "3125012 20593 86487 1\n", + "4540349 29803 4571 1" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uhwZiaBPpsQl" + }, + "source": [ + "### Negative Sampling" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ngZdyoMjizlw" + }, + "source": [ + "We do have a problem now though. After binarizing our dataset, we see that every sample in the dataset now belongs to the positive class. However we also require negative samples to train our models, to indicate movies that the user has not interacted with. We assume that such movies are those that the user are not interested in - even though this is a sweeping assumption that may not be true, it usually works out rather well in practice.\n", + "\n", + "The code below generates 4 negative samples for each row of data. In other words, the ratio of negative to positive samples is 4:1. This ratio is chosen arbitrarily but I found that it works rather well (feel free to find the best ratio yourself!)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "4T0_UVhTizVn" + }, + "source": [ + "# Get a list of all movie IDs\n", + "all_movieIds = ratings['movieId'].unique()\n", + "\n", + "# Placeholders that will hold the training data\n", + "users, items, labels = [], [], []\n", + "\n", + "# This is the set of items that each user has interaction with\n", + "user_item_set = set(zip(train_ratings['userId'], train_ratings['movieId']))\n", + "\n", + "# 4:1 ratio of negative to positive samples\n", + "num_negatives = 4\n", + "\n", + "for (u, i) in tqdm(user_item_set):\n", + " users.append(u)\n", + " items.append(i)\n", + " labels.append(1) # items that the user has interacted with are positive\n", + " for _ in range(num_negatives):\n", + " # randomly select an item\n", + " negative_item = np.random.choice(all_movieIds) \n", + " # check that the user has not interacted with this item\n", + " while (u, negative_item) in user_item_set:\n", + " negative_item = np.random.choice(all_movieIds)\n", + " users.append(u)\n", + " items.append(negative_item)\n", + " labels.append(0) # items not interacted with are negative" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9brxnZqlpvXD" + }, + "source": [ + "### PyTorch Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Br2u5nn5jAy1" + }, + "source": [ + "Great! We now have the data in the format required by our model. Before we move on, let's define a PyTorch Dataset to facilitate training. The class below simply encapsulates the code we have written above into a PyTorch Dataset class." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "pCn0M346i6Z8" + }, + "source": [ + "class MovieLensTrainDataset(Dataset):\n", + " \"\"\"MovieLens PyTorch Dataset for Training\n", + " \n", + " Args:\n", + " ratings (pd.DataFrame): Dataframe containing the movie ratings\n", + " all_movieIds (list): List containing all movieIds\n", + " \n", + " \"\"\"\n", + "\n", + " def __init__(self, ratings, all_movieIds):\n", + " self.users, self.items, self.labels = self.get_dataset(ratings, all_movieIds)\n", + "\n", + " def __len__(self):\n", + " return len(self.users)\n", + " \n", + " def __getitem__(self, idx):\n", + " return self.users[idx], self.items[idx], self.labels[idx]\n", + "\n", + " def get_dataset(self, ratings, all_movieIds):\n", + " users, items, labels = [], [], []\n", + " user_item_set = set(zip(ratings['userId'], ratings['movieId']))\n", + "\n", + " num_negatives = 4\n", + " for u, i in user_item_set:\n", + " users.append(u)\n", + " items.append(i)\n", + " labels.append(1)\n", + " for _ in range(num_negatives):\n", + " negative_item = np.random.choice(all_movieIds)\n", + " while (u, negative_item) in user_item_set:\n", + " negative_item = np.random.choice(all_movieIds)\n", + " users.append(u)\n", + " items.append(negative_item)\n", + " labels.append(0)\n", + "\n", + " return torch.tensor(users), torch.tensor(items), torch.tensor(labels)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SqMDcEYRjOZN" + }, + "source": [ + "### Model\n", + "\n", + "While there are many deep learning based architecture for recommendation systems, I find that the framework proposed by He et al. is the most straightforward and it is simple enough to be implemented in a tutorial such as this." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xwlBJpqljJvS" + }, + "source": [ + "class NCF(pl.LightningModule):\n", + " \"\"\" Neural Collaborative Filtering (NCF)\n", + " \n", + " Args:\n", + " num_users (int): Number of unique users\n", + " num_items (int): Number of unique items\n", + " ratings (pd.DataFrame): Dataframe containing the movie ratings for training\n", + " all_movieIds (list): List containing all movieIds (train + test)\n", + " \"\"\"\n", + " \n", + " def __init__(self, num_users, num_items, ratings, all_movieIds):\n", + " super().__init__()\n", + " self.user_embedding = nn.Embedding(num_embeddings=num_users, embedding_dim=8)\n", + " self.item_embedding = nn.Embedding(num_embeddings=num_items, embedding_dim=8)\n", + " self.fc1 = nn.Linear(in_features=16, out_features=64)\n", + " self.fc2 = nn.Linear(in_features=64, out_features=32)\n", + " self.output = nn.Linear(in_features=32, out_features=1)\n", + " self.ratings = ratings\n", + " self.all_movieIds = all_movieIds\n", + " \n", + " def forward(self, user_input, item_input):\n", + " \n", + " # Pass through embedding layers\n", + " user_embedded = self.user_embedding(user_input)\n", + " item_embedded = self.item_embedding(item_input)\n", + "\n", + " # Concat the two embedding layers\n", + " vector = torch.cat([user_embedded, item_embedded], dim=-1)\n", + "\n", + " # Pass through dense layer\n", + " vector = nn.ReLU()(self.fc1(vector))\n", + " vector = nn.ReLU()(self.fc2(vector))\n", + "\n", + " # Output layer\n", + " pred = nn.Sigmoid()(self.output(vector))\n", + "\n", + " return pred\n", + " \n", + " def training_step(self, batch, batch_idx):\n", + " user_input, item_input, labels = batch\n", + " predicted_labels = self(user_input, item_input)\n", + " loss = nn.BCELoss()(predicted_labels, labels.view(-1, 1).float())\n", + " return loss\n", + "\n", + " def configure_optimizers(self):\n", + " return torch.optim.Adam(self.parameters())\n", + "\n", + " def train_dataloader(self):\n", + " return DataLoader(MovieLensTrainDataset(self.ratings, self.all_movieIds),\n", + " batch_size=512, num_workers=2)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I8wT1WK9jzeJ" + }, + "source": [ + "We instantiate the NCF model using the class that we have defined above." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "F3Bh9dorjww7" + }, + "source": [ + "num_users = ratings['userId'].max()+1\n", + "num_items = ratings['movieId'].max()+1\n", + "\n", + "all_movieIds = ratings['movieId'].unique()\n", + "\n", + "model = NCF(num_users, num_items, train_ratings, all_movieIds)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K4Mw8CdVp5lF" + }, + "source": [ + "### Model Training" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xnYHNWe3kRRD" + }, + "source": [ + "> Note: One advantage of PyTorch Lightning over vanilla PyTorch is that you don't need to write your own boiler plate training code. Notice how the Trainer class allows us to train our model with just a few lines of code.\n", + "\n", + "Let's train our NCF model for 5 epochs using the GPU. " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 375, + "referenced_widgets": [ + "68bcd7bfc32f4d9ebaba5c08437bca28" + ] + }, + "id": "0JganCIMj2EW", + "outputId": "6fa64b89-c835-4f39-d6ad-bac6f2369c64" + }, + "source": [ + "trainer = pl.Trainer(max_epochs=5, gpus=1, reload_dataloaders_every_epoch=True,\n", + " progress_bar_refresh_rate=50, logger=False, checkpoint_callback=False)\n", + "\n", + "trainer.fit(model)" + ], + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "---------------------------------------------\n", + "0 | user_embedding | Embedding | 1.3 M \n", + "1 | item_embedding | Embedding | 1.7 M \n", + "2 | fc1 | Linear | 1.1 K \n", + "3 | fc2 | Linear | 2.1 K \n", + "4 | output | Linear | 33 \n", + "---------------------------------------------\n", + "3.0 M Trainable params\n", + "0 Non-trainable params\n", + "3.0 M Total params\n", + "11.907 Total estimated model params size (MB)\n", + "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", + " cpuset_checked))\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "68bcd7bfc32f4d9ebaba5c08437bca28", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R6V1Tiw1kIxk" + }, + "source": [ + "> Note: We are using the argument reload_dataloaders_every_epoch=True. This creates a new randomly chosen set of negative samples for each epoch, which ensures that our model is not biased by the selection of negative samples." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I3BXx1YzlAUq" + }, + "source": [ + "### Evaluating our Recommender System\n", + "\n", + "Now that our model is trained, we are ready to evaluate it using the test data. In traditional Machine Learning projects, we evaluate our models using metrics such as Accuracy (for classification problems) and RMSE (for regression problems). However, such metrics are too simplistic for evaluating recommender systems.\n", + "\n", + "The key here is that we don't need the user to interact on every single item in the list of recommendations. Instead, we just need the user to interact with at least one item on the list - as long as the user does that, the recommendations have worked.\n", + "\n", + "To simulate this, let's run the following evaluation protocol to generate a list of 10 recommended items for each user.\n", + "- For each user, randomly select 99 items that the user has not interacted with\n", + "- Combine these 99 items with the test item (the actual item that the user interacted with). We now have 100 items.\n", + "- Run the model on these 100 items, and rank them according to their predicted probabilities\n", + "- Select the top 10 items from the list of 100 items. If the test item is present within the top 10 items, then we say that this is a hit.\n", + "- Repeat the process for all users. The Hit Ratio is then the average hits." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B2PVVpUflN34" + }, + "source": [ + "> Note: This evaluation protocol is known as Hit Ratio @ 10, and it is commonly used to evaluate recommender systems." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "uSLTYZuhlNEV" + }, + "source": [ + "# User-item pairs for testing\n", + "test_user_item_set = set(zip(test_ratings['userId'], test_ratings['movieId']))\n", + "\n", + "# Dict of all items that are interacted with by each user\n", + "user_interacted_items = ratings.groupby('userId')['movieId'].apply(list).to_dict()\n", + "\n", + "hits = []\n", + "for (u,i) in tqdm(test_user_item_set):\n", + " interacted_items = user_interacted_items[u]\n", + " not_interacted_items = set(all_movieIds) - set(interacted_items)\n", + " selected_not_interacted = list(np.random.choice(list(not_interacted_items), 99))\n", + " test_items = selected_not_interacted + [i]\n", + " \n", + " predicted_labels = np.squeeze(model(torch.tensor([u]*100), \n", + " torch.tensor(test_items)).detach().numpy())\n", + " \n", + " top10_items = [test_items[i] for i in np.argsort(predicted_labels)[::-1][0:10].tolist()]\n", + " \n", + " if i in top10_items:\n", + " hits.append(1)\n", + " else:\n", + " hits.append(0)\n", + " \n", + "print(\"The Hit Ratio @ 10 is {:.2f}\".format(np.average(hits)))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s1XtzBFsllfN" + }, + "source": [ + "We got a pretty good Hit Ratio @ 10 score! To put this into context, what this means is that 86% of the users were recommended the actual item (among a list of 10 items) that they eventually interacted with. Not bad!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_xofdqRI29zl" + }, + "source": [ + "## NMF with PyTorch on ML-1m" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3wo7aehx3AyG" + }, + "source": [ + "import os\n", + "import time\n", + "import random\n", + "import argparse\n", + "import numpy as np \n", + "import pandas as pd \n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import torch.utils.data as data\n", + "from torch.utils.tensorboard import SummaryWriter" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "y1iNipgl3JhO", + "outputId": "363cf5f9-b062-4930-b122-d7573d824ab0" + }, + "source": [ + "DATA_URL = \"https://raw.githubusercontent.com/sparsh-ai/rec-data-public/master/ml-1m-dat/ratings.dat\"\n", + "MAIN_PATH = '/content/'\n", + "DATA_PATH = MAIN_PATH + 'ratings.dat'\n", + "MODEL_PATH = MAIN_PATH + 'models/'\n", + "MODEL = 'ml-1m_Neu_MF'\n", + "\n", + "!wget -q --show-progress https://raw.githubusercontent.com/sparsh-ai/rec-data-public/master/ml-1m-dat/ratings.dat" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\rratings.dat 0%[ ] 0 --.-KB/s \rratings.dat 100%[===================>] 23.45M 128MB/s in 0.2s \n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "EXFnsMFy3YTE" + }, + "source": [ + "def seed_everything(seed):\n", + " random.seed(seed)\n", + " os.environ['PYTHONHASHSEED'] = str(seed)\n", + " np.random.seed(seed)\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " torch.backends.cudnn.deterministic = True\n", + " torch.backends.cudnn.benchmark = True" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KvTX81Z23bFs" + }, + "source": [ + "### Dataset" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "NN5GjJCf3rI8" + }, + "source": [ + "class Rating_Datset(torch.utils.data.Dataset):\n", + "\tdef __init__(self, user_list, item_list, rating_list):\n", + "\t\tsuper(Rating_Datset, self).__init__()\n", + "\t\tself.user_list = user_list\n", + "\t\tself.item_list = item_list\n", + "\t\tself.rating_list = rating_list\n", + "\n", + "\tdef __len__(self):\n", + "\t\treturn len(self.user_list)\n", + "\n", + "\tdef __getitem__(self, idx):\n", + "\t\tuser = self.user_list[idx]\n", + "\t\titem = self.item_list[idx]\n", + "\t\trating = self.rating_list[idx]\n", + "\t\t\n", + "\t\treturn (\n", + "\t\t\ttorch.tensor(user, dtype=torch.long),\n", + "\t\t\ttorch.tensor(item, dtype=torch.long),\n", + "\t\t\ttorch.tensor(rating, dtype=torch.float)\n", + "\t\t\t)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d4xgxyBsfoJM" + }, + "source": [ + "- *_reindex*: process dataset to reindex userID and itemID, also set rating as binary feedback\n", + "- *_leave_one_out*: leave-one-out evaluation protocol in paper https://www.comp.nus.edu.sg/~xiangnan/papers/ncf.pdf\n", + "- *negative_sampling*: randomly selects n negative examples for each positive one" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HggfgX_8Oqmq" + }, + "source": [ + "class NCF_Data(object):\n", + "\t\"\"\"\n", + "\tConstruct Dataset for NCF\n", + "\t\"\"\"\n", + "\tdef __init__(self, args, ratings):\n", + "\t\tself.ratings = ratings\n", + "\t\tself.num_ng = args.num_ng\n", + "\t\tself.num_ng_test = args.num_ng_test\n", + "\t\tself.batch_size = args.batch_size\n", + "\n", + "\t\tself.preprocess_ratings = self._reindex(self.ratings)\n", + "\n", + "\t\tself.user_pool = set(self.ratings['user_id'].unique())\n", + "\t\tself.item_pool = set(self.ratings['item_id'].unique())\n", + "\n", + "\t\tself.train_ratings, self.test_ratings = self._leave_one_out(self.preprocess_ratings)\n", + "\t\tself.negatives = self._negative_sampling(self.preprocess_ratings)\n", + "\t\trandom.seed(args.seed)\n", + "\t\n", + "\tdef _reindex(self, ratings):\n", + "\t\t\"\"\"\n", + "\t\tProcess dataset to reindex userID and itemID, also set rating as binary feedback\n", + "\t\t\"\"\"\n", + "\t\tuser_list = list(ratings['user_id'].drop_duplicates())\n", + "\t\tuser2id = {w: i for i, w in enumerate(user_list)}\n", + "\n", + "\t\titem_list = list(ratings['item_id'].drop_duplicates())\n", + "\t\titem2id = {w: i for i, w in enumerate(item_list)}\n", + "\n", + "\t\tratings['user_id'] = ratings['user_id'].apply(lambda x: user2id[x])\n", + "\t\tratings['item_id'] = ratings['item_id'].apply(lambda x: item2id[x])\n", + "\t\tratings['rating'] = ratings['rating'].apply(lambda x: float(x > 0))\n", + "\t\treturn ratings\n", + "\n", + "\tdef _leave_one_out(self, ratings):\n", + "\t\t\"\"\"\n", + "\t\tleave-one-out evaluation protocol in paper https://www.comp.nus.edu.sg/~xiangnan/papers/ncf.pdf\n", + "\t\t\"\"\"\n", + "\t\tratings['rank_latest'] = ratings.groupby(['user_id'])['timestamp'].rank(method='first', ascending=False)\n", + "\t\ttest = ratings.loc[ratings['rank_latest'] == 1]\n", + "\t\ttrain = ratings.loc[ratings['rank_latest'] > 1]\n", + "\t\tassert train['user_id'].nunique()==test['user_id'].nunique(), 'Not Match Train User with Test User'\n", + "\t\treturn train[['user_id', 'item_id', 'rating']], test[['user_id', 'item_id', 'rating']]\n", + "\n", + "\tdef _negative_sampling(self, ratings):\n", + "\t\tinteract_status = (\n", + "\t\t\tratings.groupby('user_id')['item_id']\n", + "\t\t\t.apply(set)\n", + "\t\t\t.reset_index()\n", + "\t\t\t.rename(columns={'item_id': 'interacted_items'}))\n", + "\t\tinteract_status['negative_items'] = interact_status['interacted_items'].apply(lambda x: self.item_pool - x)\n", + "\t\tinteract_status['negative_samples'] = interact_status['negative_items'].apply(lambda x: random.sample(x, self.num_ng_test))\n", + "\t\treturn interact_status[['user_id', 'negative_items', 'negative_samples']]\n", + "\n", + "\tdef get_train_instance(self):\n", + "\t\tusers, items, ratings = [], [], []\n", + "\t\ttrain_ratings = pd.merge(self.train_ratings, self.negatives[['user_id', 'negative_items']], on='user_id')\n", + "\t\ttrain_ratings['negatives'] = train_ratings['negative_items'].apply(lambda x: random.sample(x, self.num_ng))\n", + "\t\tfor row in train_ratings.itertuples():\n", + "\t\t\tusers.append(int(row.user_id))\n", + "\t\t\titems.append(int(row.item_id))\n", + "\t\t\tratings.append(float(row.rating))\n", + "\t\t\tfor i in range(self.num_ng):\n", + "\t\t\t\tusers.append(int(row.user_id))\n", + "\t\t\t\titems.append(int(row.negatives[i]))\n", + "\t\t\t\tratings.append(float(0)) # negative samples get 0 rating\n", + "\t\tdataset = Rating_Datset(\n", + "\t\t\tuser_list=users,\n", + "\t\t\titem_list=items,\n", + "\t\t\trating_list=ratings)\n", + "\t\treturn torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=True, num_workers=2)\n", + "\n", + "\tdef get_test_instance(self):\n", + "\t\tusers, items, ratings = [], [], []\n", + "\t\ttest_ratings = pd.merge(self.test_ratings, self.negatives[['user_id', 'negative_samples']], on='user_id')\n", + "\t\tfor row in test_ratings.itertuples():\n", + "\t\t\tusers.append(int(row.user_id))\n", + "\t\t\titems.append(int(row.item_id))\n", + "\t\t\tratings.append(float(row.rating))\n", + "\t\t\tfor i in getattr(row, 'negative_samples'):\n", + "\t\t\t\tusers.append(int(row.user_id))\n", + "\t\t\t\titems.append(int(i))\n", + "\t\t\t\tratings.append(float(0))\n", + "\t\tdataset = Rating_Datset(\n", + "\t\t\tuser_list=users,\n", + "\t\t\titem_list=items,\n", + "\t\t\trating_list=ratings)\n", + "\t\treturn torch.utils.data.DataLoader(dataset, batch_size=self.num_ng_test+1, shuffle=False, num_workers=2)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hfXyLsgVOsBs" + }, + "source": [ + "### Metrics\n", + "Using Hit Rate and NDCG as our evaluation metrics" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KM4B7r12OvnS" + }, + "source": [ + "def hit(ng_item, pred_items):\n", + "\tif ng_item in pred_items:\n", + "\t\treturn 1\n", + "\treturn 0\n", + "\n", + "\n", + "def ndcg(ng_item, pred_items):\n", + "\tif ng_item in pred_items:\n", + "\t\tindex = pred_items.index(ng_item)\n", + "\t\treturn np.reciprocal(np.log2(index+2))\n", + "\treturn 0\n", + "\n", + "\n", + "def metrics(model, test_loader, top_k, device):\n", + "\tHR, NDCG = [], []\n", + "\n", + "\tfor user, item, label in test_loader:\n", + "\t\tuser = user.to(device)\n", + "\t\titem = item.to(device)\n", + "\n", + "\t\tpredictions = model(user, item)\n", + "\t\t_, indices = torch.topk(predictions, top_k)\n", + "\t\trecommends = torch.take(\n", + "\t\t\t\titem, indices).cpu().numpy().tolist()\n", + "\n", + "\t\tng_item = item[0].item() # leave one-out evaluation has only one item per user\n", + "\t\tHR.append(hit(ng_item, recommends))\n", + "\t\tNDCG.append(ndcg(ng_item, recommends))\n", + "\n", + "\treturn np.mean(HR), np.mean(NDCG)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HWyCD7pLOxjq" + }, + "source": [ + "### Models\n", + "- Generalized Matrix Factorization\n", + "- Multi Layer Perceptron\n", + "- Neural Matrix Factorization" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "aTQaitu7d1R3" + }, + "source": [ + "class Generalized_Matrix_Factorization(nn.Module):\n", + " def __init__(self, args, num_users, num_items):\n", + " super(Generalized_Matrix_Factorization, self).__init__()\n", + " self.num_users = num_users\n", + " self.num_items = num_items\n", + " self.factor_num = args.factor_num\n", + "\n", + " self.embedding_user = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num)\n", + " self.embedding_item = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num)\n", + "\n", + " self.affine_output = nn.Linear(in_features=self.factor_num, out_features=1)\n", + " self.logistic = nn.Sigmoid()\n", + "\n", + " def forward(self, user_indices, item_indices):\n", + " user_embedding = self.embedding_user(user_indices)\n", + " item_embedding = self.embedding_item(item_indices)\n", + " element_product = torch.mul(user_embedding, item_embedding)\n", + " logits = self.affine_output(element_product)\n", + " rating = self.logistic(logits)\n", + " return rating\n", + "\n", + " def init_weight(self):\n", + " pass" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "7kSFzPlNd50f" + }, + "source": [ + "class Multi_Layer_Perceptron(nn.Module):\n", + " def __init__(self, args, num_users, num_items):\n", + " super(Multi_Layer_Perceptron, self).__init__()\n", + " self.num_users = num_users\n", + " self.num_items = num_items\n", + " self.factor_num = args.factor_num\n", + " self.layers = args.layers\n", + "\n", + " self.embedding_user = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num)\n", + " self.embedding_item = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num)\n", + "\n", + " self.fc_layers = nn.ModuleList()\n", + " for idx, (in_size, out_size) in enumerate(zip(self.layers[:-1], self.layers[1:])):\n", + " self.fc_layers.append(nn.Linear(in_size, out_size))\n", + "\n", + " self.affine_output = nn.Linear(in_features=self.layers[-1], out_features=1)\n", + " self.logistic = nn.Sigmoid()\n", + "\n", + " def forward(self, user_indices, item_indices):\n", + " user_embedding = self.embedding_user(user_indices)\n", + " item_embedding = self.embedding_item(item_indices)\n", + " vector = torch.cat([user_embedding, item_embedding], dim=-1) # the concat latent vector\n", + " for idx, _ in enumerate(range(len(self.fc_layers))):\n", + " vector = self.fc_layers[idx](vector)\n", + " vector = nn.ReLU()(vector)\n", + " # vector = nn.BatchNorm1d()(vector)\n", + " # vector = nn.Dropout(p=0.5)(vector)\n", + " logits = self.affine_output(vector)\n", + " rating = self.logistic(logits)\n", + " return rating\n", + "\n", + " def init_weight(self):\n", + " pass" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "7DQpVuaV9cF0" + }, + "source": [ + "class NeuMF(nn.Module):\n", + " def __init__(self, args, num_users, num_items):\n", + " super(NeuMF, self).__init__()\n", + " self.num_users = num_users\n", + " self.num_items = num_items\n", + " self.factor_num_mf = args.factor_num\n", + " self.factor_num_mlp = int(args.layers[0]/2)\n", + " self.layers = args.layers\n", + " self.dropout = args.dropout\n", + "\n", + " self.embedding_user_mlp = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num_mlp)\n", + " self.embedding_item_mlp = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num_mlp)\n", + "\n", + " self.embedding_user_mf = nn.Embedding(num_embeddings=self.num_users, embedding_dim=self.factor_num_mf)\n", + " self.embedding_item_mf = nn.Embedding(num_embeddings=self.num_items, embedding_dim=self.factor_num_mf)\n", + "\n", + " self.fc_layers = nn.ModuleList()\n", + " for idx, (in_size, out_size) in enumerate(zip(args.layers[:-1], args.layers[1:])):\n", + " self.fc_layers.append(torch.nn.Linear(in_size, out_size))\n", + " self.fc_layers.append(nn.ReLU())\n", + "\n", + " self.affine_output = nn.Linear(in_features=args.layers[-1] + self.factor_num_mf, out_features=1)\n", + " self.logistic = nn.Sigmoid()\n", + " self.init_weight()\n", + "\n", + " def init_weight(self):\n", + " nn.init.normal_(self.embedding_user_mlp.weight, std=0.01)\n", + " nn.init.normal_(self.embedding_item_mlp.weight, std=0.01)\n", + " nn.init.normal_(self.embedding_user_mf.weight, std=0.01)\n", + " nn.init.normal_(self.embedding_item_mf.weight, std=0.01)\n", + " \n", + " for m in self.fc_layers:\n", + " if isinstance(m, nn.Linear):\n", + " nn.init.xavier_uniform_(m.weight)\n", + " \n", + " nn.init.xavier_uniform_(self.affine_output.weight)\n", + "\n", + " for m in self.modules():\n", + " if isinstance(m, nn.Linear) and m.bias is not None:\n", + " m.bias.data.zero_()\n", + "\n", + " def forward(self, user_indices, item_indices):\n", + " user_embedding_mlp = self.embedding_user_mlp(user_indices)\n", + " item_embedding_mlp = self.embedding_item_mlp(item_indices)\n", + "\n", + " user_embedding_mf = self.embedding_user_mf(user_indices)\n", + " item_embedding_mf = self.embedding_item_mf(item_indices)\n", + "\n", + " mlp_vector = torch.cat([user_embedding_mlp, item_embedding_mlp], dim=-1)\n", + " mf_vector =torch.mul(user_embedding_mf, item_embedding_mf)\n", + "\n", + " for idx, _ in enumerate(range(len(self.fc_layers))):\n", + " mlp_vector = self.fc_layers[idx](mlp_vector)\n", + "\n", + " vector = torch.cat([mlp_vector, mf_vector], dim=-1)\n", + " logits = self.affine_output(vector)\n", + " rating = self.logistic(logits)\n", + " return rating.squeeze()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tpBX6rqNfSc9" + }, + "source": [ + "### Setting Arguments\n", + "\n", + "Here is the brief description of important ones:\n", + "- Learning rate is 0.001\n", + "- Dropout rate is 0.2\n", + "- Running for 10 epochs\n", + "- HitRate@10 and NDCG@10\n", + "- 4 negative samples for each positive one" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Bc5Vg1Ik_gnF", + "outputId": "072e970d-c6d2-413c-d6f4-2f25e13ee4bf" + }, + "source": [ + "parser = argparse.ArgumentParser()\n", + "parser.add_argument(\"--seed\", \n", + "\ttype=int, \n", + "\tdefault=42, \n", + "\thelp=\"Seed\")\n", + "parser.add_argument(\"--lr\", \n", + "\ttype=float, \n", + "\tdefault=0.001, \n", + "\thelp=\"learning rate\")\n", + "parser.add_argument(\"--dropout\", \n", + "\ttype=float,\n", + "\tdefault=0.2, \n", + "\thelp=\"dropout rate\")\n", + "parser.add_argument(\"--batch_size\", \n", + "\ttype=int, \n", + "\tdefault=256, \n", + "\thelp=\"batch size for training\")\n", + "parser.add_argument(\"--epochs\", \n", + "\ttype=int,\n", + "\tdefault=10, \n", + "\thelp=\"training epoches\")\n", + "parser.add_argument(\"--top_k\", \n", + "\ttype=int, \n", + "\tdefault=10, \n", + "\thelp=\"compute metrics@top_k\")\n", + "parser.add_argument(\"--factor_num\", \n", + "\ttype=int,\n", + "\tdefault=32, \n", + "\thelp=\"predictive factors numbers in the model\")\n", + "parser.add_argument(\"--layers\",\n", + " nargs='+', \n", + " default=[64,32,16,8],\n", + " help=\"MLP layers. Note that the first layer is the concatenation of user \\\n", + " and item embeddings. So layers[0]/2 is the embedding size.\")\n", + "parser.add_argument(\"--num_ng\", \n", + "\ttype=int,\n", + "\tdefault=4, \n", + "\thelp=\"Number of negative samples for training set\")\n", + "parser.add_argument(\"--num_ng_test\", \n", + "\ttype=int,\n", + "\tdefault=100, \n", + "\thelp=\"Number of negative samples for test set\")\n", + "parser.add_argument(\"--out\", \n", + "\tdefault=True,\n", + "\thelp=\"save model or not\")" + ], + "execution_count": null, + "outputs": [ + { + "data": { + "text/plain": [ + "_StoreAction(option_strings=['--out'], dest='out', nargs=None, const=None, default=True, type=None, choices=None, help='save model or not', metavar=None)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RnaRWy2gg_Nw" + }, + "source": [ + "### Training" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "background_save": true, + "base_uri": "https://localhost:8080/" + }, + "id": "VyWquJG893CV", + "outputId": "61938a61-f7f5-4885-85d1-1e3e2d2a06f6" + }, + "source": [ + "# set device and parameters\n", + "args = parser.parse_args(args={})\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "writer = SummaryWriter()\n", + "\n", + "# seed for Reproducibility\n", + "seed_everything(args.seed)\n", + "\n", + "# load data\n", + "ml_1m = pd.read_csv(\n", + "\tDATA_PATH, \n", + "\tsep=\"::\", \n", + "\tnames = ['user_id', 'item_id', 'rating', 'timestamp'], \n", + "\tengine='python')\n", + "\n", + "# set the num_users, items\n", + "num_users = ml_1m['user_id'].nunique()+1\n", + "num_items = ml_1m['item_id'].nunique()+1\n", + "\n", + "# construct the train and test datasets\n", + "data = NCF_Data(args, ml_1m)\n", + "train_loader = data.get_train_instance()\n", + "test_loader = data.get_test_instance()\n", + "\n", + "# set model and loss, optimizer\n", + "model = NeuMF(args, num_users, num_items)\n", + "model = model.to(device)\n", + "loss_function = nn.BCELoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=args.lr)\n", + "\n", + "# train, evaluation\n", + "best_hr = 0\n", + "for epoch in range(1, args.epochs+1):\n", + "\tmodel.train() # Enable dropout (if have).\n", + "\tstart_time = time.time()\n", + "\n", + "\tfor user, item, label in train_loader:\n", + "\t\tuser = user.to(device)\n", + "\t\titem = item.to(device)\n", + "\t\tlabel = label.to(device)\n", + "\n", + "\t\toptimizer.zero_grad()\n", + "\t\tprediction = model(user, item)\n", + "\t\tloss = loss_function(prediction, label)\n", + "\t\tloss.backward()\n", + "\t\toptimizer.step()\n", + "\t\twriter.add_scalar('loss/Train_loss', loss.item(), epoch)\n", + "\n", + "\tmodel.eval()\n", + "\tHR, NDCG = metrics(model, test_loader, args.top_k, device)\n", + "\twriter.add_scalar('Perfomance/HR@10', HR, epoch)\n", + "\twriter.add_scalar('Perfomance/NDCG@10', NDCG, epoch)\n", + "\n", + "\telapsed_time = time.time() - start_time\n", + "\tprint(\"The time elapse of epoch {:03d}\".format(epoch) + \" is: \" + \n", + "\t\t\ttime.strftime(\"%H: %M: %S\", time.gmtime(elapsed_time)))\n", + "\tprint(\"HR: {:.3f}\\tNDCG: {:.3f}\".format(np.mean(HR), np.mean(NDCG)))\n", + "\n", + "\tif HR > best_hr:\n", + "\t\tbest_hr, best_ndcg, best_epoch = HR, NDCG, epoch\n", + "\t\tif args.out:\n", + "\t\t\tif not os.path.exists(MODEL_PATH):\n", + "\t\t\t\tos.mkdir(MODEL_PATH)\n", + "\t\t\ttorch.save(model, \n", + "\t\t\t\t'{}{}.pth'.format(MODEL_PATH, MODEL))\n", + "\n", + "writer.close()" + ], + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", + " cpuset_checked))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The time elapse of epoch 001 is: 00: 05: 41\n", + "HR: 0.626\tNDCG: 0.359\n", + "The time elapse of epoch 002 is: 00: 05: 42\n", + "HR: 0.658\tNDCG: 0.389\n", + "The time elapse of epoch 003 is: 00: 05: 47\n", + "HR: 0.664\tNDCG: 0.396\n", + "The time elapse of epoch 004 is: 00: 05: 34\n", + "HR: 0.669\tNDCG: 0.400\n", + "The time elapse of epoch 005 is: 00: 05: 44\n", + "HR: 0.671\tNDCG: 0.401\n", + "The time elapse of epoch 006 is: 00: 05: 44\n", + "HR: 0.672\tNDCG: 0.402\n", + "The time elapse of epoch 007 is: 00: 05: 39\n", + "HR: 0.668\tNDCG: 0.396\n", + "The time elapse of epoch 008 is: 00: 05: 34\n", + "HR: 0.667\tNDCG: 0.396\n", + "The time elapse of epoch 009 is: 00: 05: 41\n", + "HR: 0.668\tNDCG: 0.397\n", + "The time elapse of epoch 010 is: 00: 05: 37\n", + "HR: 0.664\tNDCG: 0.395\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "background_save": true, + "base_uri": "https://localhost:8080/" + }, + "id": "fkiRJWeD_trR", + "outputId": "d3efcab5-fa0b-4938-d5ff-7967f38dab4d" + }, + "source": [ + "print(\"Best epoch {:03d}: HR = {:.3f}, NDCG = {:.3f}\".format(\n", + "\t\t\t\t\t\t\t\t\tbest_epoch, best_hr, best_ndcg))" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best epoch 006: HR = 0.672, NDCG = 0.402\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WOTaSMGnPoAG" + }, + "source": [ + "## MF with PyTorch on ML-100k\n", + "\n", + "Training Pytorch MLP model on movielens-100k dataset and visualizing factors by decomposing using PCA" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "I2f_R0Yo6BUp" + }, + "source": [ + "!pip install -U -q git+https://github.com/sparsh-ai/recochef.git" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "KXT07lHDBzAQ" + }, + "source": [ + "import torch\n", + "from torch import nn\n", + "from torch import optim\n", + "from torch.nn import functional as F \n", + "from torch.optim.lr_scheduler import _LRScheduler\n", + "\n", + "from recochef.datasets.movielens import MovieLens\n", + "from recochef.preprocessing.encode import label_encode\n", + "from recochef.utils.iterators import batch_generator\n", + "from recochef.models.embedding import EmbeddingNet\n", + "\n", + "import math\n", + "import copy\n", + "import pickle\n", + "import numpy as np\n", + "import pandas as pd\n", + "from textwrap import wrap\n", + "from sklearn.decomposition import PCA\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "import matplotlib.pyplot as plt\n", + "plt.style.use('ggplot')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NINhOhYAxt5n" + }, + "source": [ + "### Data loading and preprocessing" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3Z4R3bXNjaNP" + }, + "source": [ + "data = MovieLens()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "A2Xgw-sXk7Ac", + "outputId": "399404ea-51bd-476f-f74e-0dc4807ebaa9" + }, + "source": [ + "ratings_df = data.load_interactions()\n", + "ratings_df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "0 196 242 3.0 881250949\n", + "1 186 302 3.0 891717742\n", + "2 22 377 1.0 878887116\n", + "3 244 51 2.0 880606923\n", + "4 166 346 1.0 886397596" + ] + }, + "metadata": {}, + "execution_count": 16 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 343 + }, + "id": "wIUc-Ba_6xBK", + "outputId": "3f1382b8-d132-48b8-f85f-fae21140922d" + }, + "source": [ + "movies_df = data.load_items()\n", + "movies_df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
01Toy Story (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Toy%20Story%2...0001110000000000000
12GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
23Four Rooms (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Four%20Rooms%...0000000000000000100
34Get Shorty (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Get%20Shorty%...0100010010000000000
45Copycat (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Copycat%20(1995)0000001010000000100
\n", + "
" + ], + "text/plain": [ + " ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n", + "0 1 Toy Story (1995) 01-Jan-1995 ... 0 0 0\n", + "1 2 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n", + "2 3 Four Rooms (1995) 01-Jan-1995 ... 1 0 0\n", + "3 4 Get Shorty (1995) 01-Jan-1995 ... 0 0 0\n", + "4 5 Copycat (1995) 01-Jan-1995 ... 1 0 0\n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "metadata": {}, + "execution_count": 17 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "X-IcaBzgmOrN", + "outputId": "7a6afd50-b93a-498b-f9ea-96d7db363795" + }, + "source": [ + "ratings_df, umap = label_encode(ratings_df, 'USERID')\n", + "ratings_df, imap = label_encode(ratings_df, 'ITEMID')\n", + "ratings_df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
0003.0881250949
1113.0891717742
2221.0878887116
3332.0880606923
4441.0886397596
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "0 0 0 3.0 881250949\n", + "1 1 1 3.0 891717742\n", + "2 2 2 1.0 878887116\n", + "3 3 3 2.0 880606923\n", + "4 4 4 1.0 886397596" + ] + }, + "metadata": {}, + "execution_count": 50 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "36dsiSqWwNRz" + }, + "source": [ + "X = ratings_df[['USERID','ITEMID']]\n", + "y = ratings_df[['RATING']]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Mp-OqA2jyNAS", + "outputId": "969eca89-0f0e-464d-d98e-70aeb215b99b" + }, + "source": [ + "for _x_batch, _y_batch in batch_generator(X, y, bs=4):\n", + " print(_x_batch)\n", + " print(_y_batch)\n", + " break" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "tensor([[873, 377],\n", + " [808, 601],\n", + " [ 90, 354],\n", + " [409, 570]])\n", + "tensor([[4.],\n", + " [3.],\n", + " [4.],\n", + " [2.]])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oQlnTmST0cx6", + "outputId": "96db7cc7-9614-4215-f0e8-be69eb17e4e0" + }, + "source": [ + "_x_batch[:, 1]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor([377, 601, 354, 570])" + ] + }, + "metadata": {}, + "execution_count": 24 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mVavggU2_WUY" + }, + "source": [ + "### Embedding Net" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D39WDxT5_f3l" + }, + "source": [ + "The PyTorch is a framework that allows to build various computational graphs (not only neural networks) and run them on GPU. The conception of tensors, neural networks, and computational graphs is outside the scope of this article but briefly speaking, one could treat the library as a set of tools to create highly computationally efficient and flexible machine learning models. In our case, we want to create a neural network that could help us to infer the similarities between users and predict their ratings based on available data." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9Gve8w7f_l8i" + }, + "source": [ + "The picture above schematically shows the model we're going to build. At the very beginning, we put our embeddings matrices, or look-ups, which convert integer IDs into arrays of floating-point numbers. Next, we put a bunch of fully-connected layers with dropouts. Finally, we need to return a list of predicted ratings. For this purpose, we use a layer with sigmoid activation function and rescale it to the original range of values (in case of MovieLens dataset, it is usually from 1 to 5)." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9zDzhH2c0Cv-", + "outputId": "134739d6-e9ff-4cd7-a0b5-0aac999500fa" + }, + "source": [ + "netx = EmbeddingNet(\n", + " n_users=50, n_items=20, \n", + " n_factors=10, hidden=[500], \n", + " embedding_dropout=0.05, dropouts=[0.5])\n", + "netx" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "EmbeddingNet(\n", + " (u): Embedding(50, 10)\n", + " (m): Embedding(20, 10)\n", + " (drop): Dropout(p=0.05, inplace=False)\n", + " (hidden): Sequential(\n", + " (0): Linear(in_features=20, out_features=500, bias=True)\n", + " (1): ReLU()\n", + " (2): Dropout(p=0.5, inplace=False)\n", + " )\n", + " (fc): Linear(in_features=500, out_features=1, bias=True)\n", + ")" + ] + }, + "metadata": {}, + "execution_count": 25 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o4vQFZ6iwiyM" + }, + "source": [ + "### Cyclical Learning Rate (CLR)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6RIaav5rwk66" + }, + "source": [ + "One of the `fastai` library features is the cyclical learning rate scheduler. We can implement something similar inheriting the `_LRScheduler` class from the `torch` library. Following the [original paper's](https://arxiv.org/abs/1506.01186) pseudocode, this [CLR Keras callback implementation](https://github.com/bckenstler/CLR), and making a couple of adjustments to support [cosine annealing](https://pytorch.org/docs/stable/optim.html#torch.optim.lr_scheduler.CosineAnnealingLR) with restarts, let's create our own CLR scheduler.\n", + "\n", + "The implementation of this idea is quite simple. The [base PyTorch scheduler class](https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html) has the `get_lr()` method that is invoked each time when we call the `step()` method. The method should return a list of learning rates depending on the current training epoch. In our case, we have the same learning rate for all of the layers, and therefore, we return a list with a single value. \n", + "\n", + "The next cell defines a `CyclicLR` class that expectes a single callback function. This function should accept the current training epoch and the base value of learning rate, and return a new learning rate value." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "eYQh4ZCmmgW9" + }, + "source": [ + "class CyclicLR(_LRScheduler):\n", + " \n", + " def __init__(self, optimizer, schedule, last_epoch=-1):\n", + " assert callable(schedule)\n", + " self.schedule = schedule\n", + " super().__init__(optimizer, last_epoch)\n", + "\n", + " def get_lr(self):\n", + " return [self.schedule(self.last_epoch, lr) for lr in self.base_lrs]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1bpK5hOvw7Hg" + }, + "source": [ + "Our scheduler is very similar to [LambdaLR](https://pytorch.org/docs/stable/optim.html#torch.optim.lr_scheduler.LambdaLR) one but expects a bit different callback signature. \n", + "\n", + "So now we only need to define appropriate scheduling functions. We're createing a couple of functions that accept scheduling parameters and return a _new function_ with the appropriate signature:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "I6st2zPctj1T" + }, + "source": [ + "def triangular(step_size, max_lr, method='triangular', gamma=0.99):\n", + " \n", + " def scheduler(epoch, base_lr):\n", + " period = 2 * step_size\n", + " cycle = math.floor(1 + epoch/period)\n", + " x = abs(epoch/step_size - 2*cycle + 1)\n", + " delta = (max_lr - base_lr)*max(0, (1 - x))\n", + "\n", + " if method == 'triangular':\n", + " pass # we've already done\n", + " elif method == 'triangular2':\n", + " delta /= float(2 ** (cycle - 1))\n", + " elif method == 'exp_range':\n", + " delta *= (gamma**epoch)\n", + " else:\n", + " raise ValueError('unexpected method: %s' % method)\n", + " \n", + " return base_lr + delta\n", + " \n", + " return scheduler" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "k-CjYin0toWa" + }, + "source": [ + "def cosine(t_max, eta_min=0):\n", + " \n", + " def scheduler(epoch, base_lr):\n", + " t = epoch % t_max\n", + " return eta_min + (base_lr - eta_min)*(1 + math.cos(math.pi*t/t_max))/2\n", + " \n", + " return scheduler" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oB_zTLW-wwdM" + }, + "source": [ + "To understand how the created functions work, and to check the correctness of our implementation, let's create a couple of plots visualizing learning rates changes depending on the number of epoch:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Dl-TWx4OwwdN" + }, + "source": [ + "def plot_lr(schedule):\n", + " ts = list(range(1000))\n", + " y = [schedule(t, 0.001) for t in ts]\n", + " plt.plot(ts, y)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "wCfhKAoMwwdN", + "outputId": "52bd67ed-e05a-4a07-9747-c8d68b1eae5a" + }, + "source": [ + "plot_lr(triangular(250, 0.005))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzda2BU13no/f/aM+KiC4KRkGSwwEbC2AiMjAbQzUaXUZIGtyU9rtvYcRqgdfvGIUfmvDlxoW/cNuWUE2JwjEid06qkqXlTUjeQxrm4GoTA0iCQABkLjI0MGMsICzRCFyQkzex1PmwskCXQbUZ7Luv3xRaz9t7Pmj0zz8zaaz9LSCkliqIoinKTZnYAiqIoSmBRiUFRFEUZQCUGRVEUZQCVGBRFUZQBVGJQFEVRBlCJQVEURRnAanYAvnLp0qUxbRcfH8/Vq1d9HE1gU30OD6rP4WE8fZ41a9aQ/65+MSiKoigDqMSgKIqiDKASg6IoijKASgyKoijKACoxKIqiKAOMaFZSXV0du3btQtd1CgsLWb169YDH+/r6KCkp4dy5c8TExFBcXExCQgIAe/fupby8HE3TWLNmDenp6QA899xzTJkyBU3TsFgsbNmyBYDOzk62b9/OlStXmDlzJs8//zzR0dG+7LOiKIpyF8P+YtB1ndLSUjZu3Mj27dupqqqisbFxQJvy8nKioqLYsWMHq1atYvfu3QA0NjbicrnYtm0bmzZtorS0FF3X+7d78cUX2bp1a39SANi3bx+LFy/mlVdeYfHixezbt89XfVUURVFGYNjE0NDQQFJSEomJiVitVrKzs6mpqRnQpra2lry8PAAyMzOpr69HSklNTQ3Z2dlERESQkJBAUlISDQ0Ndz1eTU0NK1euBGDlypWDjqWYR3a0oVftR1VqV/xBSmm8vjrazA4l7A07lOR2u4mLi+v/Oy4ujrNnz96xjcViITIyko6ODtxuN/Pnz+9vZ7PZcLvd/X9v3rwZgKKiIhwOBwBtbW3MmDEDgOnTp9PWNvSLxOl04nQ6AdiyZQvx8fHD93YIVqt1zNsGq7H2uX3fT+j+1etMn7+ASQvT/RCZ/6jzHPh6T9fR+uMfMOWLTzDtzzaMaR/B1mdf8EefTbvz+bvf/S42m422tjb+7u/+jlmzZrFw4cIBbYQQCCGG3N7hcPQnE2DMd/6pOyVHRvb1oh/4LQDXfvUfaAn3+iM0v1HnOfDpv/oPALorfkvP43+MiJg06n0EW599wZQ7n202Gy0tLf1/t7S0YLPZ7tjG6/XS1dVFTEzMoG3dbnf/tp/+NzY2lmXLlvUPMcXGxtLa2gpAa2sr06ZNG3EnFf+RdUegqxNmzUHWViK7u8wOSQkhsrsLWVsJs+ZAVyfyRLXZIYW1YRNDSkoKTU1NNDc34/F4cLlc2O32AW0yMjKoqKgAoLq6mrS0NIQQ2O12XC4XfX19NDc309TURGpqKjdu3KC7uxuAGzducPLkSebMmQOA3W7n4MGDABw8eJBly5b5sr/KGMnKMrDNRPvqN6C3B1nzltkhKSFE1rwFvT3G68s203i9KaYZdijJYrGwdu1aNm/ejK7r5Ofnk5yczJ49e0hJScFut1NQUEBJSQnr168nOjqa4uJiAJKTk8nKymLDhg1omsa6devQNI22tja+//3vA8YvjNzc3P5prKtXr2b79u2Ul5f3T1dVzCVbmuHdtxGP/xHMWwD3JCOrnPDY580OTQkRssoJ9yTDvAWInELkG3uQVz9BxCeaHVpYEjJEppio6qojN9o+6//5U+Qb/4b29/+IiEtA/699yH//Z7S/KUHMmuPHSH1HnefAJS9dRH/xG4g/XIv2udXIlmb0v/wzxON/hPZ7T41qX8HSZ19S1VWVCSd1HenaDw8tQcQZNy2KrHywWNTPfcUnZGUZWCzG6wqM19lDS5BV+5G61+TowpNKDMrdnTkJLc2InFszwERMLCxZjqyuQHr6TAxOCXbS04esroAly43X1U0ixwHuK8brT5lwKjEodyUryyAyGvFI5oB/13KLoKMNTqobEJVxOFkDHW3G6+k24pFMiIxGVjpNCiy8qcSg3JG83oE8UY3IzBs8pzztEZgeh67euMo46JVOmB5nvJ5uIyImITLzkCcOIzvbTYoufKnEoNyRPHIQPH0DhpE+JTQLIrsA6o8jW1uG2FpR7k62tkD9cUR2AUKzDHpc5DjA40EeOWRCdOFNJQbljmRlGcxJQcyZN+TjIscB8ubFaUUZJenaD1If8osHYLzu5qQgK8tUfa4JphKDMiT54Qfw0XnEZ8Z+bycS7oEFi5FVTuRtVXMVZThSSuPehQWLjdfRHYjcImg8DxfPTWB0ikoMypBkVRlYIxDLH7trO5HjgCuX4ezpCYpMCQnvn4Irl+/4a+FTYvljYI1QU6MnmEoMyiCytwd55CBiaTYi6u6LJIml2TA1Ur1xlVGRlWUwNdJ4/dyFiIpGLM1GHjmI7O2ZoOgUlRiUQeSJaui6jsi9+7c5ADF5MmLZY8jjVciu6xMQnRLsZNd15PEqxLLHEJMnD9te5Dqg+7oqrDeBVGJQBpFVTohPhAWLR9Re5BZBb68qrKeMiFEwr/eu168GWLAY4hPVr9IJpBKDMoC8ctkomJdTiNBG+PK4LxVmz1VvXGVEZGUZzJ5rvG5GQGgaIqcQzpw0Xp+K36nEoAwgXeUgBCKrcMTbCCGMn/sXziIbL/gvOCXoycYLcOEsItdxx0W4hiKyCkEINTV6gqjEoPSTuhfpcsLCdETczFFtK1bkg8VqDEMpyh3IKidYrMbrZRRE3ExYmK4K600QlRiUW06/De6rg+rWjISImYZIX4GsPoDsU4X1lMFkXx+y+gAifQUiZvQrM2q5RdB6FU7X+SE65XYqMSj9ZJUTomNgyYoxbS9yHdDZASeP+jgyJSScPAqdHSOa7TakJSsgOkYV1psAw67gBlBXV8euXbvQdZ3CwkJWr1494PG+vj5KSko4d+4cMTExFBcXk5Bg1O7fu3cv5eXlaJrGmjVr+ldqA9B1nRdeeAGbzcYLL7wAwM6dOzl9+jSRkZEAPPfcc9x3332+6KtyF7KzHVlXjVj5O4iIiLHtZGE62OLRK8uwZOT4NkAl6OmVZWCLN14nYyAiIhAr8pAVv0F2tI/pV4cyMsP+YtB1ndLSUjZu3Mj27dupqqqisbFxQJvy8nKioqLYsWMHq1atYvfu3QA0NjbicrnYtm0bmzZtorS0FP220gm//vWvmT179qBjPvPMM2zdupWtW7eqpDBBZHUFeDwjn0I4BKOwXiGcOoF0X/FdcErQk+4rcOoEIrtwyIJ5IyVyi8DrQR6p8F1wyiDDJoaGhgaSkpJITEzEarWSnZ1NTc3AGvy1tbXk5eUBkJmZSX19PVJKampqyM7OJiIigoSEBJKSkmhoaACgpaWF48ePU1g48tkvin9IKY0phHNTEffeN659iexCkNKY3aQoN0lXOUhpvD7GQdx7H8xNVYX1/GzYoSS3201cXFz/33FxcZw9e/aObSwWC5GRkXR0dOB2u5k/f35/O5vNhtvtBuDHP/4xX/nKV+ju7h50zJ/+9Ke8/vrrLFq0iKeffpqIIYY2nE4nTqcx1rhlyxbi4+NH0t9BrFbrmLcNVp/tc1/Du7g//pCYP/8WkeN9LuLjaV2cgbf6AHFf/X9Gfi+En6nzbB6p67RUH8CyOIMZDy0a9/66vvAlOn60lenXrhAxf+GAxwKlzxPJH30e0TUGXzt27BixsbHMmzePU6dODXjsqaeeYvr06Xg8Hn70ox/xi1/8gieeeGLQPhwOBw7HrYtYY10MWy0eDvob/w4Rk7i+cCldPngu9OUrkaXbuFp1APHQknHvzxfUeTaPPHMS/ZNL6I//sU/ikQuXQsQkWn/1OtpXvj7gsUDp80QaT59nzZo15L8P+3XOZrPR0nJrIZaWlhZsNtsd23i9Xrq6uoiJiRm0rdvtxmaz8d5771FbW8tzzz3Hyy+/TH19Pa+88goAM2bMQAhBREQE+fn5/UNPin/Inh7k0UOIjGxEZJRP9imWZsHUKHVPgwJ8WjAvynhd+ICIjEJkZCOPHkL2qMJ6/jBsYkhJSaGpqYnm5mY8Hg8ulwu73T6gTUZGBhUVFQBUV1eTlpaGEAK73Y7L5aKvr4/m5maamppITU3lqaee4tVXX2Xnzp0UFxezaNEivvnNbwLQ2toK0H+NIjk52cddVm4nT7igu2tcF50/S0yajFixEnn8MLKr02f7VYKP7OpEHj+MWLESMWn4gnkjJXKLoLsLedzls30qtww7lGSxWFi7di2bN29G13Xy8/NJTk5mz549pKSkYLfbKSgooKSkhPXr1xMdHU1xcTEAycnJZGVlsWHDBjRNY926dWjDjDm/8sortLcba7zOnTuXZ5991gfdVO5EVjphZhLMT/PpfkWuA1nxa+PXSN4XfbpvJXjIo4egr3fs9y7cyQOLYGaS8as0a3R3USvDEzJELu1funRpTNuF85ikbG5C3/TniNVfQVv1pE+PIaVE/9tisFiw/NU2n+57LML5PJvJ+3cbwOtF+87Lo6qNNBL6r36G3Pca2uYf9a8CFwh9nmimXGNQQpes2g9CQ2QV+HzfRmG9IviwAfnReZ/vXwl88qPz8GEDIrfI50kBMF63QlPXsvxAJYYwZRTM2w9pjyBs/pneJ1Y8BlZVWC9cySonWK3G68APhC0e0h5BulRhPV9TiSFcnaqDay1jKpg3UiJ6GuKRLGR1hSqsF2aMgnkViEeyENH+K12h5RbBNTecOuG3Y4QjlRjClF5ZBtHTYMkyvx5H5DrgegeyTi3LGE5k3RG4Po6CeSO1ZBlETzNez4rPqMQQhvS2Vnj7KCIzH2EdY8G8kXpwCdhmqoqYYUZWloFtpnH+/UhYIxCZ+fD2UWRHm1+PFU5UYghD3QffBO/4CuaNVP+yjO/WIVua/X48xXyypRnerRvd8rDjYBTW8yIPH/D7scKFSgxhRkpJt/OXcP8DiNlzJuSYnxZOk1VqWcZw8GkBxfEWzBspMXsO3P+AKqznQyoxhJsLZ/F+dN7/Y7+3EfGJ8ODDN2eP6MNvoAQtqevGbKQHHzbO+wQRuQ5o+gjP2dMTdsxQphJDmJGVZTB5CmKZf6YQ3onILYKWZjhzckKPq0ywMyehpXlChilvJ5Y9BpMmG7+GlXFTiSGMfFowb0p2AWJq5IQeWzySCZHRRmJSQpasckJktHG+J5CYGonIyOFGpRPZc2NCjx2KVGIII/JYFdzoZmrhqgk/toiYZBTWO1GNvN4x4cdX/E9ev61gXsSkCT++yHUgu7uM17kyLioxhBFZVQYJ9xAxxjV3x0vkFoGnD3nkoCnHV/xLHqkAT9+EDyP1m5+G5Z5k9avUB1RiCBPyk0vw/ilEjsMvdWtGQsyZB3PmqRIZIUpWOWHOPOM8m0AIYfwaPnsaefljU2IIFSoxhAlZ5TQK5mX7vmDeaIjcIrh4DnnxA1PjUHxLXvwALp4z79fCTVPyf8corOdSXz7GQyWGMCC9XmNu+eIMxPS44TfwI7F8JVgj1M/9ECMry8AaYZxfE1lsM2FxBtJ1AOlVhfXGSiWGcHDqOLS50XIm7t6FOxFR0YilWcgjB5F9vWaHo/iA7OtFHjmIWJqFiIo2Oxzjdd7mhvrjZocStIZdwQ2grq6OXbt2oes6hYWFrF69esDjfX19lJSUcO7cOWJiYiguLiYhIQGAvXv3Ul5ejqZprFmzhvT0Wxc+dV3nhRdewGaz8cILLwDQ3NzMyy+/TEdHB/PmzWP9+vVYrSMKU7kDvbIMYmLhYf8WzBspkVtkrNd7cwaLEtzk8cPQdd30YaR+Dy+DmFj0yjIsfi4SGaqG/cWg6zqlpaVs3LiR7du3U1VVRWNj44A25eXlREVFsWPHDlatWsXu3bsBaGxsxOVysW3bNjZt2kRpaSn6bXe+/vrXv2b27NkD9vXaa6+xatUqduzYQVRUFOXl5b7oZ9iS7a1wsgaRVYAIlAS7YDHEJaiL0CFCVjkhLsE4rwFAWK3GIj7v1Bivf2XUhk0MDQ0NJCUlkZiYiNVqJTs7m5qamgFtamtrycvLAyAzM5P6+nqklNTU1JCdnU1ERAQJCQkkJSXR0NAAQEtLC8ePH6ew8FY9FSklp06dIjPTuDkmLy9v0LGU0ZHVFeD1TmgJjOEYhfUc8O7byKufmB2OMg7y6ifw7tvGbLcJKJg3UiLXcbOwXoXZoQSlYb9Cut1u4uJuXbCMi4vj7Nmzd2xjsViIjIyko6MDt9vN/Pnz+9vZbDbcbjcAP/7xj/nKV75Cd3d3/+MdHR1ERkZisVgGtf8sp9OJ02l849yyZQvx8WNbhcxqtY5520AnpaTl8AG0BYuwLX6k/98Doc/ex5/g6i9/ytQTh4n+8p/6/XiB0OeJNhF97izbx3UhiHv8CSwB8Pz29zk+HveCReiHy4l76k9Nm6I9Efxxnk0ZWzh27BixsbHMmzePU6dOjWkfDocDh+PWt+CxLoYdyouHyw/OoDdeQP/qNwb0MSD6LKzwUDrXnf9Jd+HvIjSLXw8XEH2eYP7us9S96M7/hIfSaRVWCIDn9/Y+6yvykD8p4erRKkTKgyZH5j/jOc+zZs0a8t+H/e1ns9loaWnp/7ulpQWbzXbHNl6vl66uLmJiYgZt63a7sdlsvPfee9TW1vLcc8/x8ssvU19fzyuvvEJMTAxdXV14b04z+7S9MjayynmzYF6u2aEMSeQWgfsqvKsK6wWld0+C+2rgXHT+DLEsFyZPUVOjx2DYxJCSkkJTUxPNzc14PB5cLhd2u31Am4yMDCoqKgCorq4mLS0NIQR2ux2Xy0VfXx/Nzc00NTWRmprKU089xauvvsrOnTspLi5m0aJFfPOb30QIQVpaGtXVxjKQFRUVg46ljIy80Y08+hbCnoOYMrEF80ZKpK+AqBj1xg1SsrIMomKM8xiAxJRIhD0HWVOJvNE9/AZKv2ETg8ViYe3atWzevJnnn3+erKwskpOT2bNnD7W1tQAUFBTQ2dnJ+vXreeONN3j66acBSE5OJisriw0bNrB582bWrVuHNswFqqeffpo33niD9evX09nZSUGBuXfqBit5zAU93QH7bQ5AREQgMvOQddXIznazw1FGQXa2I+uqEZl5iAg/Lw87DiK3CHq6VWG9URIyRJY8unTp0pi2C9WxZ+//fgE629D+9oeDLrwFUp9l43n0v/nviD/+M7TC3/XbcQKpzxPFn33W9/8S+W//iPbiDxD33u+XY4zFZ/sspUT/ztchehqWb/9vEyPzH1OuMSjBR15uhIbTphbMGylx7/0wN1UtyxhEpJTGMNLc1IBKCkMRQhhToxveRTY1Dr+BAqjEEJJk1X7QNOMmnyAgch3QeAFUYb3gcPEDaLwQUPfG3I3IKgBNUzdUjoJKDCFGer3Iw+Ww2I6InWF2OCMilj8GEZPUReggISvLIGKScd6CgIidAYvtyMPlSI/H7HCCgkoMoeadWmhrRQvgi86fJSI/Lax3CNnbY3Y4yl3I3h7kkUNGwbxI8wvmjZSWWwTt16D+mNmhBAWVGEKMXuWEadNhUYbZoYyKyC2C7utGQTYlYMnjh6E7gArmjdSiDJg23SgoqQxLJYYQItsCsGDeSD2wCGYmqeGkACcry2BmknG+gsitwnq1yGtDl9lRblGJIYTIw+Wg60FzUfB2QtMQ2YXw3jvI5iazw1GGIK9chvfeQWQXBlTBvJESuQ7QdWT1AbNDCXjBd3aVIUkpjVkXqQ8hku41O5wxEdmFN5dl3G92KMoQbi0PWzh84wAkku6F1IeQlU41NXoYKjGEig/ehcsfB9/Y722ELR7SHkG6ypG6WpYxkEj95vKwaY8Y5ylIidwi+ORjaHjX7FACmkoMIUJWlsHkqYiMHLNDGRct1wGtV+FUndmhKLc7XQetV43zE8RERg5MnoqsUtey7kYlhhAgb3Qha6sQy3IRU6aaHc74LFkO0dPQ1Rs3oOiVZRA9zTg/QUxMmYpYlousrULe6DI7nIClEkMIkDWV0HMjqIeRPiWsEYjMfKg7iuxoMzscBYzzUHcUkZmPsAZuwbyRMgrr3TDeN8qQVGIIAbLKCfckw7wFZofiE8ayjB5jWVLFdPJIBXg9QTnbbUjzFsA9yapExl2oxBDkZNNH8MGZoCiYN1Ji9ly4/wFVWC8AGAXznHD/A8Z5CQH9hfU+OGO8f5RBVGIIcrLSCRYLIivP7FB8SuQ44NJFuHB2+MaK/1xogI8/NM5HCBFZ+WCxGO8fZZAR3R5bV1fHrl270HWdwsJCVq9ePeDxvr4+SkpKOHfuHDExMRQXF5OQkADA3r17KS8vR9M01qxZQ3p6Or29vbz44ot4PB68Xi+ZmZk8+eSTAOzcuZPTp08TGWmsOvbcc89x3333+bDLoUN6PDcL5i1DTAuOgnkjJZY9ivzZPyErnYj7HzA7nLAlK8tg0iTEskfNDsWnxLTp8PAyo7Del54JvkoBfjbss6HrOqWlpfzVX/0VcXFx/OVf/iV2u5177711E1V5eTlRUVHs2LGDqqoqdu/ezfPPP09jYyMul4tt27bR2trKd7/7XX7wgx8QERHBiy++yJQpU/B4PHznO98hPT2dBx4wPgCeeeYZMjMz/dfrUPFOLXS0BVXBvJESkVGIjBxkzSHkk+sQkyebHVLYkT09yJpDiIwcRGSU2eH4nJZThH6iGk7WwNIss8MJKMMOJTU0NJCUlERiYiJWq5Xs7GxqamoGtKmtrSUvLw+AzMxM6uvrkVJSU1NDdnY2ERERJCQkkJSURENDA0IIpkyZAoDX68Xr9YbM+PhE0ivLINYGi5aaHYpfiJwi6O5SyzKaRB53QXeXcR5C0aKlEGszCk8qAwz7i8HtdhMXF9f/d1xcHGfPnr1jG4vFQmRkJB0dHbjdbubPn9/fzmaz4XYbBax0Xefb3/42ly9f5vOf//yAdj/96U95/fXXWbRoEU8//TQRQ6wp63Q6cTqNE7plyxbi48d2N6bVah3ztmbyuq9wtf4YkaufJiYxcVTbBkufZdxKWu65F+3oQWy/9+S49hUsffal8fbZfaQC/Z57icteGTRf3Ebb547CVXTt280MTWKxzfRjZP7jj9e2aQNrmqaxdetWrl+/zve//30uXrzInDlzeOqpp5g+fToej4cf/ehH/OIXv+CJJ54YtL3D4cDhuHVBbKxrngbrWsD6b/4DdJ0bS3PoGWX8wdRnPTMf795/5crpk4iEodenHYlg6rOvjKfPsvkS+qkTiC89Q0tLi48j85/R9lkuzYGf/ystb7yO9sU/9GNk/mPKms82m23AC6OlpQWbzXbHNl6vl66uLmJiYgZt63a7B20bFRVFWloadXVGCYQZM2YghCAiIoL8/HwaGhpG2MXw0T+F8IE0ROLYPyyDgcgqMArrqdkjE0pW7TcK5gXJ8rBjJRJnwQNpyCpVWO92wyaGlJQUmpqaaG5uxuPx4HK5sNvtA9pkZGRQUVEBQHV1NWlpaQghsNvtuFwu+vr6aG5upqmpidTUVNrb27l+/ToAvb29nDx5ktmzZwPQ2toK0H+NIjk52Zf9DQ1nT0PzpZCbQjgUMSMOFi01Zo94VWG9iSC9XqPC7aKlxvMf4kSOA5qb4Owps0MJGMMOJVksFtauXcvmzZvRdZ38/HySk5PZs2cPKSkp2O12CgoKKCkpYf369URHR1NcXAxAcnIyWVlZbNiwAU3TWLduHZqm0drays6dO9F1HSklWVlZZGQYK4698sortLe3AzB37lyeffZZP3Y/OMnKMpgS/AXzRkrLLUL/h7+HU8fh4WVmhxP6Th2Ha260L/+52ZFMCJGRg/zp/0FWliGCbAEifxEyRH4/Xbp0aUzbBdvYs+zuQv9//wSRmYf2zHNj2kfQ9dnTh/4/10LqQ1i+vnFM+wi2PvvCWPvs/Ye/h7On0b73z0FXG2msfdb/dSey+gDa93+CmBrph8j8x5RrDEpgkTVvQW9PWAwjfUpYI4w7VU/WINuvmR1OSJPt1+Dto4is0CiYN1IixwG9vciaQ2aHEhBUYggysrIMZs2BMLsbWOQ4wOtVyzL6mayuAK83rL54AMb7adYcNcnhJpUYgoj8+CKcfx+RWxQ088p9RcyaA/MWqGUZ/ciY7VYG8xYYz3cYEUIY5bjPv2+8z8KcSgxBRFaVgcWKyMwzOxRTiNwiaPoIzr1ndiih6dx70PRRSKzrMRYiMw8sViM5hjmVGIKE9PQhDx+AJcsRMbFmh2MKYc+FSZNVHX0/kVVOmDTZeJ7DkIiJhSXLkdUHkJ4+s8MxlUoMweJkDXS2B/2au+MhpkYi7LnIo28he26YHU5IkT03kDVvIey5QTcrx5e03CLobIe3a4ZvHMJUYggSeqUTpsdB2iNmh2IqY1nGbmStKqznS7K2Cm50h+0wUr+0dJgeZxSoDGMqMQQB2doC9ccR2YUIzWJ2OOZKfQgSZ6txYB+TVWWQONt4fsOY0CyI7EI4dQLpDq/7Xm6nEkMQkK79IHVETqHZoZiuf1nGhtPIyx+bHU5IkJc/hrOnQ2p52PEQOYUgdWMRrDClEkOAk7puXBRcsBiRcI/Z4QQEkZUPmqYuQvuIrHKCphnPq2K8zxYsNgrr6brZ4ZhCJYZAd/YUXLmMCOOLzp8lpttgsV0V1vMB6fXeXB7WbjyvCoDxfrtyOWwL66nEEOBkpROmRiIeyTY7lICi5TqgrRXqj5kdSnCrPwZtrWE9220o4pFsmBoZtteyVGIIYLLrOvJ4FWL5Y2rN489aZIdp08N+9sh46ZVOmDbdeD6VfmLyZMTyx5DHXMiu62aHM+FUYghgRsG8XjWFcAjCar1VWK+t1exwgpJsb4V3am4WzDNtMceAJXKLoK8XeTT8CuupxBDAZGUZzJ4Lc1PNDiUgiZwi0HVVWG+M5OEDNwvmqS8eQ5qbCrPnhuVwkkoMAUo2XoALZ8OyYN5IiXvuhZQHkZVlqrDeKPUvD5vyoPE8KoP0F9b7sAHZeN7scCbUiH4/1tXVsWvXLnRdp7CwkNWrVw94vK+vj5KSEs6dO0dMTAzFxcUkJNweGwgAACAASURBVCQAsHfvXsrLy9E0jTVr1pCenk5vby8vvvgiHo8Hr9dLZmYmTz75JADNzc28/PLLdHR0MG/ePNavX481DH/myionWMO3YN5Iidwi5L/sgA/OhP3NWaPywRm43Ij4k/VmRxLQRGYe8j9+jKx0Iv74z8wOZ8IM+4tB13VKS0vZuHEj27dvp6qqisbGxgFtysvLiYqKYseOHaxatYrdu3cD0NjYiMvlYtu2bWzatInS0lJ0XSciIoIXX3yRrVu38r3vfY+6ujref/99AF577TVWrVrFjh07iIqKorw8/G4ykX19yOoDiCUrENHTzA4noAl7LkyeEpY/98dDVpbB5ClhWzBvpET0NMSSFcjqCmRf+BTWGzYxNDQ0kJSURGJiIlarlezsbGpqBhaYqq2tJS8vD4DMzEzq6+uRUlJTU0N2djYREREkJCSQlJREQ0MDQgimTJkCgNfrxev1IoRASsmpU6fIzMwEIC8vb9CxwsLbR6CzQ110HgExZapRWK+2Enmjy+xwgoK80Y2srTQK5k2ZanY4AU/kFsH1DuN9GSaGHaNxu93ExcX1/x0XF8fZs2fv2MZisRAZGUlHRwdut5v58+f3t7PZbLjdbsD4JfLtb3+by5cv8/nPf5758+fT3t5OZGQkFotlUPvPcjqdOJ3Gna9btmwhPj5+NP3uZ7Vax7ytv7QePYQnPpH4RwsRFt/XRgrEPo9H7+N/SGuVk+gzJ5nqeHzINqHW55G4U5+7nW/Q3nOD6Y//IZNC7Dnxx3mWjxZydfc/YD16kBlfWD38BhPMH302bfBe0zS2bt3K9evX+f73v8/FixeZPn36iLd3OBw4HLduyhnrYtiBtki8dF9BrzuCWPUkLa3+mYYZaH0eLxmXBEn30v7bn3M9PXPINqHW55G4U5+9v/05JN1LW1wSIsSeE3+dZ5mZR++vfsaV995FxM30+f7HYzx9njVr1pD/PuxQks1mo6Wlpf/vlpYWbDbbHdt4vV66urqIiYkZtK3b7R60bVRUFGlpadTV1RETE0NXVxfem2UOhmof6oyCedKo8KiMiDF7xAEfnEE2fWR2OAFNNjXCB2cQuapg3miI7EKQEnl4v9mhTIhhE0NKSgpNTU00Nzfj8XhwuVzY7QPvkszIyKCiogKA6upq0tLSEEJgt9txuVz09fXR3NxMU1MTqamptLe3c/26cTdhb28vJ0+eZPbs2QghSEtLo7q6GoCKiopBxwplRsG8/fDgw4iZSWaHE1RUYb2RkVVlqmDeGIiZSfDgw8aa42FQWG/YoSSLxcLatWvZvHkzuq6Tn59PcnIye/bsISUlBbvdTkFBASUlJaxfv57o6GiKi4sBSE5OJisriw0bNqBpGuvWrUPTNFpbW9m5cye6riOlJCsri4yMDACefvppXn75Zf7t3/6N+++/n4KCAv8+A4HkvXfg6ieI1V8xO5KgI6bNgIeXIV3lyNXPqDt5hyA9HuOmtoeXGc+XMioitwj5Ty8Z79OHlpgdjl8JGSJ3Bl26dGlM2wXS2LP+jy8h62vRtv4YMcl/tZECqc++JN8+il7yd2hf34h4ZOC1hlDt8918ts+yrhp95/9C+8ZfIZYsNzEy//HneZa9Pejf+hpikR3tz/6HX44xFqZcY1AmhrzeiTzuQixf6dekENIWZUDsDHQ1nDQkvdIJsTOM50kZNTFpMmL5SuRxF/J6p9nh+JVKDAFCHj0Enj5178I4CIsFkVUA79Qirw09zTlcyWtueKcWkVXglynQ4ULkFoGnL+QL66nEECBklROS70fMTTE7lKAmchxGYb0wXpZxKPLwAdB14/lRxkzMTYHk+0P+TnuVGAKA/Og8fNigqlz6gEiaDfMXGrNHQuPy2bhJKY0vHvMXGs+PMi4ipwgufoC8eM7sUPxGJYYAICvLwBqByFxpdighQeQUQfMlOHva7FACQ8O78MnH6ouHj4jMlWCNCOmp0SoxmEz29SKrKxCPZCKiYswOJyQIew5MmRrSb9zRkJVlMGWq8bwo4yaiYhCPZN4srNdrdjh+oRKDyWTdEejqNO7cVXxCTJ6CWPaoUVivO7wL68nuLqNg3rJHEZOnmB1OyBC5DujqRJ6oNjsUv1CJwWSysgxsM+HB0L5hZqKJHAf09hjLo4YxWVsJvT3qorOvPbgE4hJC9lepSgwmki3N8O7biJxChKZOhU/NWwD3JIfsG3ekZGUZ3JNsPB+KzwhNM+onvfu28T4OMerTyESyyijIpb7N+V5/Yb1z7yEvXTQ7HFN4PjoP595TBfP8ROQYhS5D8cuHSgwmkbpuVFJ9aAkiLsHscEKSyMwHiyXk55zfSff+N8BiMZ4HxedEXAI8tARZtT/kCuupxGCWMyehpVn9WvAjMW06LFkedssyAkhPH90HfgNLlhvPg+IXIscB7itw5m2zQ/EplRhMIivLIDJ6ULE3xbe0HAd0tNFTW2V2KBPrZC2y/ZrRf8VvxCOZEBmNrAyt4SSVGEwgr3cgT1QjMvMQEZPMDie0pS2F6Ta69//S7EgmlF5ZhmaLN/qv+I2ImITIzEOeqEZe7zA7HJ9RicEE8shBo2Ce+jbnd8JiQWQX0nviCLK1ZfgNQoBsbYH640zN/6IqmDcBRI7DKKxXfdDsUHxGJQYTyMoymJOCmDPP7FDCgsgpNArrucJjWUZ5uBykzpTCx80OJSyIOfNgToqxOl6IGNEyV3V1dezatQtd1yksLGT16tUDHu/r66OkpIRz584RExNDcXExCQnGTJu9e/dSXl6OpmmsWbOG9PR0rl69ys6dO7l27RpCCBwOB1/84hcB+NnPfsb+/fuZNm0aAF/+8pdZujR0fg7Lix/AR+cRT/2F2aGEDZEwi4i0R+irciJ/54mQvmekv2DeA4uw3nMvhNniRGYRuUXI//9V5IcfhESF5GHfIbquU1paysaNG9m+fTtVVVU0NjYOaFNeXk5UVBQ7duxg1apV7N69G4DGxkZcLhfbtm1j06ZNlJaWous6FouFZ555hu3bt7N582befPPNAftctWoVW7duZevWrSGVFOC2gnnLHzM7lLAy1fE4XLkc+oX13j8FzU1qXY8JJpY/ZhTWC5Gp0cMmhoaGBpKSkkhMTMRqtZKdnU1NTc2ANrW1teTl5QGQmZlJfX09UkpqamrIzs4mIiKChIQEkpKSaGhoYMaMGcybZwyjTJ06ldmzZ+N2h/7CKrK3B3nkIGJpNiIq2uxwwsqUrHyYGhkyb9w7kVVlMDUSsTTb7FDCioiKRizNRh49iOztMTuccRt2KMntdhMXF9f/d1xcHGfPnr1jG4vFQmRkJB0dHbjdbubPn9/fzmazDUoAzc3NnD9/ntTU1P5/e/PNNzl06BDz5s3jq1/9KtHRgz9EnU4nTqcxRWzLli3Ex8ePpL+DWK3WMW87Wt1v/RftXdeJXfXfmDxBxxzKRPY5UFitVqY++jm6K36D7Rt/iRaCiVm/3smVYy6m5v0O02bPDtvzbFafe1f9N1qPHiS64RRTH/vchB3XH30e0TUGf7lx4wYvvfQSX/va14iMjATgc5/7HE888QQAe/bs4Sc/+Qlf//rXB23rcDhwOG7N6hnrYtgTuUi89zd7IT6R9qQ5CBPHfieyz4EiPj6eHnsu/Nc+rv52H9rKL5gdks/ph34LvT302HO5evVq2J5ns/osk+YY7+/f/JzrCyduCHw8fZ41a9aQ/z7sUJLNZqOl5dY0v5aWFmw22x3beL1eurq6iImJGbSt2+3u39bj8fDSSy/x6KOPsmLFiv4206dPR9M0NE2jsLCQDz74YBTdDFzyymVVMM9s982H2XNDdjhJVjph9lyjn8qEE5pmzIA7c9J4vwexYT+hUlJSaGpqorm5GY/Hg8vlwm63D2iTkZFBRUUFANXV1aSlpSGEwG6343K56Ovro7m5maamJlJTU5FS8uqrrzJ79mwef3zglLrW1tb+/z969CjJyck+6Kb5pKschEBkFZodStjqL6x34Syy8YLZ4fiU/PhDOP++KphnMpFdCEIE/dToYYeSLBYLa9euZfPmzei6Tn5+PsnJyezZs4eUlBTsdjsFBQWUlJSwfv16oqOjKS4uBiA5OZmsrCw2bNiApmmsW7cOTdM4c+YMhw4dYs6cOXzrW98Cbk1Lfe2117hw4QJCCGbOnMmzzz7r32dgAkjdi3Q5YWE6Im6m2eGENbEiH/n6vyCrnIg/+lOzw/EZWekEixWxQhXMM5OwzYSF6UjXfuTv/jFCC84bDIUMkRXTL126NKbtJmJMUtYfR//BX6P9+f9E2HP9eqyRCPexZ++rW+C9d9C+92NERITJkY2f9PShf2sNLFiE5S9e6P/3cD/PZpG1leg/+h7af/9rxCL/X2sw5RqDMn6yygnRMbBkxfCNFb/TcougswNOHjU7FN94+yh0thv9Usy3ZAVExwT1tSyVGPxMdrYj66oRK/JC4ttpSFiYDjPi0YP4jXs7vdIJM+KNfimmExERiBV5yLojyI52s8MZE5UY/ExWV4DHo+5EDSBCsyCyC+DUCaT7itnhjIt0X4VTJxDZBUE7nh2KRG4ReD3IIxVmhzImKjH4kZTS+Dk5NxVx731mh6PcRuQ4QEpjtlgQk679IHVVqTfAiHvvg7mpyMoygvEyrkoM/vRhA3z8ofq1EIDEzCRYsBhZ5QzaZRn7l4ddsNjojxJQRG4RfPyh8TkQZFRi8CNZ5YSISapgXoASuUVw9RN47x2zQxmb9+vhymX1xSNAieWPQcSkoLwIrRKDnxgF8w4hMrIRkVFmh6MMQSzNgqlRRgIPQrLKCVOjjH4oAUdERiEyspFHDyF7gquwnkoMfiKPu6D7uvo2F8DEpMmIFY8hjx9GdnWaHc6oyK5O5DEXYsVjiEmTzQ5HuQORWwTdXcgTLrNDGRWVGPxEVjphZhLMTzM7FOUuRG4R9PUijx4yO5RRkUcPQV+v+uIR6B5YBDOTjM+DIKISgx/I5iZ47x1EjkMVzAt0c1Lg3vuC7o0rK51w731G/ErAEkIYM8bee8f4XAgS6lPLD2TVfhAaIqvA7FCUYRiF9YrgwwbkR+fNDmdEZON5+LABkVukCuYFAaOwnmZ8LgQJlRh8zCiYtx/SHkHYwmuRlGAlVqwEqzVoLkLLSidYrUbcSsATM+Jg0VKjsJ7uNTucEVGJwddO1cG1FlW3JoiI6GmI9ExkdQWyr8/scO5K9vUhqysQ6ZmI6Glmh6OMkJbjgGstxudDEFCJwcf0yjKIngZLlpkdijIKIrcIrncg66rNDuWuZN0RuN6hLjoHmyXLICY2aOpzqcTgQ7KjDd4+isjMR1hVwbyg8tDDYJsZ8BehZWUZ2GYa8SpBQ1gjEJl58PZR43MiwI1ozee6ujp27dqFrusUFhayevXqAY/39fVRUlLCuXPniImJobi4mISEBAD27t1LeXk5mqaxZs0a0tPTuXr1Kjt37uTatWsIIXA4HHzxi18EoLOzk+3bt3PlyhVmzpzJ888/T3R0cCzcLqsrwKsK5gUjo7BeIfJXe5AtzYi4BLNDGkS2XIF36xCr/kgVzAtCIqcIWfYLYyiw6PfNDueuhv3FoOs6paWlbNy4ke3bt1NVVUVjY+OANuXl5URFRbFjxw5WrVrF7t27AWhsbMTlcrFt2zY2bdpEaWkpuq5jsVh45pln2L59O5s3b+bNN9/s3+e+fftYvHgxr7zyCosXL2bfvn1+6Lbv9RfMu/8BxOw5ZoejjIHIMZZdDdTZI58uF/lpnEpwEbPnwP0PBEVhvWETQ0NDA0lJSSQmJmK1WsnOzqampmZAm9raWvLy8gDIzMykvr4eKSU1NTVkZ2cTERFBQkICSUlJNDQ0MGPGDObNmwfA1KlTmT17Nm63G4CamhpWrjRmW6xcuXLQsQLWhbNw6aKxprASlER8Ijz48M3ZI4FVWE/qujFr6sGHjTiVoCRyHXDpIpx/3+xQ7mrYoSS3201cXFz/33FxcZw9e/aObSwWC5GRkXR0dOB2u5k/f35/O5vN1p8APtXc3Mz58+dJTU0FoK2tjRkzZgAwffp02tqGHo9zOp04ncZ48JYtW4iPH9vUUKvVOuZtb9f+76V0T55C/Be+hBbgtZF81edgMtI+d//Ol2jf9tdMa/qQyQE0gaDnZC3XWpqZ9idfZ+oIz506z4FH/8KXuPKzf2bysUqmLc/xyT790ecRXWPwlxs3bvDSSy/xta99jcjIyEGPCyHueAOPw+HA4bj17Xysa576Yo1Y2dODfui/EEuzcXd1Q1f3uPbnb4GwLu5EG2mfZeoiiIyi7Vevo82+fwIiGxn9V69DZBSdqYu4PsJzp85zYBJLs+k+9F/0/N7TiMlTxr0/U9Z8ttlstLS09P/d0tKCzWa7Yxuv10tXVxcxMTGDtnW73f3bejweXnrpJR599FFWrLi1FnJsbCytra0AtLa2Mm1a4M/Vlseq4Ea3GkYKASJiEmLFSuSJauT1DrPDAUBe70QeP4xYsRIRMcnscJRxErkOuNGNPBa4hfWGTQwpKSk0NTXR3NyMx+PB5XJht9sHtMnIyKCiogKA6upq0tLSEEJgt9txuVz09fXR3NxMU1MTqampSCl59dVXmT17No8//viAfdntdg4ePAjAwYMHWbYscH7O34msKoOEe1TBvBAhcovA04c8ctDsUACQRw+Cp0/NdgsV89MgYZbxuRGghh1KslgsrF27ls2bN6PrOvn5+SQnJ7Nnzx5SUlKw2+0UFBRQUlLC+vXriY6Opri4GIDk5GSysrLYsGEDmqaxbt06NE3jzJkzHDp0iDlz5vCtb30LgC9/+cssXbqU1atXs337dsrLy/unqwYy+ckleP8U4kvPqLo1IULMSYE584yLvQWPD7+Bn8nKMpgzz4hLCXpGfS4H8uc/QX5yCZE49HCOmYQM9HlTI3Tp0qUxbTfeMUn95z9B/vbnaN8rRUyPG36DABAM47C+Nto+6+VvIH/6f9D+v+2mfiDLix+gf/d5xJefRRtlklLnOXDJay3o/3Md4gt/gPYHXx3Xvky5xqDcmfR6jcXkF2cETVJQRkasyANrhOnLMhoF8yKMeJSQIabHweIMpKsc6Q28wnoqMYzHqePQ5jYKZCkhRURFI5ZmIY8cRPb1mhKD7OtFHjmIWJqFiAqOu/+VkdNyi6DNDfXHzQ5lEJUYxkGvLIOYWHg48C+QK6MnchzQdR15/LApx5cnqqGr04hDCT2L7UZhvQC8CK0SwxjJ9lY4WYPIKkBYTb0dRPGXBx+GuATT1mmQlWUQl2DEoYQcYbUai3mdrDE+TwKISgxjZBTM86p7F0KY0DTj2/q7byOvfjKhx5ZXP4EzJ9XysCFO5DrA60UerjA7lAHUK24MjIJ5Tkh5EHFPstnhKH5kLMsoJrywXn/BvGxVMC+UiXuSIeVBZJUzoArrqcQwFufeg6aP1NhvGBBxM+GhdKTLOWHLMkrdaySih9KN4yshTeQ4oOkj43MlQKjEMAayygmTpyCW5ZodijIBRK4D3Ffh3ZMTc8AzJ8F9RQ1ThgmxLBcmTwmoNcdVYhgleaMbefQthD0HMWVw4T8l9Ij0TIiKmbB7GmSlE6JijOMqIU9MiUTYc5BH30LeCIwCnCoxjJI85oKeblW3JoyICGNZRllXjexs9+uxZGc78sRhRGYeIkItDxsuRG4R9HQbBTkDgEoMoyQryyBpNqQ8ZHYoygQSOQ7wePxeWE8eOQQej7p+FW5SHoKk2QGz5rhKDKMgLzdCw2ljCqEqmBdWRPL9MDfVr8sy9i8POzfVOJ4SNoQQxpeBhtPG54zJVGIYBVm1HzTNuClFCTsi1wGNF+DiB/45wMUPoPG8uugcpkRWAWhaQPxqUIlhhKTXizxcDovtiNgZZoejmEAsfwwiJvntIrSsdELEJOM4StgRsTNgsR15uBzp8Zgai0oMI/VOLbS1GoWvlLAkIj8trHcI2dvj033L3p5bBfMiVcG8cKXlFkH7Nag/Zm4cph49iOhVTpg2HRZlmB2KYiKRWwTdvi+sJ48fhu7rarZbuFtsh9gZRoFOE42o+ltdXR27du1C13UKCwtZvXr1gMf7+vooKSnh3LlzxMTEUFxcTEJCAgB79+6lvLwcTdNYs2YN6enpAPzwhz/k+PHjxMbG8tJLL/Xv62c/+xn79+/vX+v505XdzCTbbhbMK1qtCuaFuwcWQXyiMZyUmeez3coqJ8QnGvtXwpawWBCZ+ciyfci2VtOGrYf9xaDrOqWlpWzcuJHt27dTVVVFY+PAq+bl5eVERUWxY8cOVq1axe7duwFobGzE5XKxbds2Nm3aRGlpKbquA5CXl8fGjRuHPOaqVavYunUrW7duNT0pAMa1BV1XFwWVW4X13nsH2dzkk33KK5dVwTyln8h1gK4bnzsmGfZV2NDQQFJSEomJiVitVrKzs6mpqRnQpra2lry8PAAyMzOpr69HSklNTQ3Z2dlERESQkJBAUlISDQ0NACxcuJDo6MAfS5VSGt/mUh9CJN1rdjhKABDZBUZhPZdvCutJ134QwtivEvZE0r2QutDUwnrDjou43W7i4m4tWxkXF8fZs2fv2MZisRAZGUlHRwdut5v58+f3t7PZbLjd7mGDevPNNzl06BDz5s3jq1/96pAJxOl04nQa07q2bNlCfHz8sPsditVqveu2ve+epPXyx0z7xp8wdYzHCDTD9TkU+bTP8fG0pq/AU11B3Jr1CItlzLuSXi9XDx8gIn0FMx7w7U2T6jwHr+4vrKa95H8Re7WJSQ/dfT0Of/Q54AbMP/e5z/HEE08AsGfPHn7yk5/w9a9/fVA7h8OBw3FraGesi2EPt5C2/qvXYfJUOhcs4XoQLDI+EsGyYLov+brPcsVK9BPVXD3kRCwe+4QEWX8MvaUZzx+u8fk5Uec5eMkFS2DyVK796t/RZs66a9vx9HnWrKH3PexQks1mo6Wlpf/vlpYWbDbbHdt4vV66urqIiYkZtK3b7R607WdNnz4dTdPQNI3CwkI++MBPNxONgLzRhaytRCzLRUyZalocSgB6eDlEx4x7WUZZ6YToGGN/inKTmDIVsSwXWVuFvNE14ccfNjGkpKTQ1NREc3MzHo8Hl8uF3W4f0CYjI4OKigoAqqurSUtLQwiB3W7H5XLR19dHc3MzTU1NpKam3vV4ra23lrg7evQoycnmLYQjayqh54aaQqgMYhTWy4e6o8iOtjHtQ3a0I+uOIDLzVcE8ZRCjsN4N43Nogg07lGSxWFi7di2bN29G13Xy8/NJTk5mz549pKSkYLfbKSgooKSkhPXr1xMdHU1xcTEAycnJZGVlsWHDBjRNY926dWg3Z128/PLLnD59mo6ODv7iL/6CJ598koKCAl577TUuXLiAEIKZM2fy7LPP+vcZuAtZ5YR7kmHeAtNiUAKXyC1COv8TWV2BKPr9UW8vjxwAr0d98VCGNm8B3JNsfA49+rkJPbSQgbSe3DhcunRpTNvdaXxONn2E/p3nEE+sQfv8l8YbXkAJlXHY0fBXn72b/wf09qD99Y5RFVaUUqL/zTchYhKWTS8Nv8EYqPMc/PQ39yJf34X2tzvvuIywKdcYwpWsdILFgsjKMzsUJYCJ3CK4dBEunB2+8e0uNMDHH6pfC8pdiax8sFgmvLCeSgxDkB7PzYJ5yxDTVME85c7Eskdh0qRRv3FlZRlMmmRsryh3IKZNh4eXTXhhPZUYhvJOLXS0qYJ5yrBEZBRiaQ6y5hCyZ2SF9WRPD7LmEGJpDiIyys8RKsFOyymCjjbjc2mijjlhRwoiemUZxNpgkfnlOJTAZxTW6xrxsozyuAu6u9QwkjIyi5ZCrG1CC+upxPAZ8loLvHMMkZ0/rjtalTDyQBrMTDJmj4yArHLCzCRjO0UZhrBYjHIp7xwzPp8mgEoMnyEPHwCpI3LUtzllZPqXZXy/Htl899lxsrkJ3ntHLQ+rjIrIcYDUjc+nCaASw22MNXed8EAaIvHut6Eryu1EdiGI4ZdllFVOEJrRXlFGSCTOggfSkJUTU1hPJYbbnT0NzZeM7KwooyBmxMGipcbsEa93yDZS9xqVVBctNdoryiiInCJovmR8TvmZSgy3kZVlMGUqIiPH7FCUIKTlOuCaG04dH7rBqRNwzW20U5RREhnZMGWq39Ycv51KDDfJm7NKxPLHEJOnmB2OEoweXgYxsXecPaJXlkFMrNFOUUZJTJ6CWP4Y8lgVstu/hfVUYrhJ1rwFvT1qGEkZM2GNQGTmwckaZPu1AY/JjjZ4+ygiMw9hVQXzlLEROQ7o7TE+r/xIJYabZGUZzJoD9z9gdihKEBO5ReD1IqsHzh6Rhw+A16vuXVDG5/4HYNYcvw8nqcQAyI8vwvn3EblFagqhMi5i1hyYt2DA7BFjtlsZzFtgPK4oYySEML5cnH/f+NzyE5UYAFlVBharMQygKOMkchzQ9BGce8/4h/PvQ9NHaphS8QmRmQcWq/G55SdhnxhkX5/xM3/JckRMrNnhKCHAKKw3uf9OaKNg3mRVME/xCRETC0uWIw8fQHr6/HKMsE8MPbVV0NmuphAqPiOmRiIycpBH30K2X0PWvIXIyEFMjTQ7NCVEaLlF0NkOb9f4Zf/DruAGUFdXx65du9B1ncLCQlavXj3g8b6+PkpKSjh37hwxMTEUFxeTkJAAwN69eykvL0fTNNasWUN6ejoAP/zhDzl+/DixsbG89NKthUo6OzvZvn07V65cYebMmTz//PNER0f7qr+DdO//JUyPg7RH/HYMJfyI3CLk4XL0f3oJbnSri86Kb6Wlw/Q49ConfP73fL77YX8x6LpOaWkpGzduZPv27VRVVdHY2DigTXl5OVFRUezYsYNVq1axe/duABobG3G5XGzbto1NmzZRWlqKrusA5OXlsXHjxkHH27dvH4sXL+aVV15h8eLF7Nu3zxf9HJJsbaH3Xe3qiwAACSBJREFUxBFEdiFCUwXzFB+avxASZsG7bxv/nb/Q7IiUECI0i1FWpf443pYrPt//sImhoaGBpKQkEhMTsVqtZGdnU1Mz8OdLbW0teXl5AGRmZlJfX4+UkpqaGrKzs4mIiCAhIYGkpCQaGhoAWLhw4ZC/BGpqali5ciUAK1euHHQsX5Ku/aDriBxVt0bxLWP2iDE8KXJVwTzF90ROIUTH4Gm84PN9DzuU5Ha7iYu7VdclLi6Os2fP3rGNxWIhMjKSjo4O3G438+fP729ns9lwu913PV5bWxszZhirpk2fPp22trYh2zmdTpxO4+Leli1biI+PH64rg3TfOwdP0e8Ss3DxqLcNZlardUzPVzAzo8/6l57ieu8Nor70FFr0tAk9NqjzHPLi45G7fknE5ClM9vHqbiO6xmAWIcQdv2k5HA4cjlsXjMe0GPaSTOILHw+pxcNHItQWTB8J0/r8+1+h50Yv3Jj4Y6vzHB7G0+dZs4auIj3sUJLNZqOl5dbiEC0tLdhstju28Xq9dHV1ERMTM2hbt9s9aNvPio2NpbW1FYDW1lamTZv4b1qKoijhbNjEkJKSQlNTE83NzXg8HlwuF3a7fUCbjIwMKioqAKiuriYtLQ0hBHa7HZfLRV9fH83NzTQ1NZGamnrX49ntdg4ePAjAwYMHWbZMFRxTFEWZSEKOYNWH48eP8y//8i/ouk5+fj5/8Ad/wJ49e0hJScFut9Pb20tJSQnnz58nOjqa4uJiEhMTAfj5z3/OgQMH0DSNr33tazzyiDEt9OWX/2979xfSVP/HAfx9mCktbfOc0MiKmtmFhgkpSlCmRhdREF4IRRde5kqR6GLdRDcRBMtBTraL0PCuixTsoiBMQ0SYfzFXy8xCeKrljsqZU6fb57kQx3OenueHS5/Obzuf15U7Dvx89mZ+zvnu6NcBr9cLRVFgMplQU1ODyspKKIqCpqYmzM7OxnW76h9//O+ds/4NX3rqA/esD9xzfP5tKWlTgyER8GDYPO5ZH7hnfdDkMwbGGGP6woOBMcaYCg8GxhhjKjwYGGOMqSTNh8+MMca2h+6vGGw2m9Yl/Hbcsz5wz/rwX/Ss+8HAGGNMjQcDY4wxFcPdu3fval2E1iwWi9Yl/Hbcsz5wz/qw3T3zh8+MMcZUeCmJMcaYCg8GxhhjKv/XG/X810ZHR9Ha2opoNIqqqipcunRJ65K2bHZ2Fk6nE/Pz8xAEAWfPnsX58+cRDAbR1NSEHz9+qP5rLRGhtbUVIyMjSEtLg9VqTdg12mg0CpvNBlEUYbPZ4Pf74XA4oCgKLBYL6uvrkZKSgtXVVTQ3N+PTp0/IyMhAY2MjsrKytC4/bouLi3C5XJiZmYEgCKirq8O+ffuSOufnz5+ju7sbgiDgwIEDsFqtmJ+fT6qcW1paMDw8DJPJBLvdDgC/9P7t6enBs2fPAADV1dWx7Zc3hXQqEonQjRs36Nu3b7S6ukq3bt2imZkZrcvaMlmWaWpqioiIQqEQNTQ00MzMDLW3t1NHRwcREXV0dFB7ezsREQ0NDdG9e/coGo2Sz+ej27dva1b7VnV1dZHD4aD79+8TEZHdbqe+vj4iInK73fTy5UsiInrx4gW53W4iIurr66OHDx9qU/AWPXr0iF69ekVERKurqxQMBpM650AgQFarlVZWVohoPd/Xr18nXc4TExM0NTVFN2/ejB2LN1dFUej69eukKIrq683S7VLSx48fsXfvXmRnZyMlJQUnT56Ex+PRuqwty8zMjJ0x7Ny5Ezk5OZBlGR6PB+Xl5QCA8vLyWK+Dg4M4ffo0BEHA0aNHsbi4GNtBL5EEAgEMDw+jqqoKAEBEmJiYQFlZGQDgzJkzqp43zp7Kysrw9u1bUILdgxEKhfDu3TtUVlYCWN/reNeuXUmfczQaRTgcRiQSQTgchtlsTrqc8/Pzf9qDJt5cR0dHUVhYiPT0dKSnp6OwsBCjo6ObrkG3S0myLEOSpNhjSZIwOTmpYUXbz+/3Y3p6GkeOHMHCwgIyMzMBAGazGQsLCwDWX4e/bp4uSRJkWY49N1G0tbXh6tWrWFpaAgAoigKj0QiDwQBgfftZWZYBqLM3GAwwGo1QFCWhtpH1+/3YvXs3Wlpa8OXLF1gsFtTW1iZ1zqIo4uLFi6irq0NqaiqOHz8Oi8WS1DlviDfXv/9+++vrshm6vWJIdsvLy7Db7aitrYXRaFR9TxAECIKgUWXbb2hoCCaTKSHXzH9VJBLB9PQ0zp07hwcPHiAtLQ2dnZ2q5yRbzsFgEB6PB06nE263G8vLy3GdBSeL35Grbq8YRFFEIBCIPQ4EAhBFUcOKts/a2hrsdjtOnTqF0tJSAIDJZMLc3BwyMzMxNzcXO2sSRVG1+1Mivg4+nw+Dg4MYGRlBOBzG0tIS2traEAqFEIlEYDAYIMtyrK+N7CVJQiQSQSgUQkZGhsZdxEeSJEiShLy8PADrSyWdnZ1JnfP4+DiysrJiPZWWlsLn8yV1zhvizVUURXi93thxWZaRn5+/6Z+n2yuG3NxcfP36FX6/H2tra+jv70dxcbHWZW0ZEcHlciEnJwcXLlyIHS8uLkZvby8AoLe3FyUlJbHjb968ARHhw4cPMBqNCbW8AABXrlyBy+WC0+lEY2Mjjh07hoaGBhQUFGBgYADA+h0aG/meOHECPT09AICBgQEUFBQk3Jm12WyGJEmxLW3Hx8exf//+pM55z549mJycxMrKCogo1nMy57wh3lyLioowNjaGYDCIYDCIsbExFBUVbfrn6fovn4eHh/HkyRNEo1FUVFSgurpa65K27P3797hz5w4OHjwYexNcvnwZeXl5aGpqwuzs7E+3uz1+/BhjY2NITU2F1WpFbm6uxl38uomJCXR1dcFms+H79+9wOBwIBoM4fPgw6uvrsWPHDoTDYTQ3N2N6ehrp6elobGxEdna21qXH7fPnz3C5XFhbW0NWVhasViuIKKlzfvr0Kfr7+2EwGHDo0CFcu3YNsiwnVc4OhwNerxeKosBkMqGmpgYlJSVx59rd3Y2Ojg4A67erVlRUbLoGXQ8GxhhjP9PtUhJjjLF/xoOBMcaYCg8GxhhjKjwYGGOMqfBgYIwxpsKDgTHGmAoPBsYYYyp/AkyDqgXSPnXJAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "UPXizw35wwdO", + "outputId": "674c0c79-c01c-41d7-a10d-466fb3dbefd1" + }, + "source": [ + "plot_lr(triangular(250, 0.005, 'triangular2'))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3xU1bnw8d/aM+GSC4FJSCIYVBK8EFAkgyQhSm5oK/aUtlaPolagpSrFE6hWBN9jz/HwaVqEIASL2hRvVLFWsNpWyxADJmMkASMGFAkXIRIMZEIIBMhk9nr/GA1GLrnNZM9lff/RZNbe+1mZkGf22ms9S0gpJYqiKIryNc3oABRFURTfohKDoiiK0o5KDIqiKEo7KjEoiqIo7ajEoCiKorSjEoOiKIrSjtnoADzl4MGD3TouOjqaI0eOeDga36b6HBxUn4NDT/o8ZMiQc35f3TEoiqIo7ajEoCiKorSjEoOiKIrSjkoMiqIoSjsqMSiKoijtdGpWUmVlJatWrULXdbKzs5kyZUq7151OJwUFBezZs4eIiAhyc3OJiYkBYO3atRQVFaFpGtOmTWPMmDEAzJo1i379+qFpGiaTiby8PACOHz9Ofn4+hw8fZvDgwcyZM4fw8HBP9llRFEW5gA7vGHRdp7CwkPnz55Ofn09paSk1NTXt2hQVFREWFsby5cuZPHkyq1evBqCmpga73c6SJUtYsGABhYWF6Lredtzjjz/OokWL2pICwLp16xg9ejTLli1j9OjRrFu3zlN9VRRFUTqhw8RQXV1NXFwcsbGxmM1m0tLSKC8vb9emoqKCjIwMAFJSUqiqqkJKSXl5OWlpaYSEhBATE0NcXBzV1dUXvF55eTkTJ04EYOLEiWddSzGObGpEL92AqtSuKIGtw6Ekh8NBVFRU29dRUVHs2rXrvG1MJhOhoaE0NTXhcDgYMWJEWzuLxYLD4Wj7euHChQBMmjSJnJwcABobGxk0aBAAAwcOpLGx8Zxx2Ww2bDYbAHl5eURHR3fc23Mwm83dPtZfdbfPx9a9yMl/vM7AEVfQZ+QYL0TmPep9Dg6qzx46p0fP1gVPPPEEFouFxsZG/u///o8hQ4YwcuTIdm2EEAghznl8Tk5OWzIBur3yT62U7BzpbEF/7x0Ajv7jb2gxF3sjNK9R73NwUH3umm6vfLZYLNTX17d9XV9fj8ViOW8bl8tFc3MzERERZx3rcDjajv3mv5GRkYwbN65tiCkyMpKGhgYAGhoaGDBgQKc7qXiPrPwQmo/DkGHIihLkyWajQ1IUxUs6TAwJCQnU1tZSV1dHa2srdrsdq9Xark1ycjLFxcUAlJWVkZSUhBACq9WK3W7H6XRSV1dHbW0tiYmJnDp1ipMnTwJw6tQptm3bxrBhwwCwWq1s3LgRgI0bNzJu3DhP9lfpJlmyHiyD0e75FbScRpa/b3RIiqJ4SYdDSSaTienTp7Nw4UJ0XSczM5P4+HjWrFlDQkICVquVrKwsCgoKmD17NuHh4eTm5gIQHx9Pamoqc+fORdM0ZsyYgaZpNDY28uSTTwLuO4z09PS2aaxTpkwhPz+foqKitumqirFkfR18+jHiltth+BVwUTyy1AY33GR0aIqieIGQATLFRFVX7byu9ln/+yvIt19F+91ziKgY9H+vQ/71z2j/U4AYMsyLkXqOep+Dg+pz16jqqkq3SF1H2jfAVdcgotyLFkVqJphM7uElRVECjkoMyoV9tg3q6xATzswAExGRcM11yLJiZKvTwOAURfEGlRiUC5Il6yE0HHFtSrvva+mToKkRtqkFiIoSaFRiUM5LnmhCflSGSMlAhPRp/2LStTAwCr3EZkxwiqJ4jUoMynnJDzdCq7PdMNI3hGZCpGVB1VZkQ/05jlYUxV+pxKCclyxZD8MSEMOGn/N1MSEH5NcPpxVFCRgqMSjnJL/YDQf2ItInnbeNiLkIrhiNLLUhv1U1V1EU/6YSg3JOsnQ9mEMQ191wwXZiQg4cPgS7dvRSZIqieJtKDMpZZMtp5IcbEWPTEGEX3iRJjE2D/qFqTYOiBBCVGJSzyI/KoPkEIv3sh87fJfr2RYy7Abm1FNl8oheiUxTF21RiUM4iS20QHQtXjO5Ue5E+CVpaVGE9RQkQKjEo7cjDh9wF8yZkI7RO/npcmghDL1HDSYoSIFRiUNqR9iIQApGa3eljhBDuYad9u5A1+7wXnKIovUIlBqWN1F1Iuw1GjkFEDe7SsWJ8JpjM7mEoRVH8mkoMyhk7PgbHEXcdpC4SEQMQY8Yjy95DOlVhPUXxZyoxKG1kqQ3CI+Ca8d06XqTnwPEm2LbZw5EpitKbOtzBDaCyspJVq1ah6zrZ2dlMmTKl3etOp5OCggL27NlDREQEubm5xMS4a/evXbuWoqIiNE1j2rRpbTu1Aei6zrx587BYLMybNw+AFStWsGPHDkJDQwGYNWsWl156qSf6qlyAPH4MWVmGmPh9REhI904ycgxYotFL1mNKnuDZABVF6TUdJgZd1yksLOSxxx4jKiqKRx99FKvVysUXX9zWpqioiLCwMJYvX05paSmrV69mzpw51NTUYLfbWbJkCQ0NDTzxxBM89dRTaF/PdvnnP//J0KFD2/Z//sbdd99NSkr7Ms+Kd8myYmhtvWAJjI64C+tlI//xGtJxGGHp2nMKRVF8Q4dDSdXV1cTFxREbG4vZbCYtLY3y8vY1+CsqKsjIyAAgJSWFqqoqpJSUl5eTlpZGSEgIMTExxMXFUV1dDUB9fT1bt24lO7vzs18U75BSuqeaXpKIuPjSHp1LpGWDlO7ZTYqi+KUO7xgcDgdRUVFtX0dFRbFr167ztjGZTISGhtLU1ITD4WDEiBFt7SwWCw6HA4Dnn3+eu+6666y7BYBXXnmF119/nVGjRjF16lRCzjG0YbPZsNncM2Dy8vKIjo7uTH/PYjabu32sv/pun53Vn+L48gsifvkwoT39WURH0zA6GVfZe0Tdc3/n10J4mXqfg4Pqs4fO6dGzddKWLVuIjIxk+PDhbN++vd1rd955JwMHDqS1tZVnnnmGN998k1tvvfWsc+Tk5JCTc6ZkQ3c3w1abh4P+9l8hpA8nRo6l2QM/C/26icjCJRwpfQ9x1TU9Pp8nqPc5OKg+d82QIUPO+f0OP85ZLBbq689sxFJfX4/FYjlvG5fLRXNzMxEREWcd63A4sFgs7Ny5k4qKCmbNmsXSpUupqqpi2bJlAAwaNAghBCEhIWRmZrYNPSneIU+fRm7ehEhOQ4SGeeScYmwq9A9TaxoUxU91mBgSEhKora2lrq6O1tZW7HY7Vqu1XZvk5GSKi4sBKCsrIykpCSEEVqsVu92O0+mkrq6O2tpaEhMTufPOO1m5ciUrVqwgNzeXUaNG8eCDDwLQ0NAA0PaMIj4+3sNdVr5NfmSHk809euj8XaJPX8T4icitHyCbj3vsvIqi9I4Oh5JMJhPTp09n4cKF6LpOZmYm8fHxrFmzhoSEBKxWK1lZWRQUFDB79mzCw8PJzc0FID4+ntTUVObOnYumacyYMaNtRtL5LFu2jGPHjgFwySWXMHPmTA90UzkfWWKDwXEwIsmj5xXpOcjif7rvRjJu9ui5FUXxLiGllEYH4QkHDx7s1nHBPCYp62rRF/wSMeUutMm3efQaUkr0/80FkwnTY0s8eu7uCOb3OZioPndNt58xKIFLlm4AoSFSszx+bndhvUnwRTXywF6Pn19RFO9RiSFIuQvmbYCkaxEW70zvE+NvALMqrKco/kYlhmC1vRKO1nerYF5nifABiGtTkWXFqrCeovgRlRiClF6yHsIHwDXjvHodkZ4DJ5qQlWVevY6iKJ6jEkMQ0hsb4OPNiJRMhLmbBfM668prwDLYPftJURS/oBJDEDq58V1w9axgXmcJTUNMyIZPK5H1dV6/nqIoPacSQ5CRUnLS9hZcdjli6LBeuaZIcxdKlKUbeuV6iqL0jEoMwWbfLlwH9rrH/nuJiI6FK69G2jcgdb3XrqsoSveoxBBkZMl66NsPMe6GXr2uSJ8E9XXw2bZeva6iKF2nEkMQ+aZgXr+0LET/0F69trg2BULD1ZoGRfEDKjEEEbmlFE6dpH/25F6/tgjpc6aw3glVWE9RfJlKDEFElq6HmIsIGTmm48ZeINInQasT+WGxIddXFKVzVGIIEvKrg/D5dsSEHIQQhsQghg2HYcPVcJKi+DiVGIKELLW5C+aleb5gXleI9Emwfw9y/25D41AU5fxUYggC0uVC2otgdDJiYFTHB3iRuG4imEPcs6MURfFJKjEEg+1bodGBNqH31i6cjwgLR4xNRX64EelsMTocRVHOocMd3AAqKytZtWoVuq6TnZ3NlClT2r3udDopKChgz549REREkJubS0xMDABr166lqKgITdOYNm0aY8acefCp6zrz5s3DYrEwb948AOrq6li6dClNTU0MHz6c2bNnYzZ3KkzlPPSS9RARCVd7t2BeZ4n0ScjNm5BbP0CMn2h0OIqifEeHdwy6rlNYWMj8+fPJz8+ntLSUmpqadm2KiooICwtj+fLlTJ48mdWrVwNQU1OD3W5nyZIlLFiwgMLCQvRvrXz95z//ydChQ9ud6+WXX2by5MksX76csLAwioqKPNHPoCWPNcC2ckRqFsJXEuwVoyEqRj2EVhQf1WFiqK6uJi4ujtjYWMxmM2lpaZSXl7drU1FRQUZGBgApKSlUVVUhpaS8vJy0tDRCQkKIiYkhLi6O6upqAOrr69m6dSvZ2dlt55FSsn37dlJSUgDIyMg461pK18iyYnC5erUERkfchfVy4NOPkUe+MjocRVG+o8OPkA6Hg6ioMw8so6Ki2LVr13nbmEwmQkNDaWpqwuFwMGLEiLZ2FosFh8MBwPPPP89dd93FyZMn215vamoiNDQUk8l0Vvvvstls2GzuT5x5eXlER3dvFzKz2dztY32dlJL6D95Du2IUltHXtn3fF/rsuuVWjrz1Cv0/+oDwO37u9ev5Qp97m+pzcPBGnw0ZW9iyZQuRkZEMHz6c7du3d+scOTk55OSc+RTc3c2wA3nzcLn7M/Safej3/KpdH32iz8IMV43hhO3vnMz+AUIzefVyPtHnXqb6HBx60uchQ4ac8/sdDiVZLBbq6+vbvq6vr8disZy3jcvlorm5mYiIiLOOdTgcWCwWdu7cSUVFBbNmzWLp0qVUVVWxbNkyIiIiaG5uxuVytWuvdI8stX1dMC/d6FDOSaRPAscR+FQV1lMUX9JhYkhISKC2tpa6ujpaW1ux2+1YrdZ2bZKTkykuLgagrKyMpKQkhBBYrVbsdjtOp5O6ujpqa2tJTEzkzjvvZOXKlaxYsYLc3FxGjRrFgw8+iBCCpKQkysrc20AWFxefdS2lc+Spk8jN7yOsExD9erdgXmeJMeMhLEKtaVAUH9PhUJLJZGL69OksXLgQXdfJzMwkPj6eNWvWkJCQgNVqJSsri4KCAmbPnk14eDi5ubkAxMfHk5qayty5c9E0jRkzZqBpF85FU6dOZenSpbz66qtcdtllZGUZu1LXX8ktdjh9sld2aesuERKCSMlAbvwX8vgxRPgAo0NSFAUQUkppdBCecPDgwW4dF6hjkq7fz4PjjWj/+/RZtZF8qc+yZi/6//wX4j9/gZb9A69dx5f63FtUn4ODIc8YFP8jD9VA9Q5DC+Z1lrj4MrgkEVmyngD5jKIofk8lhgAkSzeApiFS/WMYTqTnQM0+UIX1FMUnqMQQYKTLhfygCEZbEZGDjA6nU8R1N0BIH/UQWlF8hEoMgeaTCmhsQPPhh87fJUK/Kay3Cdly2uhwFCXoqcQQYPRSGwwYCKOSjQ6lS0T6JDh5Arn1A6NDUZSgpxJDAJGNPlgwr7MuHwWD49RwkqL4AJUYAoj8oAh03acK5nWW0DREWjbs/ARZV2t0OIoS1FRiCBBSSncJjMSrEHEXGx1Ot4i0bBAa0r7B6FAUJaipxBAodn8Kh7706ZXOHRGWaEi6FmkvQuouo8NRlKClEkOAkCXroW9/RPIEo0PpES09BxqOwI5Ko0NRlKClEkMAkKeakRWliHHpiH79jQ6nZ665DsIHuLcjVRTFECoxBABZXgKnT/n1MNI3hDkEkZIJlZuRTY1Gh6MoQUklhgAgS21wUTwMv8LoUDxCpOeAqxX5YbHRoShKUFKJwc/J2gOw+zO/KJjXWWLoJXDZ5cgSmyqspygGUInBz8kSG5hMiNQMo0PxKDEhB778Avbt6rixoige1anlsZWVlaxatQpd18nOzmbKlCntXnc6nRQUFLBnzx4iIiLIzc0lJiYGgLVr11JUVISmaUybNo0xY8bQ0tLC448/TmtrKy6Xi5SUFG677TYAVqxYwY4dOwgNde86NmvWLC699FIPdjlwyNbWrwvmjUMM8I+CeZ0lxl2PfO1PyBIb4rLLjQ5HUYJKh4lB13UKCwt57LHHiIqK4tFHH8VqtXLxxWcWURUVFREWFsby5cspLS1l9erVzJkzh5qaGux2O0uWLKGhoYEnnniCp556ipCQEB5//HH69etHa2sr//3f/82YMWO4/HL3H4C7776blJQU7/U6UHxSAU2NflUwr7NEaBgieQKyfBPythmIvn2NDklRgkaHQ0nV1dXExcURGxuL2WwmLS2N8vLydm0qKirIyMgAICUlhaqqKqSUlJeXk5aWRkhICDExMcTFxVFdXY0Qgn79+gHgcrlwuVwBMz7em/SS9RBpgVFjjQ7FK8SESXCyGbml1OhQFCWodHjH4HA4iIqKavs6KiqKXbt2nbeNyWQiNDSUpqYmHA4HI0aMaGtnsVhwOByA+07kkUce4dChQ9x0003t2r3yyiu8/vrrjBo1iqlTpxISEnJWXDabDZvNBkBeXh7R0dFd6Xcbs9nc7WON5HIc5kjVFkKnTCUiNrZLx/pLn2XUROovuhht80Ys/3Fbj87lL332JNXn4OCNPhtWglPTNBYtWsSJEyd48skn2b9/P8OGDePOO+9k4MCBtLa28swzz/Dmm29y6623nnV8Tk4OOTlnisV1d89Tf90jVv/X30DXOTV2Aqe7GL8/9VlPycS19iUO79iGiDn3/rSd4U999hTV5+BgyJ7PFouF+vr6tq/r6+uxWCznbeNyuWhubiYiIuKsYx0Ox1nHhoWFkZSURGWluwTCoEGDEEIQEhJCZmYm1dXVnexi8JBSumcjXZ6EiO3+H0t/IFKz3IX1SmxGh6IoQaPDxJCQkEBtbS11dXW0trZit9uxWq3t2iQnJ1NcXAxAWVkZSUlJCCGwWq3Y7XacTid1dXXU1taSmJjIsWPHOHHiBAAtLS1s27aNoUOHAtDQ0ADQ9owiPj7ek/0NDLt2QN1B95TOACcGRcGoscgPipAuVVhPUXpDh0NJJpOJ6dOns3DhQnRdJzMzk/j4eNasWUNCQgJWq5WsrCwKCgqYPXs24eHh5ObmAhAfH09qaipz585F0zRmzJiBpmk0NDSwYsUKdF1HSklqairJye4dx5YtW8axY8cAuOSSS5g5c6YXu++fZMl66Of/BfM6S0ufhP7H38H2rXD1OKPDUZSAJ2SALC09ePBgt47ztzFJebIZ/aGfIVIy0O6e1a1z+F2fW53ov5kOI0Ziuv/Rbp3D3/rsCarPwcGQZwyKb5Hl70PL6aAYRvqGMIcgUjPh483IY0eNDkdRAp5KDH5GlqyHIcMgyFYDiwk54HIhy94zOhRFCXgqMfgR+eV+2Ps5In1S0C0IFEOGwfArVGE9RekFKjH4EVm6HkxmREqG0aEYQqRPgtoDsGen0aEoSkBTicFPyFYn8oP34JrrEBGRRodjCGFNhz593ftPKIriNSox+Itt5XD8mHtP5CAl+ocirOnIze8jT58yOhxFCVgqMfgJvcQGA6Mg6VqjQzGUSJ8Ep08iK1RhPUXxFpUY/IBsqIeqrYi0bIRmMjocYyVeBbFD3bOzFEXxCpUY/IC0bwCpIyZkGx2K4YQQ7qmr1TuQh740OhxFCUgqMfg4qevuh61XjEbEXGR0OD5BpGaCpqmH0IriJSox+Lpd2+HwIUQQP3T+LjHQAqOtqrCeoniJSgw+TpbYoH8o4to0o0PxKVp6DjQ2QNUWo0NRlICjEoMPk80nkFtLEdfdoPY8/q5RVhgw0L29qaIoHqUSgw9zF8xrcU/RVNoRZrP7WcO2cmRjg9HhKEpAUYnBh8mS9TD0Ergk0ehQfJKYMAl0XRXWUxQPU4nBR8mafbBvV1AWzOsscdHFkHAlsmS9KqynKB7U4Q5uAJWVlaxatQpd18nOzmbKlCntXnc6nRQUFLBnzx4iIiLIzc0lJiYGgLVr11JUVISmaUybNo0xY8bQ0tLC448/TmtrKy6Xi5SUFG677TYA6urqWLp0KU1NTQwfPpzZs2djNncqzIAiS21gDt6CeZ0l0ichX1gOuz9zL35TFKXHOrxj0HWdwsJC5s+fT35+PqWlpdTU1LRrU1RURFhYGMuXL2fy5MmsXr0agJqaGux2O0uWLGHBggUUFhai6zohISE8/vjjLFq0iD/84Q9UVlby+eefA/Dyyy8zefJkli9fTlhYGEVFRV7otm+TTiey7D3ENeMR4QOMDsenCWs69O2nVkIrigd1mBiqq6uJi4sjNjYWs9lMWloa5eXl7dpUVFSQkZEBQEpKClVVVUgpKS8vJy0tjZCQEGJiYoiLi6O6uhohBP369QPA5XLhcrkQQiClZPv27aSkpACQkZFx1rWCwscfwvEm9dC5E0S//u7CehUlyFPNRoej9IB0tqC//jzyyy+MDiXodThG43A4iIqKavs6KiqKXbt2nbeNyWQiNDSUpqYmHA4HI0aMaGtnsVhwOByA+07kkUce4dChQ9x0002MGDGCY8eOERoaislkOqv9d9lsNmw298rXvLw8oqOju9LvNmazudvHekvD5k20RscSfX02wuT52ki+2OeeaLnlpzSU2gj/bBv9c245Z5tA63Nn+Fufj634HSdtbyE++gDLk6vQIrp+t+xvffYEb/TZsMF7TdNYtGgRJ06c4Mknn2T//v0MHDiw08fn5OSQk3NmNXB3N8P2tc3DpeMweuWHiMm3Ud/gnWmYvtbnnpJRcRB3McfeeYMTY1LO2SbQ+twZ/tRnvdSGtL2FsKajf1TG4UWPof3qMYTWtfkx/tRnT+lJn4cMGXLO73f4U7dYLNTX17d9XV9fj8ViOW8bl8tFc3MzERERZx3rcDjOOjYsLIykpCQqKyuJiIigubkZ19dlDs7VPtC5C+ZJRJoqmNdZQgh3yZDdnyFrDxgdjtJFcv8e5OqVcOXViF/8GnH7DPikAvmv140OLWh1mBgSEhKora2lrq6O1tZW7HY7Vqu1XZvk5GSKi4sBKCsrIykpCSEEVqsVu92O0+mkrq6O2tpaEhMTOXbsGCdOnACgpaWFbdu2MXToUIQQJCUlUVZWBkBxcfFZ1wpk7oJ5G9z/QAbHGR2OX1GF9fyTbD6OvjIPwsLRfvEQQjMhMm5GXHcD8s2/ID/92OgQg1KHQ0kmk4np06ezcOFCdF0nMzOT+Ph41qxZQ0JCAlarlaysLAoKCpg9ezbh4eHk5uYCEB8fT2pqKnPnzkXTNGbMmIGmaTQ0NLBixQp0XUdKSWpqKsnJyQBMnTqVpUuX8uqrr3LZZZeRlZXl3Z+AL9n5CRz5CjHlLqMj8TtiwCC4ehzSXoSccjciCKc4+xspJfqqp8BxGO2hhYgB7qFkIQTcPQt5YC/6c0+iPZaPsATXcwOjCRkgK4MOHjzYreN8aUxSf24xsqoCbdHziD7eq43kS332JPnxZvSC/0ObNR/xnWcNgdrnC/H1Puvv/A35txcQt89Ay/nhWa/L2hr0hb+Giy9xJw5zSIfn9PU+e4MhzxiU3iFPHEdutSOum+jVpBDQRiVD5CD3NqiKT5M7q5BvvIRInoDI/o9zthEXXYz42Wz3s6PXn+/dAIOcSgw+Qm7eBK1OtXahB4TJhEjNcj+4PHruac6K8eRRB/qzf4DYixA/m33Bki/auHRE9g+QG95CLy/pxSiDm0oMPkKW2iD+MsQlCUaH4tfEhBx3Yb0PVGE9XyRbW91J4dRJtPseRfQP7fAYceu97ppYLyxH1tZ02F7pOZUYfIA8sBe+qHZXC1V6RMQNhREjkaU2VVjPB8m1L8GuHYi7ZyGGDuvUMcIcgjbzNxASgv7H3yFPnfRylIpKDD5AlqwHcwgiZaLRoQQEMWESfPUl7NphdCjKt8itHyD/vRaR8X20LhaHFJZotF88BIdqkC89rZK+l6nEYDDpbEGWFSOuTUGERRgdTkAQ1gnQr79a0+BD5FcH0Z9/Ci4dgbjt5906hxg5BvHDqcjNG5HF//JwhMq3qcRgMFn5ITQfd6/cVTxC9O2HGHe9u7DeSVVYz2jy9Gn0P/4ONBPafY8gQjqedno+4vu3wmgrcs2fkHt2ejBK5dtUYjCYLFkPlsFw5TVGhxJQxIQcaDnt3h5VMYyUErn6j3BwP9rP5yKiYnp0PqFpaDPmwEAL+jO/RzYd81CkyrepxGAgWV8Hn36MmJDd5WJhSgeGXwEXxavhJIPJ9/+N/KAIMfl2xKhkj5xThEWg3T8Pjh1F/9NipO7yyHmVM9RfIwPJ0g3A159uFY9qK6y3Zyfy4H6jwwlK8otq5CvPwshrET+43aPnFpckIu6YCTs+Qr69xqPnVlRiMIzUdXcl1auu6fHttXJuIiUTTCa1u5sB5Ikm9D/mwYBItJ//GqF5fl8Rcf1NiNRM5NtrkFVbPH7+YKYSg1E+2wb1depuwYvEgIFwzXXIsmKk02l0OEFD6jp6YT4cdaD98hFENzbc6QwhBGLqAzBkGPqfliDrD3vlOsFIJQaDyJL1EBqOuPbcG8sonqFNyIGmRk5XlBodStCQ/3odPqlA3D4DMfwKr15L9O2Ldv+joLvcD6OdLV69XrBQicEA8kQT8qMyREoGIqSP0eEEtqSxMNDCyQ1vGx1JUJCffox88y+I625AZNzcK9cUsUPQ7n0Q9n5O05+X9co1A51KDAaQH250F8xTw0heJ0wmRFo2LR+VIRvqOz5A6TbZUI/+3JMQN9Rd8uICxfE8TYxNQ9w4hZPvvJ9NuAoAACAASURBVIH+4cZeu26gUonBALJkPQxLQAwbbnQoQUFMyHYX1rNvMDqUgCVbW9Gf+T20nEa7fx6iX/9ej0H86B5CRl6DfLEA+aWaidYTndrmqrKyklWrVqHrOtnZ2UyZMqXd606nk4KCAvbs2UNERAS5ubnExLhn2qxdu5aioiI0TWPatGmMGTOGI0eOsGLFCo4ePYoQgpycHG6+2X3b+dprr7FhwwYGDHA/sLrjjjsYO3asJ/tsKLl/NxzYi7jzPqNDCRoiZgghSdfiLLUhb/5pr36SDRbyb8/D7s8QMx9GXBRvSAzCbCby109wZM496Ct/h7ZgMaJfx9VblbN1eMeg6zqFhYXMnz+f/Px8SktLqalpX/q2qKiIsLAwli9fzuTJk1m9ejUANTU12O12lixZwoIFCygsLETXdUwmE3fffTf5+fksXLiQd999t905J0+ezKJFi1i0aFFAJQX4VsG8624wOpSg0j/nFjh8CD7fbnQoAUcvL0Ha/o7I/gHauOsNjcVkiUab+TB8VYt8frkqttdNHSaG6upq4uLiiI2NxWw2k5aWRnl5ebs2FRUVZGRkAJCSkkJVVRVSSsrLy0lLSyMkJISYmBji4uKorq5m0KBBDB/uHkbp378/Q4cOxeEI/I1VZMtp5Icb3eOhYeFGhxNU+qVmQv9QtabBw2RtDfKF5ZBwpXvfBB8grhiN+NHdyC2lyA1vGR2OX+pwKMnhcBAVFdX2dVRUFLt27TpvG5PJRGhoKE1NTTgcDkaMGNHWzmKxnJUA6urq2Lt3L4mJiW3fe/fdd9m0aRPDhw/nnnvuITz87D+iNpsNm81d7iAvL4/o6O5tFm42m7t9bFedfP/fHGs+QeTkn9C3l655Lr3ZZ19hNpvpf/2NnCz+F5ZfPYoWBInZ2++zPHWS+ucWIfr2JWpeHqZo4xdqftNneddMGmv2cPr1VQwYM44+V442OjSv8cb73KlnDN5y6tQpFi9ezL333ktoqHss8MYbb+TWW28FYM2aNbz44os88MADZx2bk5NDTs6ZWT3d3Qy7NzcPd/1rLUTHcixuGMLADcuDdcP009Z0+Pc6jryzDm3i94wOyeu8+T5LKZF/WoKs2YeW+z80oIEP/E59u89y6v2wdxcNv38U7f8tdS94DEA9eZ+HDBlyzu93OJRksViorz8zza++vh6LxXLeNi6Xi+bmZiIiIs461uFwtB3b2trK4sWLuf766xk/fnxbm4EDB6JpGpqmkZ2dze7du7vQTd8lDx9SBfOMdukIGHqJGk7yAFn8L+TmjYj/uBMxcozR4ZyTCA1Hu28enDiuiu11UYd/oRISEqitraWuro7W1lbsdjtWq7Vdm+TkZIqLiwEoKysjKSkJIQRWqxW73Y7T6aSuro7a2loSExORUrJy5UqGDh3KLbfc0u5cDQ0Nbf+/efNm4uONmeHgadJeBEIgUrONDiVotRXW27cLWbPP6HD8ltz7OXLNn2C0FXHzT40O54LEsOGIqffBpx8j33zF6HD8RodDSSaTienTp7Nw4UJ0XSczM5P4+HjWrFlDQkICVquVrKwsCgoKmD17NuHh4eTm5gIQHx9Pamoqc+fORdM0ZsyYgaZpfPbZZ2zatIlhw4bx8MMPA2empb788svs27cPIQSDBw9m5syZ3v0J9AKpu5B2G4wcg4gabHQ4QU2Mz0S+/gKy1Ia4vXs7iQUz2XQMfeXvYaAFbcYcv7j71SbkoFd/ivzna8iEKxBXjzM6JJ8nZIDM5zp48GC3juuN8XZZtRX9qd+i/fI3CGu6V6/VGcH6jOGbPrtW5sHOT9D+8HyPdhPzdZ5+n6XuQl/2v+6f3SO/R1w6osNjetv5+ixbTqPn/Qbq69Aey0cMjjMgOu8w5BmD0nOy1AbhEXDN+I4bK16npU+C402wbbPRofgV+fZrsP0jxH/OxBeTwoWIPl8X25Ogr1TF9jqiEoOXyePHkJVliPEZAf3p1K+MHAODotHVQ+hOk1VbkW+/ikjJRNxwk9HhdIsYHIc2PRf270a++pzR4fg0lRi8TJYVQ2srIn2S0aEoXxOaCZGWBds/QjpUDf+OyPrD6IWLYcgwxF0P+HVJETFmPOL7P0Fuehdd1c46L5UYvEhK6Z4aeUki4uJLjQ5H+RYxIQekdM8WU85LOp3u4nitrWj3zUP07Wt0SD0mfngXXDEa+fIfkTV7jQ7HJ6nE4E1fVMOXX6i7BR8kBse5/ziU2pC6bnQ4Pkv+tRD2fo427b8QcUONDscjhMmENvMhCA1H/2MesvmE0SH5HJUYvEiW2iCkjyqY56NE+iQ48hV8XmV0KD5J/3Aj8r1/Im6cghibZnQ4HiUGDEL75W/gyFfozz+liu19h0oMXuIumLcJkZyGCA0zOhzlHMTYVOgfplZCn4P8cj/yxQJIHIn40T1Gh+MVYsRIxE/uhY/KkP9eZ3Q4PkUlBi+RW+1w8oQaRvJhok9fxPgbkFs/QDYfNzocnyFPNaOv/B3064/2y4cRZkNLqnmVmPRDGJuGfOMFpLpzbKMSg5fIEhsMjoMRSUaHolyASJ8Ezhbk5k1Gh+ITpJTIFwrgq1q0mQ8jBkZ1fJAfE0K494uOjkN/dhGysaHjg4KASgxeIOtqYecniAk5flEyIKgNS4CLL3UncgVZ9DayogTxo7sRVwRuqepvE/1D0e6fBydPoD/7B6RLFdtTf7W8QJZuAKEhUrOMDkXpgLuw3iT4ohp5ILinLsrqT5F//TNccx3iez82OpxeJS6+FHHXLPh8O3LtS0aHYziVGDzMXTBvAyRdi7AE12Y4/kqMnwhms3sWWZCSx46iP/MHsAxGm57r14vYuktLzUTc8D3ku28gK8uMDsdQKjF42vZKOFrvrsej+AURPgAxJgVZVox0Oo0Op9dJ3YX+p8Vw/BjafY8gQgN/d7vzEf/5c7gkEf3PTyHruleYMxCoxOBhesl6CB8A16jSvv5EpE+CE03Iyg+NDqXXyb+/4t5Eaup9iGEJRodjKBHSB+2+R0AI9D/+Htly2uiQDKESgwfJpkb4eLO70JhZFczzK1ddDZbBQbemQW4rR/7jNcSEHHWX+zURHYv287lQsxf5l5VGh2OITk1QrqysZNWqVei6TnZ2NlOmTGn3utPppKCggD179hAREUFubi4xMe6NwdeuXUtRURGapjFt2jTGjBnDkSNHWLFiBUePHkUIQU5ODjfffDMAx48fJz8/n8OHDzN48GDmzJlDeLh/3NrKsmJwqYJ5/shdWC8b+Y81yPo6RJTxG9t7mzzyFXphPsRfhrjzl0aH41PEaCviltuRb69BT7gK7fobjQ6pV3V4x6DrOoWFhcyfP5/8/HxKS0upqalp16aoqIiwsDCWL1/O5MmTWb16NQA1NTXY7XaWLFnCggULKCwsRNd1TCYTd999N/n5+SxcuJB333237Zzr1q1j9OjRLFu2jNGjR7NunX+sSGwrmHfZ5Yihw4wOR+kGMcG97WowFNaTzhb3TmxSuovj9fH/4nieJn7wnzByDPIvzyC/CIy95zurw8RQXV1NXFwcsbGxmM1m0tLSKC8vb9emoqKCjIwMAFJSUqiqqkJKSXl5OWlpaYSEhBATE0NcXBzV1dUMGjSI4cOHA9C/f3+GDh2Kw+EAoLy8nIkTJwIwceLEs67ls/btgoP73XsKK35JRMfClVcHRWE9+epz8EW1ewZSzEVGh+OThGZC+/mvISISfWUe8kTwrI7vcCjJ4XAQFXVm9WNUVBS7du06bxuTyURoaChNTU04HA5GjDiz05PFYmlLAN+oq6tj7969JCYmAtDY2MigQYMAGDhwII2NjeeMy2azYbO5pxfm5eURHd29qaFms7nbx37bsb8WcrJvP6K/9yM0H6+N5Kk++5PO9vnk93/EsSW/ZUDtF/T18wkE5+vzyff+xbFN7xL6o7uIyJlsQGTe4/Hf7ehoWn6zkIbHHsC8+mkGzsvzuUWr3vj3bGgRlFOnTrF48WLuvfdeQkNDz3pdCHHe+dQ5OTnk5Jz5dN7dPU89sS+uPH0afdO/EWPTcDSfhOaTPTqftwX7ns8XIhNHQWgYjf/8G9rQy3ohMu85V59lzT73ENIVozl10084HWC/B1753Y6+CPHTGbS8+iyHVz+L9v1bPXv+HjJkz2eLxUJ9fX3b1/X19VgslvO2cblcNDc3ExERcdaxDoej7djW1lYWL17M9ddfz/jxZ/ZCjoyMpKHBXa+koaGBAQMGdLaPhpFbSuHUSTWMFABESB/E+InuwnoBNnQgm0+g/zEP+oej/eIhhMlkdEh+Q2RNRoy7Hrn2ZeRn24wOx+s6TAwJCQnU1tZSV1dHa2srdrsdq9Xark1ycjLFxcUAlJWVkZSUhBACq9WK3W7H6XRSV1dHbW0tiYmJSClZuXIlQ4cO5ZZbbml3LqvVysaNGwHYuHEj48b5/u28LF0PMRepgnkBQqRPglYn8sNio0PxGCkl+gvL4Mghd3G8yEFGh+RXhBCIe34FsUPcxfYa6js+yI91mBhMJhPTp09n4cKFzJkzh9TUVOLj41mzZg0VFRUAZGVlcfz4cWbPns3bb7/N1KlTAYiPjyc1NZW5c+eycOFCZsyYgaZp7Ny5k02bNlFVVcXDDz/Mww8/zNatWwGYMmUK27Zt48EHH+STTz45a2qsr5FfHYTPt7sL5gVhGYFAJIYlwLDhAVUiQ65fB1s/QPzkZ4jL1QeY7hD9+ruL7bWcdhfba201OiSvETJAti46eLB7y9d7Oiapv/Ei8p030P5Q6DclitUzho7pRW8jX3kW7f/l++1q4G/6LD/fjr54AYwZ756aGsAfYHrjd1vfvAn53JOInB+i3T7Dq9fqDEOeMSjnJ10u95z30cl+kxSUzhHjM8Ac4vcroWVjA/qziyA6Du1nDwZ0Uugt2nU3IDInI21vup8vBiCVGHpi+1ZodKBNUA+dA40IC0eMTUV+uBHpbDE6nG6RrlZ3Ujh5HO3+eWqLWQ8St02Hyy5Hf34Z8tCXRofjcSox9IBesh4iIuFq339ArnSdmJADzSeQWz8wOpRuOb76Wfi8CjH1AcTFlxodTkAR5hB3sT2z2b347fQpo0PyKJUYukkea4Bt5YjUrIDeEzeoXXk1RMX45UNoWVlG89qXETd8Dy1NbRjlDcIyGO0XD8HB/ciXnyZAHtcCKjF0m7tgnkutXQhgQtPcdw2ffow88pXR4XSarKtF//NTmBOudO8voHiNGHkt4gd3uPfy2PiO0eF4jEoM3eAumGeDhCsRF8UbHY7iRSItG4Rwb9fqB2TLafciNiEY+JuFiJA+RocU8MTk22BUMnLNc8i9uzo+wA+oxNAde3ZC7QH3p0kloImowXDVGKTdhtR9f5N4+ZdnoGYv2ow5mFRxvF4hNA1txhwYMMj9vOH4MaND6jGVGLpBltqgbz/EuHSjQ1F6gUjPAccR+NS3SyHo7/8bWWpDTL4NoSZE9CoRPgDtvnlwrAG9MN/vq/OqxNBF8tRJ5Ob3EdYJiH5nF/5TAo8YkwJhET69pkHu3+2+W7jqGsR/3GF0OEFJXDYCcfvPoWoL8p+vGR1Oj6jE0EVyix1On1S7tAURERKCSMlAVpb55DCBbD7urpgaPsBdHE9TxfGMIiZ+3/278vdXkDs+MjqcblOJoYtkyXqIGwoJVxkditKLxIQcaG1FfrjR6FDakbqO/uel4DiMdt8jiIhIo0MKakIIxF0PwEXx6M89iXQcNjqkblGJoQvkoRqo3qEK5gUhEX8ZXJKILFnvU/PV5btvwMebET+djki40uhwFED07ecuttfair7y98hWp9EhdZlKDF0gSzeApiFS1YKhYCTSc6BmH+z3jf1/5WfbkGtfRljTEVm3dHyA0mtE3MVo9z4Iez9H/nWV0eF0mUoMnSRdLuQHRTDaqmrZBylx3Q0Q0scnHkLLo/XuOkixQxA/+5W6g/VBInkCIueHyKK30TdvMjqcLlGJobM+qYDGBjT10DloidBvCuttQracNiwO2dqK/swiaDntLo6nZsf5LPGTn0HiVcgXC5AH9xsdTqepxNBJeqkNBgyEUclGh6IYSKRPgpPGFtaTb7zgftZ19yzEkGGGxaF0TJjNaDN/A336up83nPLt/eC/0anqb5WVlaxatQpd18nOzj5rVzWn00lBQQF79uwhIiKC3NxcYmJiAFi7di1FRUVomsa0adMYM2YMAE8//TRbt24lMjKSxYsXt53rtddeY8OGDW17Pd9xxx2MHTvWI53tLtn4dcG8SVNUwbxgd/koiI51DyelZPT65eUWO3L9m4jMm9HGT+z16ytdJwZFof3iIfT8x5EvFsAvHvL5ob8O7xh0XaewsJD58+eTn59PaWkpNTU17doUFRURFhbG8uXLmTx5MqtXrwagpqYGu93OkiVLWLBgAYWFhehfrwjMyMhg/vz557zm5MmTWbRoEYsWLTI8KQDuZwu6rgrmKWcK6+38BHn4UK9eWx76Ev35p+CyyxE/NX7nMKXzxFXXIKZMRZa/jyz6h9HhdKjDxFBdXU1cXByxsbGYzWbS0tIoLy9v16aiooKMjAwAUlJSqKqqQkpJeXk5aWlphISEEBMTQ1xcHNXV1QCMHDmS8PBwz/fIw6SU7hIYiVch4i42OhzFB4i0rK8L6/VeOW55+hT6yjwwm9F++QgiJKTXrq14hvjeT+Dqcci//hm5+zOjw7mgDsdFHA4HUVFntq2Miopi165d521jMpkIDQ2lqakJh8PBiBEj2tpZLBYcDkeHQb377rts2rSJ4cOHc88995wzgdhsNmw29z/MvLw8oqOjOzzvuZjN5gse2/LpNhoOfcmAX/2M/t28hq/pqM+ByKN9jo6mYcx4WsuKiZo2G2Hy7kpjKSXHlj3BqYP7GfjfS+h7RecWV6r32ffoDz+B46HpyOeexLJ4FZoHZjh6o88+N2B+4403cuuttwKwZs0aXnzxRR544IGz2uXk5JCTc2Zop7ubYXe0kbb+j9ehb3+OX3ENJ7y8yXhv6Y0N032Np/ssx09E/6iMI+/bEF6ekKBvfAdZ/A7iB3fQdHECTZ3sh3qffZP8xcPoeb/h8O/no+X+tsclTHrS5yFDhpzz+x0OJVksFurr69u+rq+vx2KxnLeNy+WiubmZiIiIs451OBxnHftdAwcORNM0NE0jOzub3buNW0wkTzUjK0oQ49IR/fobFofig66+DsIj3Nu7epHctwv56rOQdC3iltu9ei2ld4hLEhB3/tK9AdRbrxodzjl1mBgSEhKora2lrq6O1tZW7HY7Vqu1XZvk5GSKi4sBKCsrIykpCSEEVqsVu92O0+mkrq6O2tpaEhMTL3i9hoaGtv/fvHkz8fHGbYQjy0vg9ClVME85i7uwXiZUbkY2NXrlGvL4MXdxvAGD0Gb8GqGp2eWBQqRPQqRlI99eg/xki9HhnKXDoSSTycT06dNZuHAhuq6TmZlJfHw8a9asISEhAavVSlZWFgUFBcyePZvw8HByc3MBiI+PJzU1lblz56JpGjNmzED7+pd76dKl7Nixg6amJu677z5uu+02srKyePnll9m3bx9CCAYPHszMmTO9+xO4AFlqg4viYfgVhsWg+C6RPglp+zvyw2JEzg89em6p6+iF+XDUgfZIHiJigEfPrxhLCAF33ofcvwe9cAnaY0sQ0bFGh9VGSF+qCNYDBw8e7NZx5xufk7UH0P97FuLWaWg3/ain4fkUfxiH9TRv9dm18NfgbEF7fJlH56brb69Bvrkaced9aJk3d+sc6n32fbLuIPr/zYWYIWiP/L5bs80MecYQrGSJDUwmRGqG0aEoPkykT4Ivv4B9ntvrV+74CPn3vyDGT0RkfN9j51V8j4gZgjYtF76oRq55zuhw2qjEcA6ytfXrgnnjEANUwTzl/MS466FPH/cHCQ+QjsPozy2Gi+LdJS98fIWs0nPi2hTETT9CbnwHvew9o8MBVGI4t08qoKlRFcxTOiRCwxBjJyDLNyFP96ywnmx1oj/zB3A63cXx+vbzUJSKrxM/ugcuT0K+tAJZs8/ocFRiOBe9ZD1EWmCU8eU4FN/nLqzXjNxq79F55OvPw56daPfOVqvsg4wwmdzF9vqHof8xD3my2dB4VGL4Dnm0Hj7ZgkjL9PqKViVAXJ4Eg+N6tE+DvnkTcsNbiJz/QFjTPRic4i9E5CC0mQ/DkUPozy8zdKdAlRi+Q37wHkgdMUENIymdI4RwF9b7vApZ1/XZcbL2gLvqZsKViJ/c6/kAFb8hLh+F+PHPYKsdafu7YXGoxPAtUkr3Q8TLkxCx557GpSjnItKyQWju7V+7QJ46if7HPOjT110cT5V1D3rixilwbQry9VXIXTsMiUElhm/btQPqDro//SlKF4hBUTBqLNK+AelydeoYKSXypRVw6Eu0XzzkPocS9IQQaPf+F0THoj/zB+Sxho4P8jCVGL5FlqyHfv0RyROMDkXxQ1p6Dhx1wPatnWov3/sHcvMmxA/vRFx1jZejU/yJCA1Du28eNB9Hf/bJTn/Y8BSVGL4mTzYjt5QirrtBTRNUuufqcRAR6d4GtgNy92fI1/4MV49DfP/WXghO8Tci/jLEXfe7N4V6c3WvXlslhq/J8veh5bQaRlK6TZhDECkZ8PFm5LGj520nm46hP/sHGGhBmz5HFcdTzktLy0ZcfyPyX68jKz/svev22pV8nCxZD0OGwWWXGx2K4sdE+iRwuZDnWcEqdRf6nxbDsUb3IrYw39/FUDGWuGMmDEtA//PSXttOViUGQH65H/Z+7i6Fq0oQKD0ghgyD4VcgS2znnIcu31oDOz5C3DETccmFS9ArCoAI6YN23yMgQF+Zh2zp2Qr7zlCJAZCl68Fkdg8DKEoPiQk5UHsA9uxs931ZtQX5jzWI1CzE9TcaFJ3ij8TgOLQZc2H/HuQrz3r9ekGfGKTT6V7Uds11iIhIo8NRAoC7sF5f934eX5P1deh/WgJDL0FMvV/dmSpdJq4eh7j5p8iS9Z2a4NATQZ8YTleUwvFj7qmGiuIBon8oInkCcvP7yNOnkE6nexGb7kK7bx6ib1+jQ1T8lPjhnXDl1cjVK5H793jtOp1aZllZWcmqVavQdZ3s7GymTJnS7nWn00lBQQF79uwhIiKC3NxcYmJiAFi7di1FRUVomsa0adMYM2YMAE8//TRbt24lMjKSxYsXt53r+PHj5Ofnc/jwYQYPHsycOXMID/feA7qTG96CgVGQdK3XrqEEH5E+CflBEbKiFPZ9Dl9Uo93/qFpRr/SI0Exov3gI/Ylc9JV5aI8tAaI9fp0O7xh0XaewsJD58+eTn59PaWkpNTU17doUFRURFhbG8uXLmTx5MqtXu+fc1tTUYLfbWbJkCQsWLKCwsBBd1wHIyMhg/vz5Z11v3bp1jB49mmXLljF69GjWrVvniX6ek2yop+WjDxFp2QhNFcxTPGjESIgZgvzb88jifyFu/BFibKrRUSkBQAwYiPbLR8BxGH3VU14pttdhYqiuriYuLo7Y2FjMZjNpaWmUl5e3a1NRUUFGRgYAKSkpVFVVIaWkvLyctLQ0QkJCiImJIS4ujurqagBGjhx5zjuB8vJyJk6cCMDEiRPPupYnSfsG0HXEhGyvXUMJTkIIRHoONDW6a2/9+B6jQ1ICiEi8CnHrvVD5Iae7WJ+rMzocSnI4HERFnanhEhUVxa5du87bxmQyERoaSlNTEw6HgxEjRrS1s1gsOByOC16vsbGRQYPcu6YNHDiQxsbGc7az2WzYbO4HMHl5eURHd/126uTFw2id9AMiRo7u8rH+zGw2d+vn5c+M6LP+46mcEBA6+aeYDKiDpN7nwCZvn87p+MsIuz6Hfl+PxHiKT5dyFEKcd/ZGTk4OOTlnHhh3azPsa1KIzr7FrzYP9wR/2zDdEwzr8/du5bRLggHXVu9zELjiavrperf7PGTIuZ95dTiUZLFYqK+vb/u6vr4ei8Vy3jYul4vm5mYiIiLOOtbhcJx17HdFRkbS0OCuJtjQ0MCAAQM6ClFRFEXxoA4TQ0JCArW1tdTV1dHa2ordbsdqtbZrk5ycTHFxMQBlZWUkJSUhhMBqtWK323E6ndTV1VFbW0ti4oVXe1qtVjZu3AjAxo0bGTduXDe7piiKonSHkJ14pL1161ZeeOEFdF0nMzOTH//4x6xZs4aEhASsVistLS0UFBSwd+9ewsPDyc3NJTY2FoA33niD9957D03TuPfee7n2Wve00KVLl7Jjxw6ampqIjIzktttuIysri6amJvLz8zly5EiXpqsePNj1nbMgCG89UX0OFqrPwaEnfT7fUFKnEoM/UImh81Sfg4Pqc3DwRmII+pXPiqIoSnsqMSiKoijtqMSgKIqitKMSg6IoitJOwDx8VhRFUTwj6O8Y5s2bZ3QIvU71OTioPgcHb/Q56BODoiiK0p5KDIqiKEo7pt/+9re/NToIow0fPtzoEHqd6nNwUH0ODp7us3r4rCiKorSjhpIURVGUdlRiUBRFUdrx6Y16vK2yspJVq1ah6zrZ2dlMmTLF6JB67MiRI6xYsYKjR48ihCAnJ4ebb76Z48ePk5+fz+HDh9tVrZVSsmrVKj766CP69u3LAw884LdjtLquM2/ePCwWC/PmzaOuro6lS5fS1NTE8OHDmT17NmazGafTSUFBAXv27CEiIoLc3FxiYmKMDr/LTpw4wcqVKzlw4ABCCO6//36GDBkS0O/z22+/TVFREUII4uPjeeCBBzh69GhAvc9PP/00W7duJTIyksWLFwN0699vcXExb7zxBgA//vGP27Zf7hQZpFwul/zVr34lDx06JJ1Op3zooYfkgQMHjA6rxxwOh9y9e7eUUsrm5mb54IMPygMHDsiXXnpJrl27Vkop5dq1a+VLL70kpZRyy5YtcuHChVLXdblz50756KOPGhZ7T7311lty6dKl8ne/WK3qeQAABNVJREFU+52UUsrFixfLkpISKaWUzzzzjHz33XellFK+88478plnnpFSSllSUiKXLFliTMA9tHz5cmmz2aSUUjqdTnn8+PGAfp/r6+vlAw88IE+fPi2ldL+/7733XsC9z9u3b5e7d++Wc+fObfteV9/XpqYmOWvWLNnU1NTu/zsraIeSqquriYuLIzY2FrPZTFpaGuXl5UaH1WODBg1q+8TQv39/hg4disPhoLy8nIkTJwIwceLEtr5WVFRwww03IITg8ssv58SJE2076PmT+vp6tm7dSnZ2NgBSSrZv305KSgoAGRkZ7fr8zaenlJQUqqqqkH42B6O5ufn/t3f3Lq2zYRzHvxJUqNW2ieigiK9LK7Wg4uSgg5MuDoLi4KgFxdHJP0AQHazUQdDVRcFdrEMRfC3i+1CdxKIppUVraJtnKPY5fXwGe46c0vT+bE0Due78aO/kJuHi+vqa/v5+IN3ruKKiwvA5p1IpNE0jmUyiaRpWq9VwOdvt9i89aHLN9fz8HKfTidlsxmw243Q6OT8//3YNRbuUpKoqivJvg3ZFUbi/v89jRT8vFAoRDAZpbW0lEolgs9kAsFqtRCIRIH0efm2erigKqqpm9i0UGxsbjI+P8/7+DkA0GsVkMiFJEpBuP6uqKpCdvSRJmEwmotFoQbWRDYVCVFVVsbq6yuPjI83NzUxMTBg6Z1mWGRoaYmpqirKyMjo6OmhubjZ0zp9yzfW//2+/npfvKNo7BqOLx+MsLi4yMTGByWTK+q6kpISSkpI8VfbzTk5OsFgsBblm/ruSySTBYJCBgQEWFhYoLy9nZ2cnax+j5RyLxTg6OsLj8bC2tkY8Hs/pKtgo/kauRXvHIMsyr6+vmc+vr6/IspzHin5OIpFgcXGR3t5eenp6ALBYLITDYWw2G+FwOHPVJMtyVvenQjwPt7e3HB8fc3Z2hqZpvL+/s7GxwdvbG8lkEkmSUFU1M67P7BVFIZlM8vb2RmVlZZ5HkRtFUVAUhba2NiC9VLKzs2PonC8uLqipqcmMqaenh9vbW0Pn/CnXXGVZ5urqKrNdVVXsdvu3j1e0dwwtLS08PT0RCoVIJBL4/X66urryXdYf03Udr9dLXV0dg4ODme1dXV34fD4AfD4f3d3dme0HBwfous7d3R0mk6mglhcAxsbG8Hq9eDweZmdnaW9vZ2ZmBofDweHhIZB+QuMz387OTvb39wE4PDzE4XAU3JW11WpFUZRMS9uLiwvq6+sNnXN1dTX39/d8fHyg63pmzEbO+VOuubpcLgKBALFYjFgsRiAQwOVyfft4Rf3m8+npKZubm6RSKfr6+hgeHs53SX/s5uaG+fl5GhoaMj+C0dFR2traWFpa4uXl5cvjbuvr6wQCAcrKynC73bS0tOR5FL/v8vKS3d1d5ubmeH5+Znl5mVgsRlNTE9PT05SWlqJpGisrKwSDQcxmM7Ozs9TW1ua79Jw9PDzg9XpJJBLU1NTgdrvRdd3QOW9tbeH3+5EkicbGRiYnJ1FV1VA5Ly8vc3V1RTQaxWKxMDIyQnd3d8657u3tsb29DaQfV+3r6/t2DUU9MQiCIAhfFe1SkiAIgvD/xMQgCIIgZBETgyAIgpBFTAyCIAhCFjExCIIgCFnExCAIgiBkERODIAiCkOUfC6ZQWC23jhoAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "2MMOua0WwwdO", + "outputId": "5b3a0bc1-b1f8-4157-b292-4f81795f459c" + }, + "source": [ + "plot_lr(triangular(250, 0.005, 'exp_range', gamma=0.999))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deUBV1733//faBxwYRA+IRMVE0AwOEeWoQDSiEk1iBjPZzFVzmza25pI8t7c26W36u328zdPEoRFT015qJtuYSTOZGAnihAOoOCYqDkmIGISDCqJMe/3+OIaEODCdwz7D9/WXwB4+i4182WuvvZbSWmuEEEKIcwyrAwghhPAuUhiEEEI0IoVBCCFEI1IYhBBCNCKFQQghRCNSGIQQQjQSZHUAdzl69Gir9ouKiqK0tNTNabybtDkwSJsDQ1va3LNnzwt+Xu4YhBBCNCKFQQghRCNSGIQQQjQihUEIIUQjUhiEEEI0IoVBCCFEI1IYhBBCNCKFQTSbPnUCc0MWMlO7EP7Nb15wE55nvvDf8GUhqtflcEV/q+MIITxE7hhEs+jiIviy0PXv7ZstTiOE8CQpDKJZ9IdLoWMn6HU5umCT1XGEEB4khUE0SRcXofPWosZOQo2eAEe/Qn/burmphBDer1nPGAoKCli8eDGmaTJ+/HgmT57c6Ou1tbVkZGRw6NAhwsPDSU9PJzo6GoBly5aRnZ2NYRhMmzaNhISEhv1M02TWrFnY7XZmzZoFQElJCfPnz6eiooK4uDhmzpxJUJA8CrGS/vAN6NARNWEy1FSj3/g7umATauKdVkcTQnhAk3cMpmmSmZnJU089xbx589iwYQNFRUWNtsnOziY0NJQFCxYwadIklixZAkBRURG5ubnMnTuXp59+mszMTEzTbNhvxYoV9OrVq9GxXn/9dSZNmsSCBQsIDQ0lOzvbHe0UraSLv0bnrXPdLYRHoCKjoU88ert0Jwnhr5osDIWFhcTExNCjRw+CgoJISUkhLy+v0Tb5+fmkpqYCkJSUxO7du9Fak5eXR0pKCsHBwURHRxMTE0NhoesBZllZGdu2bWP8+PENx9Fas2fPHpKSkgBITU0971yifekPl567W7ij4XNq6Eg4tA99wmlhMiGEpzRZGJxOJ5GRkQ0fR0ZG4nQ6L7qNzWYjJCSEioqK8/a12+0N+7788ss8+OCDKKUavl5RUUFISAg2m+287UX700e/ct0tjJuECu/S8Hk1NBm0Ru/cYmE6IYSnWNJ5v3XrViIiIoiLi2PPnj2tOkZWVhZZWVkAPPvss0RFRbXqOEFBQa3e11c1t80nXnmBmo6dibr3EYwuXRs+ryMjKYvphW33Nrrd+aAno7qNXOfAIG120zGb2sBut1NWVtbwcVlZGXa7/YLbREZGUl9fT1VVFeHh4eft63Q6sdvt5Ofnk5+fz/bt26mpqeHMmTO88MILzJw5k6qqKurr67HZbA3bX0haWhppaWkNH7d2BSNZ8enC9NGvMDd8hrrxLpw1dfCj7c1rh1P/2Ycc//orVOcQT8Z1C7nOgUHa3DKtXsEtPj6e4uJiSkpKqKurIzc3F4fD0WibxMREcnJyANi0aRMDBw5EKYXD4SA3N5fa2lpKSkooLi6mX79+3H///SxatIiFCxeSnp7OoEGDePzxx1FKMXDgQDZtcj3YzMnJOe9con24ni10co1EugA1NAnq69C78ts5mRDC05q8Y7DZbEyfPp3Zs2djmiZjx44lNjaWpUuXEh8fj8PhYNy4cWRkZDBz5kzCwsJIT08HIDY2luTkZJ588kkMw+CRRx7BMC5dix544AHmz5/PG2+8Qd++fRk3bpx7WiqaTX/zFTp/Peqmu1FhXS68UdxVEB4BBZthxPXtG1AI4VFK+8mMaEePtu6FK7n1PJ/59+fRO/Iwnv37xQsDYL6agc5bhzH3dVRwsCeiuo1c58AgbW6ZVnclicCiS46i89ajUm+6ZFGAc91JZ8/AFzvbKZ0Qoj1IYRCN6I/fgaAg1ITbm9746iHQsTN6+0bPBxNCtBspDKKBdh5Hb1yNGnUDqku3JrdXwcGowYnogs1os74dEgoh2oMUBtFAr1wG6JbNgZQwEipOwqH9HsslhGhfUhgEAPpUOXrdp6ikVFRk92bvpwY7wBYkcycJ4UekMAgA9Kr3oa4OdePdLdpPhYTC1YPR2zfKkp9C+AkpDAJ9uhKdswLluA4V06vpHX5EDU2G48fg6FceSCeEaG9SGAR69Ydw9gzq5pbdLXxHDRnhOo6MThLCL0hhCHD67Bl01gcwZASqd99WHUN1tUP81eitUhiE8AdSGAKcXvsJnK7AuKl1dwvfUYnXQdFhdIks+SmEr5PCEMB0bQ360+Vw9bWo+KvbdCw1LNl1TLlrEMLnSWEIYDo3G06WY9x8T5uPpSKj4Yr+6K0b3JBMCGElKQwBSpv1rruFy/vB1de65ZhqWAp8WYguK3HL8YQQ1pDCEKgKNkPJUYwb72y0vGpbqMTvupNy3XI8IYQ1pDAEIK015ifvQvcYOPdswB1UdE/o3Re9TQqDEL5MCkMAqt1bAIf3oyZMRhk2tx5bJabAwS/Q5WVNbyyE8EpSGALQ6eX/hLAuqJTxbj+2SkwB5GU3IXyZFIYAo7/5ipr8Dahxt6A6dHT78dVlsXBZrDxnEMKHSWEIMPrTZdCxE2rszR47h0pMgQN70afKPXYOIYTnSGEIILq8DL15DZ3Tbmly2c62UIkpoE309s0eO4cQwnOkMAQQ/dn7YJqE3nqvZ0/U6wqIvkxGJwnho6QwBAhddRq95hOU4zpsPXp69FxKKdddwxc70ZWnPHouIYT7SWEIEHrdStfU2i1ZtrMN1LAUME30ji3tcj4hhPsENWejgoICFi9ejGmajB8/nsmTJzf6em1tLRkZGRw6dIjw8HDS09OJjo4GYNmyZWRnZ2MYBtOmTSMhIYGamhqeeeYZ6urqqK+vJykpiSlTpgCwcOFC9u7dS0hICAC//OUvueKKK9zY5MCj62rRWe/DNUNQl8e3z0kv7weR0a7RSdeltc85hRBu0WRhME2TzMxMfve73xEZGclvf/tbHA4HvXv3btgmOzub0NBQFixYwIYNG1iyZAlPPPEERUVF5ObmMnfuXMrLy/njH//IX/7yF4KDg3nmmWfo1KkTdXV1/P73vychIYErr7wSgIceeoikpCTPtTrA6Lz1cMKJ8dOZ7XZOpRRqWDI6+yN01WnXEqBCCJ/QZFdSYWEhMTEx9OjRg6CgIFJSUsjLy2u0TX5+PqmpqQAkJSWxe/dutNbk5eWRkpJCcHAw0dHRxMTEUFhYiFKKTp06AVBfX099fb3b5usRjWmt0VnvwWWxMHBYu55bJV4H9XXondKdJIQvafKOwel0EhkZ2fBxZGQkBw4cuOg2NpuNkJAQKioqcDqd9O/fv2E7u92O0+kEXHciv/nNbzh27BgTJ05stN2//vUv3n77bQYNGsQDDzxAcHDwebmysrLIysoC4NlnnyUqKqol7W4QFBTU6n19Qc3u7ZR/dYjwx35DSPfuQPu1WdtTKLVHEbwrn663tH1q77bw9+t8IdLmwOCJNjfrGYMnGIbBc889x+nTp3n++ef56quv6NOnD/fffz9du3alrq6Ol156iffee4+77z5/dbG0tDTS0r7vuy4tLW1VjqioqFbv6wvq33kNwsI5PchB1bl2tmebdUIS1WtXcvzrr1CdQ9rlnBfi79f5QqTNgaEtbe7Z88IjFJvsSrLb7ZSVfT8hWllZGXa7/aLb1NfXU1VVRXh4+Hn7Op3O8/YNDQ1l4MCBFBQUANCtWzeUUgQHBzN27FgKCwub2UTxY7qkGHZsRl1/k0emv2gONXwU1NXK6CQhfEiThSE+Pp7i4mJKSkqoq6sjNzcXh8PRaJvExERycnIA2LRpEwMHDkQphcPhIDc3l9raWkpKSiguLqZfv36cOnWK06dPA1BTU8POnTvp1asXAOXlrmkUvntGERsb6872BhSd/SEYNo9Of9GkuKuhayQ6f711GYQQLdJkV5LNZmP69OnMnj0b0zQZO3YssbGxLF26lPj4eBwOB+PGjSMjI4OZM2cSFhZGeno6ALGxsSQnJ/Pkk09iGAaPPPIIhmFQXl7OwoULMU0TrTXJyckkJiYC8MILL3DqlOulqMsvv5xHH33Ug833X7rqNHp9Fmr4aFRXe9M7eIgyDJRjFHr1R+iqSlRImGVZhBDNo7TW2uoQ7nD06NFW7eevfZLmp8vQby3G+K95qD6N311o7zbrQ/sw//Rr1NR/x7jO/VN9N4e/XudLkTYHBkueMQjfo+vr0dkfwZWDzisKluh7petlt/x1VicRQjSDFAZ/VLAJykow0m6zOglw7mU3xyj4fIfMnSSED5DC4IfMVe+51nMeMtzqKA3U8NFQX4/evsnqKEKIJkhh8DP68H44+AVq/K1uX8+5TfrEQfcYdJ50Jwnh7aQw+Bmd9T50DkFZ9JD3YpRSrruGL3ahT52wOo4Q4hKkMPgRfaIMvXUDatQNqE7WvWV8MWr4KNfKbrKAjxBeTQqDH9FrV4JpolItfKHtUnpdATG90fkbrE4ihLgEKQx+QtfVugrDoERU9GVWx7kgV3fSKNi/G33CaXUcIcRFSGHwE3prLpwsxxg3yeool6Qco0BrV14hhFeSwuAn9OqPIPoyGDDU6iiXpHr2gV6Xy8tuQngxKQx+QH950DVEdezNKMP7L6lyjILCz9HO41ZHEUJcgPf/FhFN0qs/gg4dUSneNUT1YtTw0QDSnSSEl5LC4ON05Sn0lrWo5LE+M3Op6tET+sTJy25CeCkpDD5Ob8iC2hrUWO9+6PxjyjEaDu9Hl35rdRQhxI9IYfBh2qxHr14BVw1G9brc6jgtohzXAchdgxBeSAqDL9uZ75pF1cfuFgBU9xiIvxq9eY3VUYQQPyKFwYeZqz+CblGQMNLqKK2iRo6Bb75EFx2xOooQ4gekMPgoXVwEewtQY25E2bxoFtUWUInXgWGgt6y1OooQ4gekMPgonbMCgoJQoydYHaXVVJeuMCABvWUt2jStjiOEOEcKgw/S1WfRG1ejhl3n+uXqw9SIMVBWAoe+sDqKEOIcKQw+SOetgzOnUWNutDpKm6mhI6FDB/Rm6U4SwltIYfBBes0ncFks9B9gdZQ2U51CUENGovPXo+vqrI4jhEAKg8/RXx6EIwdQY25CKWV1HLdQI66HylPweYHVUYQQQFBzNiooKGDx4sWYpsn48eOZPHlyo6/X1taSkZHBoUOHCA8PJz09nejoaACWLVtGdnY2hmEwbdo0EhISqKmp4ZlnnqGuro76+nqSkpKYMmUKACUlJcyfP5+Kigri4uKYOXMmQUHNihkQ9NpPoEMHVHKq1VHcZ9AwCAlDb16DGuywOo0QAa/JOwbTNMnMzOSpp55i3rx5bNiwgaKiokbbZGdnExoayoIFC5g0aRJLliwBoKioiNzcXObOncvTTz9NZmYmpmkSHBzMM888w3PPPcef//xnCgoK2L9/PwCvv/46kyZNYsGCBYSGhpKdne2BZvsmfabK9ctz+GifmRepOVRQMMpxHbpgM7r6rNVxhAh4TRaGwsJCYmJi6NGjB0FBQaSkpJCXl9dom/z8fFJTUwFISkpi9+7daK3Jy8sjJSWF4OBgoqOjiYmJobCwEKUUnTp1AqC+vp76+nqUUmit2bNnD0lJSQCkpqaed65ApjfnQPVZ1JibrI7idmrEGKg+i96xxeooQgS8JvtonE4nkZGRDR9HRkZy4MCBi25js9kICQmhoqICp9NJ//79G7az2+04na4lHU3T5De/+Q3Hjh1j4sSJ9O/fn1OnThESEoLt3AtbP9z+x7KyssjKygLg2WefJSoqqiXtbhAUFNTqfduT1hrn+lUQdyV2R3Kbni94Y5u1/XpKI7sTVLCJbjff6fbje2ObPU3aHBg80WbLOu8Nw+C5557j9OnTPP/883z11Vd07dr8MflpaWmkpaU1fFxaWtqqHFFRUa3etz3pg19gfnkQ9dAMysrK2nQsb22zThxFzWfvc/zIIVRYF7ce21vb7EnS5sDQljb37Nnzgp9vsivJbrc3+kVUVlaG3W6/6Db19fVUVVURHh5+3r5Op/O8fUNDQxk4cCAFBQWEh4dTVVVFfX39RbcPVHrNJ9Cxs2sEj59SI8dAfb0s4COExZosDPHx8RQXF1NSUkJdXR25ubk4HI1HjiQmJpKTkwPApk2bGDhwIEopHA4Hubm51NbWUlJSQnFxMf369ePUqVOcPn0agJqaGnbu3EmvXr1QSjFw4EA2bdoEQE5OznnnCkT6dAU6fz0qaQyqU4jVcTwnti9cFoveIjOuCmGlJruSbDYb06dPZ/bs2ZimydixY4mNjWXp0qXEx8fjcDgYN24cGRkZzJw5k7CwMNLT0wGIjY0lOTmZJ598EsMweOSRRzAMg/LychYuXIhpmmitSU5OJjExEYAHHniA+fPn88Ybb9C3b1/GjRvn2e+AD9Abs12L8fjhQ+cfUkqhRlyPfm8J2nkcZe9udSQhApLSWmurQ7jD0aNHW7Wft/dJaq0xfz8DQsKw/fY5txzTm9usS4oxn/456q6fYtx4l9uO681t9hRpc2Cw5BmDsNj+PXDsG7+YF6k5VPRlEHcVelOO1VFEO9Nnz2Au/V908ddWRwl4Uhi8nF67EjqHohJHWR2l3ajksa4FfL4+bHUU0U60aWJmzkNnvY+5ZBF+0pHhs6QweDF9uhK9LRc1cgyqY0er47Qb5RgFtiDXsxUREPTy16Bgk2tiyH274IudVkcKaFIYvJjenAN1tajRN1gdpV2psC5wrQO9eQ363NBl4b/M3Gz0x++grp+I8cR/Q7cozGWvyV2DhaQweCmtNXrdKugTj+oTb3Wcdmckj4NTJ2CvzLjqz/TBL9CvZcDV16Lu+zkquAPqlp/A4f2wU6bDsYoUBm/11UEoOowaFVh3Cw0GJ0JouHQn+TFdXob51z9BtyiMX/wGdW4WZZUyHrrHYC5fIku+WkQKg5fS6z6F4A6okf77pvOlqKBg1PDRrhlXz1RZHUe4ma6tcRWFs2cwfvk0KjS84WsqKAh1231QdFjegreIFAYvpKur0VvWohJT/Gp67ZZSyWOhtga9dYPVUYQbaa3Rr/8VDu/HmP4Eqtfl522jRlzvegv+/SXynMkCUhi8kN66Ac5UoUZNsDqKtfpeCT16oTeutjqJcCOd/SE69zPULfeihiVfcBtl2DBufwCOfeMahCHalRQGL6Q3rILoy+DKgVZHsZRSynXXsH83uvRbq+MIN9Cf70C/mQkJI1G33nvpjYclQ5949Pv/QtfVtk9AAUhh8Dr62Dewfw9q1A1+s6ZzW6ikVAB5E9oP6OPHMP/2Z+jRy9WFZFz6149SCmPyg1BWgl6f1U4pBUhh8Dp6QxYYBipZJg8EUJHRcOUg9MbVMq7dh+nqs5gv/glM0/WwuXMzZwkeNAzir0Z/tBRdU+3ZkKKBFAYvouvq0LmfwWAHqqusQ/EdlTwWSo66xrYLn+N62PwifHME42f/gepx4YnbLkQphXHHQ3DC6VqTRLQLKQzeZFc+nDqBMTrAHzr/iEq8DoI7yENoH6XXfILelIO67T7UoMQW76+uGgzXDEGveEuGLrcTKQxexFy/CiLs0Ir/PP5MdQ5BDU1C561D18pDSF+iD+9HL/07DEpE3Tyl1ccx7ngYKk+hP13mxnTiYqQweAldXga7tqJSxqFsNqvjeB2VPBZOV7juqoRP0BWnMBf9P4iwY/zbk00+bL4U1bc/yjEK/ely9MlyN6YUFyKFwUvoTatBm6jr0qyO4p2uSYAuXTGlO8knaLMe83/nwKlyjMdmNXqzubXU5Aehvg794VI3JBSXIoXBC2it0bnZ0G9Aix7MBRJls6FGjoFd+eiKU1bHEU3QHy6FvdtR9z2KuryfW46pevREjZ6IXrcS/W3rVmwUzSOFwRsc3g/HilApMkT1UlTKeNdfjFvWWB1FXILetRX94VJU8jjU6IluPba65ScQFIxe/rpbjysak8LgBXTuZ9Chg2v0jbgo1fsKuLyfvOzkxXRZCWbmXOh1OeqBx9z+kqaK6Ia64XZ0/nr04QNuPbb4nhQGi+naGnTeOtTQZFRIqNVxvJ66Ls016+ZXB62OIn5E19Vh/u05qK9zPVfw0KqDasIdENYF891X5KVHD5HCYDFdsAWqTks3UjOpEde7uhLkrsHr6PeWwKF9qId/hYr23LMy1TnE1aX0xU5ZyMlDpDBYTG/Mhm5RcPW1VkfxCSo0zPVOw+Y16Noaq+OIc/TubehP3kGNnoAxfLTHz6euvxGiemC+87Is5uMBQc3ZqKCggMWLF2OaJuPHj2fy5MmNvl5bW0tGRgaHDh0iPDyc9PR0oqOjAVi2bBnZ2dkYhsG0adNISEigtLSUhQsXcuLECZRSpKWlcfPNNwPw5ptv8tlnn9GlSxcA7rvvPoYNG+bONnsNfcIJu7ehbroLZci7C82lrktzvexWsAU1fJTVcQKePuHE/Mc813OFn/ysXc6pgoNRkx9E/+8cV1fsyDHtct5A0WRhME2TzMxMfve73xEZGclvf/tbHA4HvXv3btgmOzub0NBQFixYwIYNG1iyZAlPPPEERUVF5ObmMnfuXMrLy/njH//IX/7yF2w2Gw899BBxcXGcOXOGWbNmce211zYcc9KkSdx2222ea7WX0JtzXO8uJI+1OopvueZasEehc7NACoOltFnvethcfQbj0dkee65wIWr4aPTKd9HLXkMPS0EFB7fbuf1dk11JhYWFxMTE0KNHD4KCgkhJSSEvr/Ei3fn5+aSmpgKQlJTE7t270VqTl5dHSkoKwcHBREdHExMTQ2FhId26dSMuLg6Azp0706tXL5xOp/tb58Ua3l2IvxoV07vpHUQDZdhcs8/u2Y52llodJ6DpFW/BFztR9/0c1bNPu55bGQbGXVNd03LnrGjXc/u7Ju8YnE4nkZGRDR9HRkZy4MCBi25js9kICQmhoqICp9NJ//79G7az2+3nFYCSkhIOHz5Mv37fvwSzcuVK1q5dS1xcHA8//DBhYecvb5mVlUVWlusB5LPPPktUVFRz2nueoKCgVu/bFrWFn+M8+hXhv/hPQtr5/Fa12Z3qJt1N2UdvErJzM6F3/7TJ7f2hzS3l6TbX7NlO+Qdv0On6CXS5/V5r1g8ZcwPlOR9Ru+JN7LfcI9fZXcd069Fa6OzZs8yZM4epU6cSEuKan33ChAncfffdACxdupRXX32VGTNmnLdvWloaaWnfTx9RWtq6vxyjoqJavW9bmCvegaBgTl+TQFU7n9+qNrtVcCe4chCVq96naszNTf5S8os2t5An26wrT2E+/3uI6kHNPdMpKyvzyHmaleX2B9H/nU7pqy8S/ctZcp1boGfPC48ea7IryW63N7roZWVl2O32i25TX19PVVUV4eHh5+3rdDob9q2rq2POnDmMHj2akSNHNmzTtWtXDMPAMAzGjx/PwYP+N15d19ait6xDDU1ChZx/NySaR12XBiXFcGCv1VECitYa85UMqDyJ8eh/ojo1c9EdD1G9r0CNSkOvXkHd0a8tzeIvmiwM8fHxFBcXU1JSQl1dHbm5uTgcjkbbJCYmkpOTA8CmTZsYOHAgSikcDge5ubnU1tZSUlJCcXEx/fr1Q2vNokWL6NWrF7fcckujY5WXfz9z4pYtW4iNjXVDM73Mzjw4XSHvLrSRSkyBTp1dq96JdqPXr4KCTag7HkJdHm91HADU7Q9AUDCVr75odRS/0GRXks1mY/r06cyePRvTNBk7diyxsbEsXbqU+Ph4HA4H48aNIyMjg5kzZxIWFkZ6ejoAsbGxJCcn8+STT2IYBo888giGYfDFF1+wdu1a+vTpw69//Wvg+2Gpr7/+OkeOHEEpRffu3Xn00Uc9+x2wgJn7GXS1w4AEq6P4NNWxk2tkypa16Pt+ZvlfroFAf3sU/cbf4ZohqLTbrY7TQEV0Q914J9XvLcG4fiLqykFWR/JpSvvJO+VHj7ZutsX27nvWp05g/noq6obJGHdPbbfz/pA/9bfrg19gPvufqJ/OxBh1w0W386c2N5e726zr6jD/32+gpBjjmRdQdu96yKurq+GZGZhhERhPPd+m9R98iSXPGIR76bz1YJqu4Zai7eKugphe6A2fWZ3E7+kP34AjBzAemuF1RQFAdexI2AO/gC8L0VvWWh3Hp0lhaGd6cw7E9kX1at8x3/5KKeV6CF24F32syOo4fksf2Ite8bZrKm2H975U2GnMROgTj172Krqm2uo4PksKQzvSx76Bw/tRI1OtjuJXVPI4sNnQ61ZZHcUv6arTrrebI7uj7vPuZ37KMDCmTAdnKTrrfavj+CwpDO1Ib14DSrlmCBVuoyK6wbXD0Ruz0XW1VsfxO/pffwNnKcYjT6I6e/8DfnXVYBgyAv3x2+hTsj50a0hhaCdaa1c30tXXorpFNrm9aBlj9ESoOAkFm62O4lfMvPXoTatRk6ag+l1jdZxmM+6eCrU16Pf+aXUUnySFob0c2gfHj0k3kqcMTAB7d8x1n1qdxG/ok+XoJX+FK/qjJk2xOk6LqJjeqNSb0es+RX91yOo4PkcKQzvRm3MguANqWLLVUfySMmyoUTfA3gL08WNWx/F5WmvM1xZC9VmM6emoIEtnz2kVdet9EBqO+cbfZKW3FpLC0A50XZ1rzviEkT7RR+ur1HVpoAzXm7miTfTG1bBjC+qOB1GX+ebsAyo0DDX5QTiwF52/3uo4PkUKQ3vYsx0qK6QbycOUPQoGDUNv+AxdX291HJ+lnaWut5v7DUCl+fa6KGr0DRDbF/32YtcLcKJZpDC0A705B8LCYeBQq6P4PeP6iXDSCbvymt5YnEdrjfnqAqivw5j27z6/sqAybBj3PuoavrryHavj+AwpDB6mz1ShCzajHKN9sp/W5wx2QFc75lp5CN0aet2nsGc76u6pqOjLrI7jFurKga45tT55F11WYnUcnyCFwcP09o1QW4NKSrU6SkBQNhsqJQ12b0M7j1sdx6fo0m/Rb/7DNaR6zE1Wx3ErdfdUUKDfWmx1FNipEk0AACAASURBVJ8ghcHD9KYc6B7jmtNHtAs1Kg20iV4v03E3lzZNzJdfAAXG1Mf9bgI6Ze+OuvFu9NYN6H27rY7j9fzr6nsZfaLMtR7uyFRrlj0MUKp7DAwYit6wCm3KQ+jm0DkrYN8u1JRHUJHRVsfxCDXhDte7Lm/8XX4umiCFwYP0lrWgNWrkGKujBBzj+gngLHWNCBOXpEu/Rb/7Kgwc6noXxE+pjh0x7pkGRYfR8gzqkqQweJDevMb11mhML6ujBJ4hIyA8Qh5CN8H1ItuLgMJ46Jf+f2ebeB1cOQj93uvoylNWp/FaUhg8RB8rgq8OoUbKhHlWUEHBrqVTd25Bn3BaHcdr6Y2rYe921F0P+20X0g8ppTDuexSqTqOXvW51HK8lhcFD9JZ1rplUvXjuen+nRk8E05Q3oS9CnypHL/1f6HeN341CuhTV+wrUuFvR61aiDx+wOo5XksLgAVprdN46uHIQqqvMpGoV1aMnXDPE9QtA3oQ+j/7n36CmGuOnM/1uFFJT1G33QZdumEv+Kg+iLyCwfhray9eH4VgRavhoq5MEPCP1JnCWUr011+ooXkVv24jeugF1y09QMb2tjtPuVOcQ1D3TXMuAynOo80hh8ACdtw5sNtSwFKujiCEjoaudM5+8a3USr6FPV2L+cxH07ouaeKfVcSyjRlwPVw1GL3sNXXHS6jheRQqDmzV0I12TgArvYnWcgKdsNtToidRs34wuKbY6jlfQby+GipOuF9kCeJoWpRTG/T+H6jPod162Oo5XadZPRUFBAYsXL8Y0TcaPH8/kyZMbfb22tpaMjAwOHTpEeHg46enpREe7RjgsW7aM7OxsDMNg2rRpJCQkUFpaysKFCzlx4gRKKdLS0rj55psBqKysZN68eRw/fpzu3bvzxBNPEBYW5uZme9ChfVBWgrrtfquTiHPU6Anoj95Er/0Edfc0q+NYSn++A71+FerGu1CXx1sdx3KqZx9U2u3ole+iR92A6jfA6kheock7BtM0yczM5KmnnmLevHls2LCBoqKiRttkZ2cTGhrKggULmDRpEkuWLAGgqKiI3Nxc5s6dy9NPP01mZiamaWKz2XjooYeYN28es2fPZuXKlQ3HXL58OYMHD+aFF15g8ODBLF++3APN9hydtw6CglFDk6yOIs5R3SLpOGI0ekMWurbG6jiW0TXVrsV3oi9D3Xqv1XG8hrrlJ9AtCnPJSzJI4ZwmC0NhYSExMTH06NGDoKAgUlJSyMtrPKVxfn4+qampACQlJbF792601uTl5ZGSkkJwcDDR0dHExMRQWFhIt27diIuLA6Bz58706tULp9M11jwvL48xY1xvCo8ZM+a8c3kzbda7FgS51iEL8niZzjfeAZUV6K0brI5iGb3iLTh+DOPBGagOHa2O4zVUp84YP3nE9UZ0zgqr43iFJruSnE4nkZHfD7mMjIzkwIEDF93GZrMREhJCRUUFTqeT/v37N2xnt9sbCsB3SkpKOHz4MP369QPg5MmTdOvWDYCuXbty8uSFHwplZWWRleWaJO3ZZ58lKiqqycZeSFBQUKv3/bGanfmUnywnYtwkOrnpmJ7gzjb7Clt0NBU9+2CsX4X9lnusjtMufnid674+QtnKd+k0ZiIRo8dbnMxzWvuzrSfcxolNq6l9/590u+FWbHbf+f/hif/Plj55Onv2LHPmzGHq1KmEhJz/F7ZS6qKv6KelpZGWltbwcWlpaasyREVFtXrfHzOzPoSOnam44ioq3XRMT3Bnm31FVFQU5qgbqH8zk+Pb81Cxfa2O5HHfXWetNWbG/0CHTtTc9oBfX/u2/Gzru6ej//AryhY9h/Hor92czHPa0uaePXte8PNNdiXZ7XbKysoaPi4rK8Nut190m/r6eqqqqggPDz9vX6fT2bBvXV0dc+bMYfTo0YwcObJhm4iICMrLywEoLy+nSxffGNmj62rR2zaiEkagOsptujdSKeMguAM652Oro7QrnZsN+3ej7vopqktXq+N4LdWjJ+qme9B569C7t1odx1JNFob4+HiKi4spKSmhrq6O3NxcHA5Ho20SExPJyckBYNOmTQwcOBClFA6Hg9zcXGpraykpKaG4uJh+/fqhtWbRokX06tWLW265pdGxHA4Ha9asAWDNmjUMHz7cTU31sL0FcLoCNVzmRvJWKjTctZLX5hz0mSqr47QLXXEK/fY/XNNe+PHMqe6ibrobYnpjvv5XdPVZq+NYxvaHP/zhD5fawDAMYmJiWLBgAZ988gmjR48mKSmJpUuXcvbsWXr27EmfPn1Yv349//znPzly5AiPPvooYWFhREREUFlZyUsvvcT69euZPn06PXv2ZN++fbz88stUV1ezatUqVq1aRVRUFJdddhlxcXG89957vPPOO1RWVjJt2jQ6dOjQZEMqKipa9Q0ICQmhqqrtvyT0h0vBWYp68DGvXyfXXW32JQ1t7mp3PWDsFoXq27/pHX1YSEgIpzPnwpEDGDP/CxXRzepIHtfWn21ls6F6X47Oeh/q61ADvH+d9ra0OTw8/IKfV1pr3ZZQ3uLo0aOt2s8d/e26phrzyYdRI0ZjPPyrNh2rPQTqM4aG/vb/+yTU1WL8YYFfTzPd5duvKf/dL1E33oVx10+tjtMu3PWzbb6agd6QhfH0XFSfODck8xxLnjGIZtiVD9VnZG4kH6CUQqXeBEe/ggN7rY7jMbqullOLnoPIaNQt8s5CS6m7pkJouKtABOAke1IY3MDMWwfhEXDlIKujiGZQI8ZASCh69UdWR/EYvXIZ9UVHMB74hQyGaAUVGob6yb+5JtlbHXjvNkhhaCNdfRZ25aMSU1A27362IFxUx46oUTegt+Winf7XpabLStAr3qRjcipqsKPpHcQFqRHXw8Ch6GWvo53HrY7TrqQwtNWufKipQSVeZ3US0QIq9WbQGr3mE6ujuJ35ZiagCJ/2uNVRfJpSCuOBx0DXY/7rb1bHaVdSGNpI528414000OooogVU9xi4drhrER8/mj9J79kO2zaibr4HW/cYq+P4PNU9BnXrfVCwGb1to9Vx2o0UhjbQ1dXoXfmoYcleP0RVnM8YdwtUnHRNfOgHdF0t5ht/c02SN+EOq+P4DZV2O/S+AvNfL6GrTlsdp11IYWiL3flQUy3dSL7qmiFwWSw6+yP8YdS2znofjn2Dce+jqOBgq+P4DRUU5BqGfvJEwKzbIIWhDb7vRpLRSL5IKYUaNwm+LHSto+HDdHmZ6yXLISNQgxOtjuN3VN8rURNuR69dif58h9VxPE4KQyvp6mr0zjzU0GQZjeTDVNJY6ByK/uwDq6O0iX7rH1Bfj/GTf7M6it9St90P0T1d7zacPWN1HI+SwtBau7e6upEc0o3ky1Snzqjr0lxDV0+UNb2DF9L7dqHz1qFuusv1UF14hOrQEWPq41BWgl7+utVxPEoKQyvprRsgrIt0I/kBNfZmME30mpVWR2kxXVfnGkoZGY268S6r4/g91X8AauwkdPaHaD9+c14KQyvomnPdSMOkG8kfqOjLYFAieu0n6Npaq+O0iM75CL75EuPef5NV2dqJuuMhsHfHfGUBuqba6jgeIYWhNXZvheqzMhrJjxjjb4VTJ9Bb11sdpdl0xUn0+/+CgUNhyMimdxBuoTp1do1S+vYb1/ffD0lhaAWdf64b6arBVkcR7nLNEIjphc72nfmT9HtLoPosxk9+5tezxHojNSABNXoC+tPl6MMHmt7Bx0hhaKGGbqShSdKN5EeUYaDGToLD+9E+MHRVFx1Br/0UNXYS6rLeVscJSOruaRDRDfOVF9B1vtUF2RQpDC21e5urG0lGI/kdlTIOOod4/dBVrbVrPqTOIahbZUptq6iQUIwHZ8A3X6I/esvqOG4lhaGF9NZcCAuHq661OopwM9UpxDXrav56755Nc2cefL4Dddv9qNALr8Al2ocaMhyVNBb98VvoLwutjuM2UhhaQNfWoHdukZfa/JgafytovPZZg66rxXzzHxDTGzXmRqvjCEDd+zMI74qZOc9vJmSUwtASe7bB2TMyGsmPqchoVGKKa+oDL3y7Va9eASVHMaY8ggoKsjqOwLWoj/HTmVD8td+8+CaFoQX0to0QEiajkfycSrsNzpxGb/jM6iiN6IpT6A/egEHDZD4kL6MGDUONuRG96j30/t1Wx2kzKQzNpOvq0Du2uPoU5S81v6bir4b4q9Gfve9V6/3q9/8J1Wcw7pludRRxAeruaRDVA3PxX9Bnq6yO0yZSGJpr/26oOo0almx1EtEOjBtuh+PHYEee1VEA0N98iV7zCSr1ZlTPPlbHERegOnXGmJ7umkvpzX9YHadNpDA0k96+ETp0hAFDrY4i2kNCEkRGY65abnWSc8NT/yHDU32A6jcANeEO9LpP0bvyrY7Tas3qEykoKGDx4sWYpsn48eOZPHlyo6/X1taSkZHBoUOHCA8PJz09nejoaACWLVtGdnY2hmEwbdo0EhISAHjxxRfZtm0bERERzJkzp+FYb775Jp999hldunQB4L777mPYsGFuaWxradNEb98MgxJlPpoAoWw21Phb0W9moo8cQF3R37owe7bD3u2onzyCCutiXQ7RLOr2B9C7t2K+koHx/y3wySHFTd4xmKZJZmYmTz31FPPmzWPDhg0UFRU12iY7O5vQ0FAWLFjApEmTWLJkCQBFRUXk5uYyd+5cnn76aTIzMzFNE4DU1FSeeuqpC55z0qRJPPfcczz33HOWFwUADu+Hk07pRgowatQN0KkzetX7lmXQZj3mOy9D9xhU6s2W5RDNp4KDXV1KlSfRSxZZHadVmiwMhYWFxMTE0KNHD4KCgkhJSSEvr3G/a35+PqmpqQAkJSWxe/dutNbk5eWRkpJCcHAw0dHRxMTEUFjoeglkwIABhIWFub9FHqC3bQRbEGqww+oooh2pziGoURPQW9ejnaWWZNAbc6DoCOqOh1FBslynr1B94lG33IvOW4fpg2uKN9mV5HQ6iYyMbPg4MjKSAwcOXHQbm81GSEgIFRUVOJ1O+vf//hbcbrfjdDqbDLVy5UrWrl1LXFwcDz/88AULSFZWFllZWQA8++yzREVFNXncCwkKCrrkvlprynZuwXatg259Lm/VObxNU232R61tc/09D1Oa/QGdNmUT/vAMDyS7OF1dTekH/8TW7xrsN97e4ony5DpbSz/0c8o/L6BuySK6OZKxeWgRJU+02evGXU6YMIG7774bgKVLl/Lqq68yY8b5/yHT0tJIS0tr+Li0tHV/0UVFRV1yX110GPPYN5g3TG71ObxNU232R61usxEMQ5OoWrmMs+NuRXXq7P5wF2F+/Da67DhMe4KyspavLifX2Xr6p4+j/zud0uf/C+P//F+U4f4ZE9rS5p49e17w8012Jdnt9kY/lGVlZdjt9otuU19fT1VVFeHh4eft63Q6z9v3x7p27YphGBiGwfjx4zl48GBTET1Kb9sESqESRliaQ1jHuGEyVJ1G57bfC2+64hT647dhyAjUVbJKoK9S0Zeh7n8U9u9Bf/yO1XGarcnCEB8fT3FxMSUlJdTV1ZGbm4vD0bivPTExkZycHAA2bdrEwIEDUUrhcDjIzc2ltraWkpISiouL6dev3yXPV15e3vDvLVu2EBsb24pmuY/evhH6XYPq0s3SHMI6DS+8rXoPXd8+L7zpj5bC2bMYdz7cLucTnqOSx6GGj0Z/8C/04f1Wx2mWJruSbDYb06dPZ/bs2ZimydixY4mNjWXp0qXEx8fjcDgYN24cGRkZzJw5k7CwMNLT0wGIjY0lOTmZJ598EsMweOSRRzAMVy2aP38+e/fupaKigl/84hdMmTKFcePG8frrr3PkyBGUUnTv3p1HH33Us9+BS9Alxa4Hf1MesSyD8A7GxDsxX/wf9LZc1PDRHj2XLjmKzlmBGn2DvMzmB5RS8OBj6INfYP79eYzf/6VduyRbQ2mttdUh3OHo0aOt2u9S/XPmymXotxdj/OnvqKgebYnnVbytH7Y9tLXN2jQxf/9L6NgJ43dzPbpimvnSn9E78zBmv4Tqeumu10uR6+xd9P7dmM//DpUyFmPqv7vtuJY8YwhkevtG6BPnV0VBtI4yDNTEO+Crg/DFTo+dRx/ah85fj5pwR5uKgvA+6spBqJvuRm/4DJ3v3WuLS2G4CH3CCQe/QA2Vl9qEi0oa61rK8RPPPETUWrteZguPQE2c3OT2wveoW++FvldivrbQNeLMS0lhuAhdsBlA3nYWDVRwMGr8bbC3AP2VB0bL7cqH/XtQt96H6hTi/uMLy6mgIIx/exLqTcx/zPWq2Xt/SArDRejtG6FHL7jM2lFRwruoMRNd02SsXObW42rTxFz2umvqi9E3uPXYwruo6J6o+84NYfXStaKlMFyAPl0J+3ahhiV59CGj8D0qJAx1/Y2udaGPH3PbcXX+eig67FrHWaa+8HsqZRxq5Bj0B2+g9+2yOs55pDBcgN6ZB/X18nxBXJAafysoA53lnsn1dF0d+r0l0Oty1AjPDoUV3kEphXrwMYi+DPPvc9CnTlgdqREpDBegCzZD10i4/NIv44nApOxRrr/21n+KrjjV5uPp3M+gpBhj8oMemTJBeCfVKQTj5/8JpyswM+ehz8087Q2kMPyIrq2BPdtQCSNQhnx7xIWpiXdATQ06Z0WbjqNra1zrOMddBUNk2pVAo2L7ou79Gezdjl75rtVxGshvvh/7YhdUn0XJf1JxCapnHxgyAp39Ibq6utXH0atXwIkyjDsekudZAUpdP9E1Zcby19EH9lodB5DCcB5dsBk6doarrrU6ivByxsQ7ofIUesOqVu2vz1ShP34LBiSgrpaft0CllEI99EvXUrJ/fx5d2fbuybaSwvAD2jTRO7fAoKGoYBkZIprQ7xrX5HqfLkfX1bV4d73qPaiswJj8kAfCCV+iOodg/Pw3UHEC8x/zsXqmIikMP/TVQTjhRA0ZaXUS4QOUUhg33wNlJegta1q0r644hV61HIYlo/pauJ608Brq8njUPdNhV77rZ8NCUhh+QBdsBsNADU60OorwFYMdENsX/fHbLXqLVX/yNlRXY9z+gAfDCV+jxk6CYSnod1+19HmDFIYf0Du2QL8BqLAuVkcRPqLhruHYN7BtY7P20eVl6OyPUEmpMq22aEQphfHTmRDZwzXL7snypnfyACkM5+jjx1xrL8hoJNFSw5IhphfmR281q29Yf/wWaNM1oZoQP6JCQjEemwVnKjH/9udWPb9qKykM5+ideQCoBHm+IFpGGTbUTXdD0WHXRHiXoJ3H0es+RaWMR3locXjh+1TvK1AP/co1n9KyV9v9/FIYztE7tsBlsajoy6yOInyQGjHGNdxwxaXvGvTHb4MGNWlKO6YTvshISkWNneQa9bZ1Q/ueu13P5qV0VSXs341KkG4k0ToqKAg18U44+AVcZFI0XXYcvW4ValQaKjK6nRMKX6SmTIf4qzEXv4AuLmq380phAPSura5J82SYqmgDNSrNtZDPigtPpaxXvAkK1M33tHMy4atUUDDGo/8JHTpg/vVP6LNn2uW8UhgAdmyB8Ajoe6XVSYQPU8EdUDdMhs93oA/ta/Q1XfotekMWatQElL27RQmFL1L2KIyf/Qcc+wb9yoJ2efkt4AuDrq1F796KGiKT5om2U2NuhNDw8+4a9Iq3QCnXQ2ohWkhdMwR1x0OudUDcNN37pQT8b8KavQVwpkpGIwm3UJ06u9Zr2LEFXXQYcA2F1rmfoa6/EWWPsjih8FXqxjthaBL67cXoz3d49FwBXxiqt6yDDh3g6iFWRxF+Qo27xbX854q3AdAfLQXDhrrpLouTCV+mlMKYng49ernebyj91mPnCmrORgUFBSxevBjTNBk/fjyTJ09u9PXa2loyMjI4dOgQ4eHhpKenEx3tGnWxbNkysrOzMQyDadOmkZCQAMCLL77Itm3biIiIYM6cOQ3HqqysZN68eRw/fpzu3bvzxBNPEBYW5q72NqK1pjpvHQwYiurY0SPnEIFHhYa5hhl+8g56ZCp642rU2EmorpFWRxM+TnUKwfjV05iz/w/mwv/BmPVnj5ynyTsG0zTJzMzkqaeeYt68eWzYsIGiosbDprKzswkNDWXBggVMmjSJJUuWAFBUVERubi5z587l6aefJjMzE/PcKkWpqak89dRT551v+fLlDB48mBdeeIHBgwezfLkHJ5P6+jDm8W/lbWfhduqGydChI+aiP4EtCHWj3C0I91DRPV0Po785gn7lBY88jG6yMBQWFhITE0OPHj0ICgoiJSWFvLy8Rtvk5+eTmpoKQFJSErt370ZrTV5eHikpKQQHBxMdHU1MTAyFhYUADBgw4IJ3Anl5eYwZMwaAMWPGnHcud9I7trgeCF7r8Ng5RGBS4V1QqTdBXR0q9SZUV7vVkYQfUYMSXQ+j89ZRveEztx+/ya4kp9NJZOT3t8CRkZEcOHDgotvYbDZCQkKoqKjA6XTSv//3Uwrb7XacTuclz3fy5Em6desGQNeuXTl58uQFt8vKyiIrKwuAZ599lqiolj/UOxN7OXVptxIeF1jTHgcFBbXq++XLrGizef+jVAbZCJvyCEaXiHY9N8h19nf6wZ9z9vI4QkffQCc33zU06xmDVZRSF13uMC0tjbS0tIaPS0tLW36ChGSi0m5t3b4+LCoqStrcXiY/jLOmFiw4t1znADBgGJ21bnWbe/bsecHPN9mVZLfbKSsra/i4rKwMu91+0W3q6+upqqoiPDz8vH2dTud5+/5YREQE5eWuqWbLy8vp0kWmwBZCiPbUZGGIj4+nuLiYkpIS6urqyM3NxeFo3CefmJhITk4OAJs2bWLgwIEopXA4HOTm5lJbW0tJSQnFxcX069fvkudzOBysWeNaDWvNmjUMHz68lU0TQgjRGko345H2tm3beOWVVzBNk7Fjx3LnnXeydOlS4uPjcTgc1NTUkJGRweHDhwkLCyM9PZ0ePXoA8O6777J69WoMw2Dq1KkMHToUgPnz57N3714qKiqIiIhgypQpjBs3joqKCubNm0dpaWmLhqsePXq0Vd+AgLv1RNocKKTNgaEtbb5YV1KzCoMvkMLQfNLmwCBtDgyeKAwB/+azEEKIxqQwCCGEaEQKgxBCiEakMAghhGjEbx4+CyGEcI+Av2OYNWuW1RHanbQ5MEibA4Mn2hzwhUEIIURjUhiEEEI0YvvDH/7wB6tDWC0uLs7qCO1O2hwYpM2Bwd1tlofPQgghGpGuJCGEEI1IYRBCCNGIVy/U42kFBQUsXrwY0zQZP348kydPtjpSm5WWlrJw4UJOnDiBUoq0tDRuvvlmKisrmTdvHsePH280a63WmsWLF7N9+3Y6duzIjBkzfLaP1jRNZs2ahd1uZ9asWZSUlDB//nwqKiqIi4tj5syZBAUFUVtbS0ZGBocOHSI8PJz09HSio6Otjt9ip0+fZtGiRXz99dcopXjsscfo2bOnX1/nDz/8kOzsbJRSxMbGMmPGDE6cOOFX1/nFF19k27ZtREREMGfOHIBW/f/Nycnh3XffBeDOO+9sWH65WXSAqq+v17/61a/0sWPHdG1trf6P//gP/fXXX1sdq82cTqc+ePCg1lrrqqoq/fjjj+uvv/5av/baa3rZsmVaa62XLVumX3vtNa211lu3btWzZ8/Wpmnqffv26d/+9reWZW+rDz74QM+fP1//6U9/0lprPWfOHL1+/XqttdYvvfSSXrlypdZa608++US/9NJLWmut169fr+fOnWtN4DZasGCBzsrK0lprXVtbqysrK/36OpeVlekZM2bo6upqrbXr+q5evdrvrvOePXv0wYMH9ZNPPtnwuZZe14qKCv3LX/5SV1RUNPp3cwVsV1JhYSExMTH06NGDoKAgUlJSyMvLszpWm3Xr1q3hL4bOnTvTq1cvnE4neXl5jBkzBoAxY8Y0tDU/P5/rr78epRRXXnklp0+fblhBz5eUlZWxbds2xo8fD4DWmj179pCUlARAampqozZ/99dTUlISu3fvRvvYGIyqqio+//xzxo0bB7jWOg4NDfX762yaJjU1NdTX11NTU0PXrl397joPGDDgvDVoWnpdCwoKuPbaawkLCyMsLIxrr72WgoKCZmcI2K4kp9NJZGRkw8eRkZEcOHDAwkTuV1JSwuHDh+nXrx8nT56kW7duAHTt2pWTJ08Cru/DDxdPj4yMxOl0NmzrK15++WUefPBBzpw5A0BFRQUhISHYbDbAtfys0+kEGl97m81GSEgIFRUVPrWMbElJCV26dOHFF1/kyy+/JC4ujqlTp/r1dbbb7dx666089thjdOjQgSFDhhAXF+fX1/k7Lb2uP/799sPvS3ME7B2Dvzt79ixz5sxh6tSphISENPqaUgqllEXJ3G/r1q1ERET4ZJ95a9XX13P48GEmTJjAn//8Zzp27Mjy5csbbeNv17myspK8vDwWLlzISy+9xNmzZ1v0V7C/aI/rGrB3DHa7nbKysoaPy8rKsNvtFiZyn7q6OubMmcPo0aMZOXIkABEREZSXl9OtWzfKy8sb/mqy2+2NVn/yxe/Dvn37yM/PZ/v27dTU1HDmzBlefvllqqqqqK+vx2az4XQ6G9r13bWPjIykvr6eqqoqwsPDLW5Fy0RGRhIZGUn//v0BV1fJ8uXL/fo679q1i+jo6IY2jRw5kn379vn1df5OS6+r3W5n7969DZ93Op0MGDCg2ecL2DuG+Ph4iouLKSkpoa6ujtzcXBwOh9Wx2kxrzaJFi+jVqxe33HJLw+cdDgdr1qwBYM2aNQwfPrzh82vXrkVrzf79+wkJCfGp7gWA+++/n0WLFrFw4ULS09MZNGgQjz/+OAMHDmTTpk2Aa4TGd9c3MTGRnJwcADZt2sTAgQN97i/rrl27EhkZ2bCk7a5du+jdu7dfX+eoqCgOHDhAdXU1WuuGNvvzdf5OS69rQkICO3bsoLKyUQmWAAAAAShJREFUksrKSnbs2EFCQkKzzxfQbz5v27aNV155BdM0GTt2LHfeeafVkdrsiy++4Pe//z19+vRp+E9w33330b9/f+bNm0dpael5w90yMzPZsWMHHTp0YMaMGcTHx1vcitbbs2cPH3zwAbNmzeLbb79l/vz5VFZW0rdvX2bOnElwcDA1NTVkZGRw+PBhwsLCSE9Pp0ePHlZHb7EjR46waNEi6urqiI6OZsaMGWit/fo6v/nmm+Tm5mKz2bjiiiv4xS9+gdPp9KvrPH/+fPbu3UtFRQURERFMmTKF4cOHt/i6Zmdns2zZMsA1XHXs2LHNzhDQhUEIIcT5ArYrSQghxIVJYRBCCNGIFAYhhBCNSGEQQgjRiBQGIYQQjUhhEEII0YgUBiGEEI38/6cUaXkSH5aaAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "gdwJu6i3wwdP", + "outputId": "3501bcf8-9895-44e1-e6bd-c6b2051e69f7" + }, + "source": [ + "plot_lr(cosine(t_max=500, eta_min=0.0005))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU1f3H8fe5kwWyEJgJIYBRIcSFTSpBQhQJJKCCKCJal7oArQuKjdRfq9haa0ulWhYlWKiNiIiCVXYVNcZgYYwEEWRRJOAWDQYyCAlhSXLP74+R1JQl2yR35s739Tx9nk5y753PyR38zjn33nOU1lojhBBC/MiwOoAQQgj/IoVBCCFELVIYhBBC1CKFQQghRC1SGIQQQtQihUEIIUQtIVYH8JXvvvuuUfvFxsayb98+H6fxb9Lm4CBtDg5NaXOnTp1O+nPpMQghhKhFCoMQQohapDAIIYSoRQqDEEKIWqQwCCGEqKVedyVt2rSJefPmYZom6enpjBo1qtbvKysrycrKYvfu3URHR5OZmUlcXBwAS5cuJTc3F8MwGDt2LH369AHgmWeeYePGjcTExDBt2rSaY5WXlzNjxgz27t1L+/btuf/++4mKivJVe4UQQtShzh6DaZpkZ2czefJkZsyYwbp16ygqKqq1TW5uLpGRkcyaNYsRI0awcOFCAIqKinC73UyfPp2HH36Y7OxsTNMEIC0tjcmTJ5/wfsuWLaNXr148/fTT9OrVi2XLlvminUIIIeqpzsJQWFhIfHw8HTp0ICQkhNTUVAoKCmpts2HDBtLS0gBISUlh69ataK0pKCggNTWV0NBQ4uLiiI+Pp7CwEIDu3buftCdQUFDAoEGDABg0aNAJ7+VL5gfvUf7KPMycFZjud9Gfb0Mf/AGZiVz4gj5Ujvnmq5hvLcV8fzV6ywZ0STHarLY6mhCnVedQksfjweVy1bx2uVzs3LnzlNs4HA4iIiIoKyvD4/GQlJRUs53T6cTj8Zz2/Q4cOEC7du0AaNu2LQcOHDjpdjk5OeTk5AAwdepUYmNj62rKCfZv/pBDH7lrXh8vB4YrjtBeFxLWux/hFw3EiLTXUFZISEij/l6BzIo2H95awMElL9S8Pv75Uq1aE3J+b8J69SW8/yBCOiU0y/vLeQ4OzdFmv37yWSmFUuqkv8vIyCAjI6PmdaOe/LvrQeLatmXft9/AoTL4vhi9pwi96zOObHBzJG81hITCBf0wLr0Mzu9zyjyBRJ4ObRnmgYMAGI/OgtaRUFqC3lME3+zm2GdbOPbxh5S/8AycnYRKTUelDkGFt/LZ+8t5Dg7N8eRznYXB6XRSWlpa87q0tBSn03nSbVwuF9XV1VRUVBAdHX3Cvh6P54R9/1dMTAz79++nXbt27N+/nzZt2tQVsUlUSAgqMhoioyGuE6pXXxh6Ndo04cud6PXvo9e/j/mRGxK6oC6/FpV8CcqQG7pEXX7sI7RqjXLGgjMWldT9v7/17ENv+A86Pw/90hz08oWowcNRGVejbNZLFYGlzv+6JSYmUlxcTElJCVVVVbjdbpKTk2tt07dvX/Ly8gDIz8+nR48eKKVITk7G7XZTWVlJSUkJxcXFdOvW7bTvl5yczJo1awBYs2YN/fr1a2TTmkYZBqrruRg3/Arjb8+hbpsIVVXoZ/+OOeU36B1bLMkl7EM5YzGGXYPjkacwfjcVkrqjX38F8+E7MXNWoKsqrY4ogpTj0UcfffR0GxiGQXx8PLNmzWL16tUMHDiQlJQUFi9ezJEjR+jUqRNnnnkma9eu5aWXXuLLL7/kjjvuICoqipiYGMrLy5k7dy5r165l3LhxNV2XmTNnsnjxYkpLS8nJySEiIoIuXbrQtWtXli9fzmuvvUZ5eTljx44lLCyszoaUlZU16g8QERFBRUXFabdRDgfqzETUoCugQ0f4ZD363VXw/bdwTg+fdv9bQn3abDdWtFl/sxs2fejtAUREnnZb5WyPcdGlqD4p6G+/hLw30R+5UWclensbjSDnOTg0pc3R0dEn/bnSNrkFpyVnV9XHjqJXv4Z+41VoHYFx812o5Esa9f5WkHHYlmGufQc9fxbG1GyUq32999Naw5YNmAvnwP59qPSRqNG3okLr/oL0U3Keg4PMruonVFg4xlU3YfxhBrjiMOc+gbnwH+jKY1ZHEzaglEL17ofxp1moQVegc1ZgPv5/6JLGffkRoqGkMDSB6nwWxoNPoC67Bp33JubU36L37rE6lvAXxzvjjbyRTbXy9kaNiX+A0r2Yf74f/dE63+UT4hSkMDSRCgnBGDMW497fw74SzL8+gC781OpYwkZU734YjzwFnc7EnPM3zNdfkYcwRbOSwuAj6oKLMCb/HSIiMaf9HrNgrdWRhN9o+rMvytUe44G/olLS0MteRM9/Wu5aEs1GCoMPqQ6dMB58Es5OQv/zCcx3V1kdSdiICg1FjbsfNfIG9Lp3MbP+gj561OpYwoakMPiYim6DMenP8LMU9KJ/Yq5+zepIwkaUUhhX3eR9rmb7JsxZj6GPHLY6lrAZKQzNQIWGYtzxW1S/gejX5mOueMnqSMIKNReffT+NinHJUNT4SbBzG+bMP6IrDvn8PUTwksLQTFRICOqXk1Cp6eiVizDffNXqSMJmjP6DMO78LXy5E3PWn2VYSfiMFIZmpAwH6raJqIsuRS95ATPvTasjCSs047yL6sJUjF/+BnZ9hjnncbkgLXxCCkMzU4aBGpsJvZLRL83B/HCN1ZFEi2mZW0pV8iWoWybA1o3of02X9R5Ek0lhaAEqJATjrt9BUg/0vJnoTzdbHUnYjDFwGOq6seiP1qEXPWt1HBHgpDC0EBUWjnHPw9ChM+Y/pqKLv7E6kmhuNR2GllnDwxh2DWrYKPR7b8it0qJJpDC0IBUR6Z3eICTEe7Gw7KDVkYTNqGtvgwsuQi/+F0d/sjqhEA0hhaGFqdgO3p7DDx7MZ6agK+Vioe214Kp/ynB4L0YnnM2Bvz+CLvqyxd5b2IcUBguoxPO8F6QLP0UvlvFg27JoPiPVqjXGvX9AtY7AfOav6IpyS3KIwCWFwSJGv0u8s7KuWY3pzrU6jrAZ1c5FzG+ngGcv5nMzvUvVClFPUhgspK65Fc7thX7xGfQ3X1gdRzSXlhtJqiXsvF6o68bD5vVoecBSNIAUBgsphwPjjgcgMgrzH49Ll992rJ8aWw0Z4X3AcvlC9PaPrY4jAoQUBoupNu0w7noQPPsw58+SefaFTymlULfeCx0TMJ+dhv7BY3UkEQCkMPgBlXge6ppbYOMH6LXvWB1H+EoLP8dwKiq8lXdOpWNHMOc9JdcbRJ2kMPgJNfRqOP8C9KJn0cVFVscRNqM6nYm6/pew/WN0znKr4wg/J4XBTyjDwBh3P4SFYz77pDzfYCct+BzD6ahLL4M+KeglC9Bf7bI6jvBjUhj8iGrrxLj9PvjmC/TSF6yOI5rMv64XKaUwbrsXottgPvt39NEjVkcSfkoKg59RF1yEShuOfmc5+vOtVscRNqOi2nh7pt9/i14iXz7EyUlh8ENqzO3QPh7z+aflW10gq7nDzD+Gko5T51+AGnIlOncVeod8+RAnksLgh1R4K++Q0t496NfmWx1H2JAafav3y8f8p2XNaHECKQx+Sp3TE5U+Ev3e6+gdW6yOI5rCvzoMwPEvH7+Gfd+jl8iXD1GbFAY/pq65BeI6eu89l291gce/rj2fQJ3Twzuk9N4b6M8+sTqO8CNSGPxYzbc6z1700gVWxxE2pK651fvlY/4s9NGjVscRfkIKg59TSd1RaVd4h5S+2Gl1HNEgP3YZ/OQ5hpNR4eEYt070Dim9vsjqOMJPSGEIAGrULRDTDnNBFrpaFnoXvqXO7Ym6OAP99jJZ2EcAEFKfjTZt2sS8efMwTZP09HRGjRpV6/eVlZVkZWWxe/duoqOjyczMJC4uDoClS5eSm5uLYRiMHTuWPn36nPaYW7duZcGCBVRVVdGlSxfuvvtuHA6HL9sccFREJMYNd2DOmYp+dyVq2Ki6dxJ+xH97DMepMbejN6/HfPEZjN9ORRnynTGY1Xn2TdMkOzubyZMnM2PGDNatW0dRUe25fHJzc4mMjGTWrFmMGDGChQsXAlBUVITb7Wb69Ok8/PDDZGdnY5rmKY9pmiazZ8/m17/+NdOmTaN9+/asWbOmeVoeaC4cAL37eadPLi2xOo2ojwCaKVdFtUFdPx52fYZ+/y2r4wiL1VkYCgsLiY+Pp0OHDoSEhJCamkpBQUGtbTZs2EBaWhoAKSkpbN26Fa01BQUFpKamEhoaSlxcHPHx8RQWFp7ymOXl5YSEhNCpUycAevfuzYcffuj7VgcgpRTGTXcCYL40V6bnFj6nUtLgvN7oJS/I9NxBrs6hJI/Hg8vlqnntcrnYuXPnKbdxOBxERERQVlaGx+MhKSmpZjun04nH46k5zv8eMzo6murqanbt2kViYiL5+fns27fvpLlycnLIyckBYOrUqcTGxta3zbWEhIQ0et8WFxvLoZt+RfnzWUQXbqPVgLRGHSag2uwjVrT5UGQk5YAr1oURGd2i7w2Na3PVvZMpvf9WQpcvoO1v/txMyZqPfLZ9dEyfHq2JlFJkZmYyf/58KisrueCCCzBOMdaZkZFBRkZGzetTFZC6xMbGNnpfK+iUdMhZxYF/zaDszCRUeHiDjxFobfYFK9psHvKuyFdaWoo63PK3gjaqzeERqMuv5ejKl9nbfzDqvN7NE66ZyGe7YY6PzvyvOoeSnE4npaWlNa9LS0txOp2n3Ka6upqKigqio6NP2Nfj8eB0Ok97zHPOOYfHHnuMxx9/nPPPP5+OHTs2oJn2pxwOjBvv8D7bsPo1q+OIevH/i88/pS4fDa44zEXPyl1wQarOwpCYmEhxcTElJSVUVVXhdrtJTk6utU3fvn3Jy8sDID8/nx49eqCUIjk5GbfbTWVlJSUlJRQXF9OtW7fTHvPAgQOA906n5cuXM2zYMB83OfCpc3qi+g1Er34NvXeP1XHEqQToZSAVFo5x/Xj49it03ptWxxEWqHMoyeFwMG7cOKZMmYJpmgwePJiEhAQWL15MYmIiycnJDBkyhKysLCZOnEhUVBSZmZkAJCQkMGDAACZNmoRhGIwfP75maOhkxwRYsWIFGzduxDRNhg0bRs+ePZux+YFLjRnrvb3w38/hmDDZ6jjCbn6WAt37oFcsRF80EBUdY3Ui0YKUtsntLd99912j9gvkMUnzjX+jly7AuP9PqO4/q/d+gdzmxrLkGsPby9D/fg7j6UWo1hEt+t7Q9Dbr4m8w/3QfKjUd49Z7fZis+chnu2EafY1B+C81dJR36uSXn0VXVVkdR9iM6pjgnWRv7TvoL2U6lmAihSGAqdBQjJ//CvYUoXNXWR1H2JC68gaIjsF8+Z9o07Q6jmghUhgCnLqgH/Tsi161GF120Oo4opbjk+hZm6IpVESkd1Gf3TvQBf+xOo5oIVIYbMC4biwcPYxeJbNjCt9TA4ZAQhf00gXoymNWxxEtQAqDDahOZ6IGDkOveRO951ur44jjam7rCOAuA6AMA+O6cVBagn53pdVxRAuQwmAT6qqbIDQM87XnrY4ibEidf4F3Esc3/o0uO2B1HNHMpDDYhGrTFnX5tbDpQ/SOrVbHETZkjLkdjh5Br5QhS7uTwmAjaujV4IzF/PdzcgeJX/D/FdwaQnVMQF16Gfr91eg9RXXvIAKWFAYbUWHhqGtuga8K0etlHQvhe2rkjd4hy1eftzqKaEZSGGxGXTQIzurmvYPkmCzubqmaSQXs0WOAH4csrxgDm9ejd2yxOo5oJlIYbKbmDhLPPvS78tCb8D2VcdWPQ5bzZMEom5LCYEPq3J7QKxm9+lX0j2sCCOErKiwcddXN8FUhbPzA6jiiGUhhsCnjmlug4hD6rSVWRwle9htJqqEGpEHHBMxlC2TNBhuSwmBTKqEL6qJB6HdXyPq9wueU4fB++djzLdr9rtVxhI9JYbAxdfVNUF2Nfn2x1VGCm01uVz1Bn/7Q5Rz0ykVyo4PNSGGwMRXXETXwMvR/3kaXNG69CtEU9r4wq5TCuPY22L8PnfeG1XGED0lhsDk14npwhKCXv2R1FGFD6txe0ONn6DdeRVccsjqO8BEpDDan2jpRGVeh17+P/nq31XGCiw2fYzgZ45pb4VAZ+u2lVkcRPiKFIQioy66BiCjMZS9aHUXYkDorEdVvIPqd5eiD+62OI3xACkMQUBFRqOFjYMsG9OcywV6Ls3eHAQB19c1QVYleJTc62IEUhiChBo+Atk7MZS/K06otJYj+zqpDJ9QlQ9Hvv40uLbE6jmgiKQxBQoWFo4ZfDzu3c2xzgdVxhA2pEdeDAv36K1ZHEU0khSGIqEuGgjOWQy8/K72GFhUEY0mAcrb33h7tfhe9d4/VcUQTSGEIIio0FDXi51R+vg22fmR1HGFDavgYMBxyrSHASWEIMio1HUeHTpjLX5Jeg/A51daFGnQF+oP3ZP3xACaFIciokBAirx/rnRlz84dWx7E3ba8V3OpLXTEaQkPRq2QJ0EAlhSEItRp0GcR19PYaZAlQ4WOqTTvU4BHehyq/+9rqOKIRpDAEIeUIQY28AYq+hI9lPv3mc7zHYG0KK6jLRkNYK/RK6TUEIikMQUpddCnEn/Fjr0Hm0xe+paLboDJGojesRRd9YXUc0UBSGIKUMhyoq26E4m/QBWutjiNsSA0dBa0jMVe8bHUU0UAh9dlo06ZNzJs3D9M0SU9PZ9SoUbV+X1lZSVZWFrt37yY6OprMzEzi4uIAWLp0Kbm5uRiGwdixY+nTp89pj7llyxZefPFFTNOkVatW3HPPPcTHx/uyzeJHqu/F6M6veOfTT74E5XBYHcleam76CsKxJEBFRqGGXo1e8RL6q12osxKtjiTqqc4eg2maZGdnM3nyZGbMmMG6desoKiqqtU1ubi6RkZHMmjWLESNGsHDhQgCKiopwu91Mnz6dhx9+mOzsbEzTPO0x//WvfzFx4kSefPJJLrnkEl577bVmaLYAUIaBcdWN8P236A/XWB1H2JBKH+mdwHH5QqujiAaoszAUFhYSHx9Phw4dCAkJITU1lYKC2lMqbNiwgbS0NABSUlLYunUrWmsKCgpITU0lNDSUuLg44uPjKSwsrPOYhw8fBqCiooJ27dr5sLniBD8bAGd2Ra9ahK6qsjqNvQTp7ao/pSIivbP7btmA3r3D6jiinuocSvJ4PLhcrprXLpeLnTt3nnIbh8NBREQEZWVleDwekpKSarZzOp14PJ6a45zsmHfddRePP/44YWFhtG7dmilTppw0V05ODjk5OQBMnTqV2NjYejX4f4WEhDR630D1v20+cvOdHHj8d0Rt30jrIcMtTNZ8rDjP5ZERHAJiY2NRFhQHf/lsm9fdxr6cFYS8vZR2v/97s76Xv7S5JTVHm+t1jaElvf766zz00EMkJSWxYsUKXnjhBe66664TtsvIyCAjI6Pm9b59+xr1frGxsY3eN1D9b5t1l/MgoQsHF2dT3qOvLa81WHGezUMVgPezaUVh8KvPdvpIji17kb0bPkCdnVTn5o3lV21uIU1pc6dOnU768zqHkpxOJ6WlpTWvS0tLcTqdp9ymurqaiooKoqOjT9jX4/HgdDpPecyDBw/y1Vdf1fQyUlNT2bFDup/NTSmFceUNUFKMLnjf6jg24h1KsqIo+Bs15ErvtQaZQykg1FkYEhMTKS4upqSkhKqqKtxuN8nJybW26du3L3l5eQDk5+fTo0cPlFIkJyfjdruprKykpKSE4uJiunXrdspjRkZGUlFRwXffeReu/+STT+jcubPvWy1O1Kc/dD4L/for8lyD8DnVOgI19CrYvB799S6r44g61DmU5HA4GDduHFOmTME0TQYPHkxCQgKLFy8mMTGR5ORkhgwZQlZWFhMnTiQqKorMzEwAEhISGDBgAJMmTcIwDMaPH49heGvRyY4JcOeddzJt2jQMwyAyMpK77767GZsvjlOGgTHyBsw5f0MXrEX1H2R1pMAncxTWooaMRL+9HHPVYhwTJlsdR5yG0jaZYvN4L6OhZEzyv7RpYv7pPtAa49GnUYZ9rjVYco1h+UvoVYtwPLuiRd/3OH/8bJsrXkKvXITxx6dQZ3Tx+fH9sc3NzZJrDCJ4KMNAXflz79PQH7mtjiNsSKVfBa0jMFfKtQZ/JoVB1KL6pkLHBPSqxTLzapPpoH6G4WRUZJT3obeNbvS3X1kdR5yCFAZRizIc3rV7v/taZl4VzUJlXAWtWssqb35MCoM4gep3CcR3xpReQ9NoTbDOk3Q6KjIaNeRK9EfrZL0GPyWFQZygptdQ9CVsklXehO+pjKshLBz9+itWRxEnIYVBnJTqd6l3lbdVi2RtaOFzKrqNd5W3gv+gi4vq3kG0KCkM4qSU48dewzdfwOb1VscJTDKSdFpq2CgIDUO/Ib0GfyOFQZyS6p8G7eMxV0qvQfieio5BpQ1Hf/g+es+3VscRPyGFQZyScjhQw6+Dr3fBlg1WxxE2pC4bBaEh6Df+bXUU8RNSGMRpqZTBENtBeg2NIs8x1EW1aYe69Ar0h3nokmKr44gfSWEQp6VCQry9hi93wraNVscRNqQuuwYc0mvwJ1IYRJ3UgMHgbC+9hoaS5xjqRbV1oi69DJ3/HnrvHqvjCKQwiHpQIaHeXsPuHbB9k9VxhA2py0eDMtBvvmp1FIEUBlFPKjUd2sVirnxZeg3C51RbF2rgULT7XfS+762OE/SkMIh6UaGhqCvGwK7P4LNPrI4TILSMJDWAunwMKIV+8zWrowQ9KQyi3tQlQ6GtS3oNolkoZyzqkqHodTno0r1WxwlqUhhEvXl7DdfCzu2wY4vVcfyfBukyNIy6fAwAerVca7CSFAbRIGrgMIhxYq5cZHUUYUPK1R51cQZ67TtoT3CtxOZPpDCIBlGhYd47SD7fit6x1eo4wobUFdeC1tJrsJAUBtFg6tLLIKYd5irpNZyWliefG0PFdkClpqP/8zZ6f6nVcYKSFAbRYCosHHXZaPjsE/TO7VbHETakrhjj7TW8tcTqKEFJCoNoFHXp5RAdg7nyZauj+DG5XbWxVPt4VMpg9PtvoX/wWB0n6EhhEI2iwn/sNXy6GV34qdVxhA2p4ddBdZX0GiwghUE0mkq7wttrkGsNohmouI6o/mnoNavRB/ZbHSeoSGEQjabCW3lX4dr2MXr3Dqvj+B95jqHJ1IjroUp6DS1NCoNoEpU2HKKi5bkG0SxUh06o/oPQa95EH/zB6jhBQwqDaBLVqjVq6CjY+hH6i8+tjuNn5HZVX1AjroPKKvTbS62OEjSkMIgmU0NGQGQ05qrFVkcRNqTiz0D1G4h+7w102QGr4wQFKQyiyVSrCNTQq+GTAvRXhVbHETakrrweKo+h315mdZSgIIVB+IQaPAIiIuVaw0/JCm4+ozomoJIvQb/3OrrsoNVxbE8Kg/AJFRGJyrgaNq9Hf73L6jjChtSVP4djR9E5y62OYnsh9dlo06ZNzJs3D9M0SU9PZ9SoUbV+X1lZSVZWFrt37yY6OprMzEzi4uIAWLp0Kbm5uRiGwdixY+nTp89pj/nII49w+PBhAA4ePEhiYiK//e1vfdZg0XxU+pXod5ZjrlqMY8Jkq+NYT5as8CnV6UxU34vRuavQw0ahIqOtjmRbdfYYTNMkOzubyZMnM2PGDNatW0dRUVGtbXJzc4mMjGTWrFmMGDGChQsXAlBUVITb7Wb69Ok8/PDDZGdnY5rmaY/52GOP8eSTT/Lkk0+SlJRE//79m6HZojmoiChUxkj4OB9d9IXVcfyDjCT5lBpxPRw5jH5Heg3Nqc7CUFhYSHx8PB06dCAkJITU1FQKCgpqbbNhwwbS0tIASElJYevWrWitKSgoIDU1ldDQUOLi4oiPj6ewsLBex6yoqGDbtm3069fPd60VzU6lXwWtIzBXyh1KwvfUGWfDhaneXsOhcqvj2FadQ0kejweXy1Xz2uVysXPnzlNu43A4iIiIoKysDI/HQ1JSUs12TqcTj8dTc5zTHbOgoICePXsSERFx0lw5OTnk5OQAMHXqVGJjY+tqykmFhIQ0et9A1axtjo2l/MrrOfTv54k5dIDQsxKb530ayIrzXNa6FYcNw7LPl10/25W33IXn/ltp7c4h6sZf1vqdXdt8Os3R5npdY7DCunXrGDJkyCl/n5GRQUZGRs3rffsat9pTbGxso/cNVM3dZn3xUFi5mP0L5mDc9btme5+GsOI8m4cPo7W27PNl2892VFv4WQqHVi7m8MUZqIioml/Zts2n0ZQ2d+rU6aQ/r3Moyel0Ulr638UySktLcTqdp9ymurqaiooKoqOjT9jX4/HgdDrrPObBgwcpLCzkwgsvrGfzhD9RkdGoIVeiN7rR335tdRzraLn63FyMK38Ohw+h311ldRRbqrMwJCYmUlxcTElJCVVVVbjdbpKTk2tt07dvX/Ly8gDIz8+nR48eKKVITk7G7XZTWVlJSUkJxcXFdOvWrc5j5ufnc+GFFxIWFubb1ooWozKuhrBw9OvBfq1Brj43B3VmIlxwETpnObrikNVxbKfOoSSHw8G4ceOYMmUKpmkyePBgEhISWLx4MYmJiSQnJzNkyBCysrKYOHEiUVFRZGZmApCQkMCAAQOYNGkShmEwfvx4DMNbi052zOPcbvcJt8SKwKKi26CGjECvXoIefp33oqEQPmSMvBHzL/ejc5ajrrrJ6ji2orS2R3/3u+++a9R+MibZfPShMsyHfgXn9bb8uQZLrjEsehb9QS6Op6xZ5S4YPtvV/3gctm/CePxZVFSboGjz/7LkGoMQjaUio71DSh/nyxxKolkYV90ER4+g35KZV31JCoNoVirjKu/Mq8tfsjpKy7NHZ9yvqc5nofpd6n2u4aCs8uYrUhhEs1IRkd61obdsCNK1oeXic3NTI2+Aqkr0m69ZHcU2pDCIZqeGjPCuDb18odVRhA2p+M6oAUPQeW9Sva/E6ji2IIVBNDsV3go1/Dr47BP0p5utjtNytKzg1lLUlT8HrTn06nyro9iCFAbRItSgy6GtC3P5QmxyI5zwIyq2A2rgMA7nrEDv3WN1nIAnhUG0CKO8+P8AABcTSURBVBUa5v1Wt+sz2LrR6jjChtSI68DhQMsSs00mhUG0GHVxOsR2wFz2YpD0GrRce25Bqq2LiMtHoz94D72nqO4dxClJYRAtRoWEokbeCF/vgo/zrY4jbCjyml9AWBh6hTUPFdqFFAbRolTKIIg/w3utway2Ok7z0iBdhpZltHWi0keiC/4ji0U1gRQG0aKU4fDOa/Pd1+j171sdR9iQGnYNtI7EXCa3RzeWFAbR4lTfVDgzEb1sIbqy0uo4wmZUZBTq8tGweT36821WxwlIUhhEi1OGgXHtbVBagl7zhtVxmpE8x2AVlX4VtHViLpkfJDc6+JYUBmEJ1b0PdP8Z+vVXZD594XMqPNw7ZLnrM9j0odVxAo4UBmEZ49pbobwM/dYSq6M0D/mmaimVmu690WHJC+hqm9/o4GNSGIRl1JmJqIsGeVfh+qG07h2EaADlcGCMvhX2FKHX5VgdJ6BIYRCWUqNuhmoTvXKR1VGEHfXpD4nnoVe8jD561Oo0AUMKg7CUah+PSrsCvfYddLHNnlbVyMVniymlMK69HQ540DnLrY4TMKQwCMupEddDWDjm0hesjiJsSCV1hwsuQr+1BF120Oo4AUEKg7Ccio7xLubzcb7NFvORi8/+whh9Kxw5gn7jFaujBAQpDMIvqKFXQ4wT85Vse913LkNJfkF1OhN1SQb6vTfQ339ndRy/J4VB+AUV3go1+hb44nOZKkM0C3X1zRASivnqPKuj+D0pDMJvqJTBcFY39Gvz7XEHidbIJHr+Q8W0867ZsOnD4FpJsBGkMAi/oQwD4/rxsH8f+u2lVscRNqQyrgJXnHfI0u6z+zaBFAbhV9Q5PVB9L0avfg29Xx56E76lQsMwxtwORV+i18pDb6cihUH4HXXtbWCaaDvcviojSf6n78XQrTt62YsyT9cpSGEQfke1j0cNvcq7ROMXO62OI2xGKYXx8/FQdgD9xr+tjuOXpDAIv6SuuA7atMVc/Gzg3r4qF5/9ljo7CTVgCPrdFeiSYqvj+B0pDMIvqdYRqGtugV2fofPzrI4jbEiNvgUcoZiL/2V1FL8jhUH4LZWaDl3PRf/7OXRFudVxGi5QezpBQrV1oUbeAJ8UoGXNhlpC6rPRpk2bmDdvHqZpkp6ezqhRo2r9vrKykqysLHbv3k10dDSZmZnExcUBsHTpUnJzczEMg7Fjx9KnT5/THlNrzaJFi8jPz8cwDIYOHcrw4cN92WYRIJRhYNx0F+aU36CXLUTddKfVkRpOnnz2ayp9JHpdDuaiZzHO74MKD7c6kl+os8dgmibZ2dlMnjyZGTNmsG7dOoqKas+CmZubS2RkJLNmzWLEiBEsXOhdhLuoqAi328306dN5+OGHyc7OxjTN0x4zLy+P0tJSZsyYwYwZM7j44oubodkiUKizElFpl6Pz3kR/vcvqOMJmVEgIxs13eZeZfVMuRB9XZ2EoLCwkPj6eDh06EBISQmpqKgUFBbW22bBhA2lpaQCkpKSwdetWtNYUFBSQmppKaGgocXFxxMfHU1hYeNpjvv3224wZMwbD8EaLiYnxcZNFoFGjfgFR0ZgL56BN0+o4DSBDSYFAndsL1X+Qd/ZVmUcJqMdQksfjweVy1bx2uVzs3LnzlNs4HA4iIiIoKyvD4/GQlJRUs53T6cTj8dQc52TH/P7773G73axfv542bdowduxYOnbseEKunJwccnK8D6hMnTqV2NjYejf6p0JCQhq9b6AKvDbHcvj2iRyc9ReiPvmQ1hkjG3wEK9p8ILwVxxwOy/7WgXeem66xba6+8wFK791AyKvP0faRGagAGgJsjvNcr2sMLamyspLQ0FCmTp3Khx9+yD/+8Q8ee+yxE7bLyMggIyOj5vW+ffsa9X6xsbGN3jdQBWKbda9+0K07B+dnUd6tByqqTYP2t6LN5pEjaLPasr91IJ7npmpSm0feyLHF/2Lf2ytQfQNnCLspbe7UqdNJf17nUJLT6aS09L9TE5SWluJ0Ok+5TXV1NRUVFURHR5+wr8fjwel0nvaYLpeL/v37A3DRRRfx1Vdf1beNwsaUUhg33wkVh9CvPGd1nAYInG+ewU4NHgEJXTBffjYw74LzoToLQ2JiIsXFxZSUlFBVVYXb7SY5ObnWNn379iUvLw+A/Px8evTogVKK5ORk3G43lZWVlJSUUFxcTLdu3U57zH79+rF161YAtm/ffsqKJoKPOqML6vJr0R/kord9bHUcYTPK4cC4bSIc/AH96vNWx7FUnUNJDoeDcePGMWXKFEzTZPDgwSQkJLB48WISExNJTk5myJAhZGVlMXHiRKKiosjMzAQgISGBAQMGMGnSJAzDYPz48TUXlU92TIBRo0bx9NNP8/rrr9OqVSvuvDMAb1EUzUZd+XP0RjfmgtkYj85CtWptdaRTk+cYAo46qxtq2NXot5aiL7oUdV5vqyNZQumAnW+gtu++a9zdBDIOG3j0zu2YTzyISh+JccOv6rWPJdcYnpuJ/nwrjqnWPFkb6Oe5MXzRZn30KOafJgJg/HGW3z/bYMk1BiH8jUrqjkobjs5dhd71mdVxTsMW37mCjgoPx7j1Xti7B73yJavjWEIKgwhIavSt0NaFOX8WurLS6jjCZtR5vVEDh6HfXo7+Mvhm+JXCIAKSah2BccsEKP4maL/ViealxtwOMW29Q4KVx6yO06KkMIiApXole7/VrV6C3rnd6jgnkpGkgKYiojBuu8/75WPJAqvjtCgpDCKgqevHedfwfW4G+kiF1XFOFEBP0IoTqZ4XogYPR+csR3+62eo4LUYKgwhoqlUExvj7oXQvenG21XGEDalrx0KHzpjPPxU0D75JYRABT3Xrjrr8GvTad9Cb8q2O8xNaegw2oMLDvV8+fvCgX/6n1XFahBQGYQvqqpu80xnMz0LvL617ByEaQHU5BzXi5+j8PMwgWFFQCoOwBRUSivGr/4PKY5jPPomurrY6kjz5bDNqxPXQrTv6xWfQe4rq3iGASWEQtqE6noH6xd2wczt6hdzCKnxLORwYv3oAQsMw5/wNfeyo1ZGajRQGYStGymDUJUPRb76K3rrR6jjCZpQz1nu94duv0IuetTpOs5HCIGxH3XAHdDoTM3s62rPXuiAaufhsQ6pnX9QVY9D/eRvzg/esjtMspDAI21Hh4Rh3/g6qKjGfedzWXX5hDXX1zXBOT/SC2bacMkMKg7Al1fEMjF/+Br7ehZ4/C2smEZaLz3alHA6Mu34Hbdpizv4r+geP1ZF8SgqDsC11wUWoUb9Ar3+fCsumNJChJLtS0TEY9zwMFeWYc6baajJHKQzC1tQVY1D9BlK+cC5683qr4wibUQldMMbdD7s+897GapNblKUwCFtTSqFuu4+Qrudi/vMJ9O4dLffmNvmPhDg91TcVNfJGtPtd9IqXrY7jE1IYhO2p8HDa/v7vEOPEnPVYyz6cJHclBQU18gbUxRnoVYsw8960Ok6TSWEQQcHR1omR+SgoA3Pmo+gfZNoM4TtKKdQt90DvfuiX5qI3fmB1pCaRwiCChorrhHHfI1B+EHP6I+iDP7TAmzb/Wwj/oBwOjDv+D87uhvns39FbP7I6UqNJYRBBRZ2dhDHxD1D6Pea037dMcRBBQ4W3wvj1H6FTgvc21gB9+l4Kgwg66txeGBMfgX17MKf/AV12oHneSC4+ByUVGY0x6c/Q8QzM2VMCsjhIYRBBSZ3XG+PeP8DeYswnHkKXljTXOzXTcYU/O6E4fOS2OlKDSGEQQUudfwHGr/8EB/djTv0tuugL376B9BiCmopqg/Gbv8BZiZhz/4b53utWR6o3KQwiqKlzemD8diqgvD2HbR/7+A2kxxDManoOP96tZL76PNr0g7VC6iCFQQQ91fksjIeeAGd7zKf+hPnmq7Z5glVYT4WFY9z9ECrtCvRbSzCffgx9qMzqWKclhUEIQDnbYzz0JCr5YvSSF7wLsRxq2sLvWibREz9SDgfGzXejbr0XdmzB/Mskv56VVQqDED9S4a1Qv3oAdd1Y2JSP+ehE3w8tiaBmDByG8X+PQ3U15uP/h7niJXRVldWxTiCFQYifUEphDLsG46EnoXUE5sw/Yi6YjS4/2PCDSYdBnITqei7GH59GXXQpeuUi740PftZ7kMIgxEmos5Mwfj8dNWwUeu07mA/fhZm7quHf7uTiszgJFRmFMX4Sxt0Pwv59mH99APP5p/xmXYcQqwMI4a9UWDjqunHo1HTMRc+iX/4n+q2lqMtHoy7OQIWFWx1RBDh1YSrG+X3Qqxaj312JXv8f1CVDUZeNRrnaW5arXoVh06ZNzJs3D9M0SU9PZ9SoUbV+X1lZSVZWFrt37yY6OprMzEzi4uIAWLp0Kbm5uRiGwdixY+nTp89pjzl79my2b99OREQEAPfccw9nn322r9orRIOpzmd5bznc+hHm6694J0lb/hIqJQ2VOgQSuqJO2jOQsSRRN9U6AnXdWPSgy9BvvoZ+fzV6zZvQKxnj4gzo2RcVGtqimeosDKZpkp2dze9//3tcLhcPPfQQycnJnHHGGTXb5ObmEhkZyaxZs1i3bh0LFy7k/vvvp6ioCLfbzfTp09m/fz9//vOfeeqppwBOe8xbbrmFlJSUZmqyEA2nlPL+Q+3ZFz7fil7j/cer310J7WJRPS+EbuejzjgbOiagQsOO72hpbhE4VFwn1G0T0VfegM57A/3Be5ib10N4Kzi3F+r83qiErtD5LIiMPsWXEd+oszAUFhYSHx9Phw4dAEhNTaWgoKBWYdiwYQPXXXcdACkpKTz33HNorSkoKCA1NZXQ0FDi4uKIj4+nsLAQoM5jCuGPlFLef6Tn9kIfKkNv/AC9dSN6w1r4z9v/7SO0joDDFdDpTCvjigCkXO1R196GHvUL2L4JvaXA+xn7pOC/n6+QUIhqA60jqPrDNAht5dMMdRYGj8eDy+Wqee1yudi5c+cpt3E4HERERFBWVobH4yEpKalmO6fTicfjqTnOqY758ssv8+qrr9KzZ09uvvlmQk/SjcrJySEnJweAqVOnEhsbW68G/6+QkJBG7xuopM0+EhsLZ3WBa25CV1dTvaeIqi93UfXtV5gHf0CXHSCsV19aW/S3lvNsAx0ug8GXAVDt2UfV17uo+moX5g/7vZ+xwxWERkQSG9POp2/rdxefb7rpJtq2bUtVVRVz585l+fLljBkz5oTtMjIyyMjIqHm9b9++Rr1fbGxso/cNVNLmZhIeCef29v7vR5XAIYv+1nKebeiMRO//fkLHtGt0mzt16nTSn9d5u6rT6aS09L+rXZWWluJ0Ok+5TXV1NRUVFURHR5+wr8fjwel0nvaY7dq1QylFaGgogwcPrhl6EkII0TLqLAyJiYkUFxdTUlJCVVUVbreb5OTkWtv07duXvLw8APLz8+nRowdKKZKTk3G73VRWVlJSUkJxcTHdunU77TH3798PUHONIiEhwcdNFkIIcTp1DiU5HA7GjRvHlClTME2TwYMHk5CQwOLFi0lMTCQ5OZkhQ4aQlZXFxIkTiYqKIjMzE4CEhAQGDBjApEmTMAyD8ePHYxjeWnSyYwI8/fTTHDzofcr0rLPO4o477miutgshhDgJpW0yjeR3333XqP1sPyZ5EtLm4CBtDg5NaXOjrzEIIYQILlIYhBBC1CKFQQghRC1SGIQQQtRim4vPQgghfCPoewwPPvig1RFanLQ5OEibg0NztDnoC4MQQojapDAIIYSoxfHoo48+anUIq3Xt2tXqCC1O2hwcpM3BwddtlovPQgghapGhJCGEELVIYRBCCFGL3y3U05I2bdrEvHnzME2T9PR0Ro0aZXWkJtu3bx+zZ8/mhx9+QClFRkYGw4cPp7y8nBkzZrB3717at2/P/fffT1RUFFpr5s2bx8cff0x4eDgTJkwI2DFa0zR58MEHcTqdPPjgg5SUlDBz5kzKysro2rUrEydOJCQkhMrKSrKysti9ezfR0dFkZmYSFxdndfwGO3ToEHPmzOGbb75BKcXdd99Np06dbH2eV61aRW5uLkopEhISmDBhAj/88IOtzvMzzzzDxo0biYmJYdq0aQCN+vebl5fHkiVLABg9ejRpaWn1D6GDVHV1tb733nv1nj17dGVlpX7ggQf0N998Y3WsJvN4PHrXrl1aa60rKir0fffdp7/55hu9YMECvXTpUq211kuXLtULFizQWmv90Ucf6SlTpmjTNPWOHTv0Qw89ZFn2plq5cqWeOXOmfvzxx7XWWk+bNk2vXbtWa6313Llz9VtvvaW11nr16tV67ty5Wmut165dq6dPn25N4CaaNWuWzsnJ0VprXVlZqcvLy219nktLS/WECRP00aNHtdbe8/vee+/Z7jxv27ZN79q1S0+aNKnmZw09r2VlZfqee+7RZWVltf5/fQXtUFJhYSHx8fF06NCBkJAQUlNTKSgosDpWk7Vr167mG0Pr1q3p3LkzHo+HgoICBg0aBMCgQYNq2rphwwYuvfRSlFKcc845HDp0qGaxpEBSWlrKxo0bSU9PB7wLPW3bto2UlBQA0tLSarX5+LenlJQUtm7dig6wezAqKir49NNPGTJkCOBd6zgyMtL259k0TY4dO0Z1dTXHjh2jbdu2tjvP3bt3JyoqqtbPGnpeN23aRO/evYmKiiIqKorevXuzadOmemcI2qEkj8eDy+Wqee1yudi5c6eFiXyvpKSEL774gm7dunHgwAHatfMuGN62bVsOHDgAeP8OP1083eVy4fF4arYNFM8//zy/+MUvOHz4MABlZWVERETgcDgA7/KzHo8HqH3uHQ4HERERlJWV0aZNG2vCN0JJSQlt2rThmWee4auvvqJr167cfvvttj7PTqeTkSNHcvfddxMWFsYFF1xA165dbX2ej2voef3f/7799O9SH0HbY7C7I0eOMG3aNG6//XYiIiJq/U4phVLKomS+99FHHxETExOQY+aNVV1dzRdffMGwYcN44oknCA8PZ9myZbW2sdt5Li8vp6CggNmzZzN37lyOHDnSoG/BdtES5zVoewxOp5PS0tKa16WlpTidTgsT+U5VVRXTpk1j4MCB9O/fH4CYmBj2799Pu3bt2L9/f823JqfTWWv1p0D8O+zYsYMNGzbw8ccfc+zYMQ4fPszzzz9PRUUF1dXVOBwOPB5PTbuOn3uXy0V1dTUVFRVER0db3IqGcblcuFwukpKSAO9QybJly2x9nrds2UJcXFxNm/r378+OHTtsfZ6Pa+h5dTqdbN++vebnHo+H7t271/v9grbHkJiYSHFxMSUlJVRVVeF2u0lOTrY6VpNprZkzZw6dO3fmyiuvrPl5cnIya9asAWDNmjX069ev5ufvv/8+Wms+//xzIiIiAmp4AeCmm25izpw5zJ49m8zMTHr27Ml9991Hjx49yM/PB7x3aBw/v3379iUvLw+A/Px8evToEXDfrNu2bYvL5apZ0nbLli2cccYZtj7PsbGx7Ny5k6NHj6K1rmmznc/zcQ09r3369GHz5s2Ul5dTXl7O5s2b6dOnT73fL6iffN64cSPz58/HNE0GDx7M6NGjrY7UZJ999hmPPPIIZ555Zs0/ghtvvJGkpCRmzJjBvn37TrjdLTs7m82bNxMWFsaECRNITEy0uBWNt23bNlauXMmDDz7I999/z8yZMykvL6dLly5MnDiR0NBQjh07RlZWFl988QVRUVFkZmbSoUMHq6M32JdffsmcOXOoqqoiLi6OCRMmoLW29Xl+5ZVXcLvdOBwOzj77bO666y48Ho+tzvPMmTPZvn07ZWVlxMTEcP3119OvX78Gn9fc3FyWLl0KeG9XHTx4cL0zBHVhEEIIcaKgHUoSQghxclIYhBBC1CKFQQghRC1SGIQQQtQihUEIIUQtUhiEEELUIoVBCCFELf8PrUnKrhV08RQAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "agK_98cWwwdQ" + }, + "source": [ + "### Training Loop\n", + "\n", + "Now we're ready to start the training process. First of all, let's split the original dataset using [train_test_split](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) function from the `scikit-learn` library. (Though you can use anything else instead, like, [get_cv_idxs](https://github.com/fastai/fastai/blob/921777feb46f215ed2b5f5dcfcf3e6edd299ea92/fastai/dataset.py#L6-L22) from `fastai`)." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "216J9e39wwdR" + }, + "source": [ + "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n", + "datasets = {'train': (X_train, y_train), 'val': (X_valid, y_valid)}\n", + "dataset_sizes = {'train': len(X_train), 'val': len(X_valid)}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fFgH5tuLwwdR", + "outputId": "4e64cc2c-8377-46a1-d220-96fbee9a59bb" + }, + "source": [ + "minmax = ratings_df.RATING.astype(float).min(), ratings_df.RATING.astype(float).max()\n", + "minmax" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(1.0, 5.0)" + ] + }, + "metadata": {}, + "execution_count": 35 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WlAmBo9Jys0m", + "outputId": "59234538-cd74-426f-8543-72cac11d7ee0" + }, + "source": [ + "n_users = ratings_df.USERID.nunique()\n", + "n_movies = ratings_df.ITEMID.nunique()\n", + "n_users, n_movies" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(943, 1682)" + ] + }, + "metadata": {}, + "execution_count": 36 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KHeaux79wwdS" + }, + "source": [ + "net = EmbeddingNet(\n", + " n_users=n_users, n_items=n_movies, \n", + " n_factors=150, hidden=[500, 500, 500], \n", + " embedding_dropout=0.05, dropouts=[0.5, 0.5, 0.25])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "onmMNylKwwdS" + }, + "source": [ + "The next cell is preparing and running the training loop with cyclical learning rate, validation and early stopping. We use `Adam` optimizer with cosine-annealing learnign rate. The rate is decreased on each batch during `2` epochs, and then is reset to the original value.\n", + "\n", + "Note that our loop has two phases. One of them is called `train`. During this phase, we update our network's weights and change the learning rate. The another one is called `val` and is used to check the model's performence. When the loss value decreases, we save model parameters to restore them later. If there is no improvements after `10` sequential training epochs, we exit from the loop." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4mRf0N9kwwdT", + "scrolled": false, + "outputId": "50f21317-af76-4dd8-a8f0-5ab7b61cc726" + }, + "source": [ + "lr = 1e-3\n", + "wd = 1e-5\n", + "bs = 50\n", + "n_epochs = 100\n", + "patience = 10\n", + "no_improvements = 0\n", + "best_loss = np.inf\n", + "best_weights = None\n", + "history = []\n", + "lr_history = []\n", + "\n", + "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", + "\n", + "net.to(device)\n", + "criterion = nn.MSELoss(reduction='sum')\n", + "optimizer = optim.Adam(net.parameters(), lr=lr, weight_decay=wd)\n", + "iterations_per_epoch = int(math.ceil(dataset_sizes['train'] // bs))\n", + "scheduler = CyclicLR(optimizer, cosine(t_max=iterations_per_epoch * 2, eta_min=lr/10))\n", + "\n", + "for epoch in range(n_epochs):\n", + " stats = {'epoch': epoch + 1, 'total': n_epochs}\n", + " \n", + " for phase in ('train', 'val'):\n", + " training = phase == 'train'\n", + " running_loss = 0.0\n", + " n_batches = 0\n", + " \n", + " for batch in batch_generator(*datasets[phase], shuffle=training, bs=bs):\n", + " x_batch, y_batch = [b.to(device) for b in batch]\n", + " optimizer.zero_grad()\n", + " \n", + " # compute gradients only during 'train' phase\n", + " with torch.set_grad_enabled(training):\n", + " outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n", + " loss = criterion(outputs, y_batch)\n", + " \n", + " # don't update weights and rates when in 'val' phase\n", + " if training:\n", + " scheduler.step()\n", + " loss.backward()\n", + " optimizer.step()\n", + " lr_history.extend(scheduler.get_lr())\n", + " \n", + " running_loss += loss.item()\n", + " \n", + " epoch_loss = running_loss / dataset_sizes[phase]\n", + " stats[phase] = epoch_loss\n", + " \n", + " # early stopping: save weights of the best model so far\n", + " if phase == 'val':\n", + " if epoch_loss < best_loss:\n", + " print('loss improvement on epoch: %d' % (epoch + 1))\n", + " best_loss = epoch_loss\n", + " best_weights = copy.deepcopy(net.state_dict())\n", + " no_improvements = 0\n", + " else:\n", + " no_improvements += 1\n", + " \n", + " history.append(stats)\n", + " print('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n", + " if no_improvements >= patience:\n", + " print('early stopping after epoch {epoch:03d}'.format(**stats))\n", + " break" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "loss improvement on epoch: 1\n", + "[001/100] train: 0.9756 - val: 0.8986\n", + "loss improvement on epoch: 2\n", + "[002/100] train: 0.8512 - val: 0.8780\n", + "[003/100] train: 0.8775 - val: 0.8851\n", + "loss improvement on epoch: 4\n", + "[004/100] train: 0.8071 - val: 0.8705\n", + "loss improvement on epoch: 5\n", + "[005/100] train: 0.8338 - val: 0.8697\n", + "loss improvement on epoch: 6\n", + "[006/100] train: 0.7598 - val: 0.8624\n", + "[007/100] train: 0.7931 - val: 0.8698\n", + "[008/100] train: 0.7192 - val: 0.8733\n", + "[009/100] train: 0.7555 - val: 0.8743\n", + "[010/100] train: 0.6720 - val: 0.8844\n", + "[011/100] train: 0.7104 - val: 0.8882\n", + "[012/100] train: 0.6229 - val: 0.9149\n", + "[013/100] train: 0.6686 - val: 0.8936\n", + "[014/100] train: 0.5796 - val: 0.9359\n", + "[015/100] train: 0.6257 - val: 0.9201\n", + "[016/100] train: 0.5433 - val: 0.9525\n", + "early stopping after epoch 016\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EGrWJQGnwwdT" + }, + "source": [ + "### Metrics\n", + "\n", + "To visualize the training process and to check the correctness of the learning rate scheduling, let's create a couple of plots using collected stats:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "y3vAtcy9wwdU", + "outputId": "612b10fe-440e-4422-90ca-cef19cc5ce36" + }, + "source": [ + "ax = pd.DataFrame(history).drop(columns='total').plot(x='epoch')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEJCAYAAACE39xMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUVfrA8e+5M+mVSUhCIHQQEAExdOkoCoigAiuC8mOLiKu7rmXtnbWytrUDKqwoFkBREEXpUQywKFVAQwkESIFUUib3/P4YCEQS0qZl8n6exyeZmTv3fWcw75w559xzlNZaI4QQot4zPJ2AEEII55CCLoQQPkIKuhBC+Agp6EII4SOkoAshhI+Qgi6EED7C6snghw8f9mT4cqKjo8nIyPB0GpXy9vzA+3P09vzA+3P09vzA93OMj4+v9DFpoQshhI+Qgi6EED5CCroQQvgIKehCCOEjpKALIYSPkIIuhBA+Qgq6EEL4iHpX0PVvv2AufM/TaQghhNepfwV9/6/oZZ+iU1M8nYoQQniVelfQVeKlYLGgf1jt6VSEEMKr1L+CHhYOF3ZH/7gGbZqeTkcIIbxGvSvoAKrXQDieAXu2ezoVIYTwGvWzoHftBQFB6A3S7SKEEKfVz4IeEIDq3hu9cT26pNjT6QghhFeolwUdQPUaBCfzYetGT6cihBBeod4WdDp0gfBITOl2EUIIoB4XdGWxoHoOgJ+T0QV5nk5HCCE8rt4WdDg128VuR29K8nQqQghRLXlFpS47d70u6LRoC7FNZbaLEKJeSE7NY9qS31i286hLzl+vC7pSytFK370NnZXu6XSEEKJCJaUmszYe5cnVqUQHW+kUG+aSOPW6oMOpbhet0T+u8XQqQghxjkM5xdyzfD9LfjnOqAsa8ezwFrSwBbskltUlZ3UjFdMEWl/g6Ha54lpPpyOEEGW++y2bN5OP4GcxeGBgU3o2c03L/LR630KHU6301H3oQ/s9nYoQQlBQUsoL6w/z0vdptLUF8tKIli4v5uArBb1HfzAM9IZVnk5FCNHA7ck8yR1L97Fmfw4Tu0Tz+NDmRAX7uSW2bxT0sAjHCowbVssKjEIIjzC1ZvHOTO79ej92UzNjWHMmXBSNxVBuy8EnCjqc6nbJyoC9OzydihDCQ9Jyi9mceoJSU7s17olCO0+sTOWdzekkNg3lxRGt6BTjmoHP86n3g6KnqW690AGB6A2rUe07ezodIYSb7c44yWMrD5JXbNIoyMqAFmEMbBVB60YBKOW6VvKWtHxeTDpMXrHJtB6xXNEu0qXxzsd3CnpAIOri3uiN69B/+AvKzz19VkIIz9t+tIDHV6USGWjh9oFt+HZnGl/uPs5nu47TLNyfgS3DGdgqnNhQf6fFtJua+T+ls3BHFk3D/Xl0SAItGwU67fy14TMFHRzdLvqHVbBtE1zc29PpCCHcYPPhPJ5ac4iYED8eH5rABc1j6RVjIbeolPUHclidksP7P2fw/s8ZdIgOYmCrcC5tHkZ4YO3L39G8YmauP8wvGYVc3jaCP10SS4DV8z3YPlXQ6dgNwiIwN6zCIgVdCJ/3/cFcnl93iISIAB4bkkDEWUU6LMDCFe0acUW7RhzLK2HNvhxW78vmzeSjzNp4lO7xIQxoGUGvZqE1Ksbr9ufw6oYjKODuS+O5tEW4C15Z7fhUQT+9AqNe/RW6IB8VHOLplIQQLrIqJZuXvk+jXVQgDw9OINTfUumxMaF+XNc5imsvtLHvRBGrUnJYuy+H5EOHCbQa9EkIZWCrCLrEBlc6K6XIbvL2xqN882s2F0QHcme/eKd24TiDTxV0ONXt8u0S9OYk1KWXeTodIYQLLN9zgtd/PELn2GAeGNiMIL/qtbCVUrRqFEirRoHc2K0x248VsHpfDkkHclmZkkNkoIX+LcMZ2DKctrbAssHNfccLeW7dYQ7lFHPdhVFc3yUaqxunI1aXzxV0WraDmCaOpQCkoAvhcz7bmcWczce4JD6Ef/ZvWuu+a4uh6BIXQpe4EG7uEcvGQ3ms3pfDst0nWLLrOE3D/RnQMpwgq8G8LemE+hs8OiSBbk2895u/zxV0xwqMg9BffIg+nolqFOXplIQQTqC15qNtmcz/OYO+zcP4R994/CzOaSX7Wwz6Ng+nb/Nw8opKSTqYy+qUbD74OQOAS+JDuL1PEyLrMJDqDt6dXS2p3gPRSz5A/7gGNXysp9MRQtSR1pr3/pfOop1ZDGkdzl97NXHZFZihARYubxvJ5W0jSc8vIS23mM6xwRgemlteE56fZ+MCKiYeWrWXtV2EcKKNh/J4eNkuVqVku/VKTFNr3kw+yqKdWVzZLpLberuumP9e4xA/usSF1ItiDj7aQgcc3S4fvoU+dADVtLmn0xGi3jqaV8zsTcfYkJpHoNXg290m83/O4JpONoa2jsDP4rp2YampeeWHNFam5HBNJxs3dmvssasw6wOfbKEDqB6XygqMQtRBcanJgq0Z/PWLFH46ks9N3Rqz7Obe3D+gKeEBFl7/8Sh/+ew3PtuZRaHd+YvilZRqnl9/mJUpOdzQJVqKeTX4bgs9PBI6dUP/uAY9ZhLK8NnPLiGcbuOhPN7eeJQjeSX0ax7G1EtiiA72w99q0CshjJ7NQvnpSAGfbM9kzuZjfLw9k9EXNGLEBY3OOx+8uorsJs+sPcSmw/lM7R7D1R1tTnhVvs9nCzqc6naZ/W/YuxPaX+jpdITwemd3rzQL9+exSqbpKaXo1iSEbk1C2JV+kk+2Oy6tX7gjixHtIxndwUZkUO3KS0FJKf9afYhtRwuY3jOO4e0i6/qyGoxqveNbtmzhnXfewTRNhg4dypgxY8o9np6ezuuvv05OTg6hoaHcdtttREV5frqg6tYL7R9wagVGKehCVKa41GTRjiw+2Z6JoeCmbo25qoOtWtMCOzQO4sFBCaQcL+TjbZks3JHFkl+Oc1nbSMZ2tNE4pPoL5eUVlfL4qoPsySzk732bMKhVRF1eVoNTZUE3TZPZs2fz4IMPEhUVxX333UdiYiLNmjUrO2bevHkMGDCAQYMGsW3bNubPn89tt93m0sSrQwUGobr1Rm9aj77+zyirrMAoxO9V1r1SU60aBXJP/6Ycyinm0+2ZfLX7OMv3HGdQqwiu7RRFfPj5L5PPLrTzyHcHOZhdzD39m9InwfVbtvmaKjuW9+7dS1xcHLGxsVitVvr27UtycnK5Y1JTU+nc2bEG+YUXXsjGjRtdky2QWVDC5sN5aF29aVOq90DIz4Vtm12WkxD10dG8Yv61OpUnVqViMRSPDUngnv5Na1XMz9Y03J/b+zThzavbMLxtJGv25XDrF7/x3LpD7DteWOFzMgtKuP+bAxzKKeaBgVLMa6vKFnpWVla57pOoqCj27NlT7pgWLVrw448/MmLECH788UdOnjxJbm4uYWHl/1FWrFjBihUrAHj66aeJjo6uccILk/bzXnIq7RqHcMMlzRjc7vxrKuj+w0h/92X8/vc9kcNGVnqc1WqtVT7u4u35gffn6O35gXtyLLKbzN+UytzkVCwGTO/XkvEXx1dr+mFN8ouOho4tmjAtv5gF/zvMwq1prNufS79WjbixRwKdmzhWKUzLKeTBL7ZxorCUF8Z2plvTunWzNOR/Z6cMik6ePJk5c+awatUqOnbsiM1mw6hgVsmwYcMYNmxY2e2MjIwax7qqTRDhljgW7cji0a9+4bW1vzGmo41hbSIqX9Phkn4UrfuG9IMHUEEVbwsVHR1dq3zcxdvzA+/P0dvzA9fn+Pvulf/rHkPjED+yj2e5NL9xHUK5slVrvtx9nCW7srg55ThdYoMZ2iaCuVvSKbKbPDY4gWYBJXV+/b7+7xwfH1/pY1UWdJvNRmZmZtntzMxMbDbbOcfcddddABQWFrJhwwZCQlyzgI2fxWBYm0iGtI4gOTWPT3dk8dbGo3ywNYNR7R3TpsIDyk+bUr0Gold+id78ParfUJfkJYQ3O3v2StPzzF5xpdAACxMuimZ0Bxtf7z3Bop1ZvJCURkSghRnDmnt8tx9fUGVBb9OmDWlpaRw7dgybzUZSUhK33357uWNOz24xDINFixYxePBglyV8mqFU2XzYneknWbgjkw+2ZrBwRybD2kYypoONmNBTfYGtL4DGcY6LjKSgiwbk7NkrCrixW2NGV3P2iqsE+Rlc3dHGiPaRJB3I5YLoIOLCvGtd8fqqyoJusViYOnUqM2bMwDRNBg8eTEJCAgsWLKBNmzYkJiayY8cO5s+fj1KKjh078sc//tEduQOO+bCdYoLpFBPM/hNFLN7pGF1ftvs4/VuEc00nGy0bBTrmpH+5AH0iExXp+SmVQjhLqanJLS7lxEk72UWlZBeWcqLQTnZhKev255zTveIt/CwGA2VaolMpXd3pIi5w+PBhl5w3Pb+EJbuyWL43m0K7SfcmIYxtYtLp37dhjJuKcfmYc57jrH43rTW5RaUE+RlOXePC1/sF3cHb84MzORbZzbKifHaBPudnkZ3colIqWivLoqB5ZABTLo5xWvdKfXoPvZnH+tDro8Yhfky9JJbxnaNZuuc4X+w6zkNppbTrcxdjtv2PPsN0nVdrM7UmPb+Eg9nFpOYUcTC7mEM5xaRmF5FbbBLsZ9CzaSh9m4fRrUmIV2wgK7xPXnEpuzNOsjP9JL9knCTj5D4y84srXRslyGoQEWghMtBKXJgfHRoHld2ODLQQEWgtux3ib9SbVQKFc/hkQT8tNMDC+M7RXN3Bxne/ZbN4czHPBQwnfvFuxnaJZVCrcPyraEWXlGrScos5mFNEanYxqdmO3w/lFFNceqZZFBFgoVmEP32bh9M03J8D2UVsOJjLqn05BFoNejQNoW/zMC6Jr9mGtMJ3aK05nFvCrvQCdmWcZFf6SQ5mF6MBQ0HLyAA6xYYTaNjPFOgAK5FBjp8RgRb5f0ecl08X9NMCrAZXtm/EZTGa7599gUUXjeHVDZr5P6UzqoONK9pFElRsZ0+m4w8sNbuI1JxiDmYXcySvuNzX2ZgQK83CA7goNpiEiACahfvTLCLgnJk1ALf0jGPb0QKSDuTy/cFc1u7PJcCiSDzVcr8kPrTaeyGK+qfIbrIns5Bd6SfZlVHAroxCcotKAQjxN+gQHUT/FuF0aBxEu6gggvyMetFdILxXgyjop1kjbfSNMejz0+ts/9uLLNx5nHlb0vng53Ts5pmLpSwKmoT50yLSn37Nw2gW4U9CRABNw/0JrKKFpPNz0csXodp2xNqlR9kCRjf3iGX7sTPFff2BXPwtikviQ+jbPJzEpiEE+9V9lTrhOen5JaeKt6P1nXK8kNNf4pqF+9OrWSgdooPo0DiIpuH+0h0inK5BFXRwrMDInBfokn+ArkM68VtWIatSsomzhWOz2mkW4U9cqH+Nd/TWpole9w160VzIy0UHhWA8/ioq0jFn/+wNaf+cGMuu9JOsP5jL9wdy+f5gHn6G4uL4EPo1D6NH01BCnLAEqag9U2vs5un/cPwsPfs+zUm7ya9Zp1rg6SfJPGkHIMCiaBcdxNhOUXRsHET76KAKv8EJ4WwNr6Bf3Avt7+9YgbFtJ1rbAmltC6zTV12dshtz/puwbw+064Qx7GrMt59HL5iFuvmec463GIoLY4O5MDaYP10Swy+ninvSgVx+TM3DakC3OEefe69mYYRKMTgvrTVFpZrsQjs5RaXkFJY6fpb955gRUqwPU1BUUq4o/75Ql5z6WZMd1mJCrFwYE0yHxo7Wd8vIALdtkSbE2RpeQQ8MRnXthU5eh57wpzqtwKhzs9EL56LXfQMRNtSf7kT1HIBSCjVyPPqz99F9h6AuSqz0HIZSdIwJpmNMMFO7x7Ans5CkA7kkHchh4w/5WNQRusaF0Kt1ISWFBRhKYTUUFoMzvyvHh4RFOe63GAqrUhgGpx5Xpx4/c5y/VRERYPHaHWAKSkrJyHcU6Owi+7lFuvD0Y6XkFpWWG6A+m6EgPODUoGJIAIFWhdUwsBqO987PUFgtqux2ufsNhdVCpY/5WRQtIgOIquNiVkI4S4Mr6ACq9yB08lrY/j/o2rPGz9elpejVy9CfvQ9FhajLx6KumoAKPLNOjLriGvSPazDffwPj0VdQgUFVntdQiguig7ggOogpFzdmb9bp4p7L6+v31TjPqgT7GSScGh9IiPAnITyAhIgAokOsbuvfzSsuJTW7mAPZRRzMdkz/PJhdREaBvcLjQ/wMwgIshAdYiAq20rJR4KmCbSE80HF/eIDV8TPQQoifUfahJQOOwtc1yIJOp4shNNzR7VLDgq737HB0r6SmQMeuGNf/BdUk4ZzjlNUPY/KtmM/ei/58Pmp8za6eVUrRLsox++HGbo0JibBxLD2DUn2mS8Buakq1ptR0XC34+9/Pd9zJEpPDuY6ZPBsP5bHi19Ky2IFWRbPwgPLFPiKAmBC/Wncl5BaVlivYB079nnXyTOH2tygSIvzpHBNMQmQAsSF+RJwu0oFWwvwtHr1kXQhv1yALurJaUYmXotevQJ8sqHQFxrPpE1noT99F/7AKbNEY0+6F7n3O22Wh2nVCDRiOXrEE3WsQqkWb2uWrFMH+Fpf2pecUlZJ6VsE9mF3Ez0cKWJmSU3aMv0XRNPx0S/5MsY8LOzOInFNo52AFLe7jhed+YHRrElz2raB5pD+NQ/xk5ocQddAgCzqc6nZZtRT9v+9RfStfsEvb7ejvlqCXfAj2EtTI8agrr0MFVG9lOHXtTeiffsSc9yrGfc+hLN45wBkeYClbE+ds+cWlp+bknynOuzIKWLP/TKG3GhAX6k9eya+cOFlSdn+Q1dGl0z0+tOwDoLmbu3SEaEgabEE/swLjaqikoOudP2F+8BakHYSLEjH+8CdUTOXrKFREBYeiJvwZ/daz6O++QF12tTOyd5sQf0tZv/7ZTpaYHMo505pPzSmmcUQIMQG6rHhHB1u9dtBVCF/UYAu6UgrVcwB66SfoE1mO7VVO0Vnp6I/moDeth8ZxGH99CNW1R+1jJfZDf5/omPXSvS8qqrEzXoJHBfkZtI0KpG3UmW8qMugohGc16OvOVa9BoE3HjBdAl5RgfvkR5kPT0T8no66eiPHYf+pUzMHx4WHcMA20xpz/RrX3QxVCiJposC10ANWkGbRoi96wmqL2nTDfeh6OpUH3PhjjpqKiY50XKyoGdfUN6I/nwOYkuKSf084thBDQwFvo4Niejv17OfHknWAYGH9/DMst9zm1mJfFGnoVNG+D+cFb6II8p59fCNGwSUHvPQjadiT0xukYj7yMuvBi18WyWDBuvBVyHFeYCiGEM0lBD4vA8s9nCBk7qU7LAFQ7Xou2qKFXoVd/hd67w+XxhBANR4Mv6J6grp4ItsaYc19F20uqfoIQQlSDFHQPUIFBGBOnQdpB9PJFnk5HCOEjpKB7iOraA3VJP/QXC9BHXbNZthCiYZGC7kHqD38GP3/Mea/K3HQhRJ1JQfcgFWlDXXMj/LIV/f13nk5HCFHPSUH3MDVgOLTpgP54Djo32y0x9e5tmO+/js7PdUs8IYR7SEH3MGUYGJP/CidPoj+a49JYOvs45ux/Yz53P3rVMvTCeS6NJ4RwLynoXkA1bY4afg36h5XoHVucfn5dWoq54nPMh25Bb1znWAJ40JXotcvR+/Y4PZ4QwjOkoHsJNXIcxDTB/O9r6OIip51X79mB+eQd6AWzoPUFGI/+B2PMJNTYGyEsAnP+m2jTdFo8IYTnSEH3Eso/AGPSdEg/gv7yozqfT+ccx5zzIuaz90JBHsYt92L87VFUrGM9dxUcghr3f5CyG71+RZ3jCSE8r0GvtuhtVMeuqD5D0MsXonv0RzVrWeNzlG1gvfh9KC5y7K40cnyFOyypXoPQa5ajF76H7t4HFRLmhFchhPAUaaF7GTVuKgQFO7peatgVovfuxJzxD/QHb0GrdhiPvoxxzY2VbpenlMKYeDMU5KMXyQCpEPWdFHQvo8LCUeP+CL/uQq/5qlrP0TknMN99CfOZf0JeLsa0f2L8/TFUXLOq4zVrhRo80tFSlwFSIeo1KeheSPUZDB27ohfORZ/IrPQ4bZZirlrqmL3ywyrUFddiPP4q6pJ+NdrLU42eKAOkQvgAKeheSCmFMekWsNsxP3i7wmP0b79gzrgL/f4b0LwNxiMvY1x7EyowqMLjzxsvOAR1nQyQClHfSUH3UiomHjVyPGxOQm/ZUHa/zs3BnPsfzKfuhpzjqL/cg/GPJ1BNEuoWr/cgaNvJMUAqV5AKUS9JQfdiavhYaNoCc/6bmPl5mKu/wnxwGjrpW9TlYzGeeA2jx6U16l6pNJZSGDfIAKkQ9ZkUdC+mrH4Yk2+FE5lkTLsO/d/XIKEVxkMvYYz7P1RgsHPjnT1Aun+vU88thHC9as1D37JlC++88w6maTJ06FDGjBlT7vGMjAxeffVV8vPzMU2TiRMn0r17d5ck3NCoNh1Ql41BbVoPf/gzqucAp7TIK403eiI6eS3m+29g3PssypDPfCHqiyr/Wk3TZPbs2dx///288MILrF+/ntTU1HLHfPrpp/Tp04dnn32Wv//978yePdtlCTdE6ropRL+9CKPXQJcWc5ABUiHqsyoL+t69e4mLiyM2Nhar1Urfvn1JTk4ud4xSioKCAgAKCgpo1KiRa7JtoJRSLi/k5eLJAKkQ9VKVBT0rK4uoqKiy21FRUWRlZZU7Zty4caxdu5Zp06bx1FNPMXXqVOdnKtxGBkiFqJ+cspbL+vXrGTRoEFdddRW7d+/mlVdeYebMmRi/639dsWIFK1Y4vsY//fTTREdHOyO8U1itVq/K5/fcnl90NLkjrqPgy4+JuGo8fm06VPkUeQ/rzttz9Pb8oGHnWGVBt9lsZGaeuVoxMzMTm81W7pjvvvuO+++/H4D27dtTUlJCbm4uERER5Y4bNmwYw4YNK7udkZFRp+SdKTo62qvy+T1P5KcvGwtrvibr1aerNUDqzBy11qC1Uwdlvf3fGLw/R2/PD3w/x/j4+Eofq/KvpU2bNqSlpXHs2DHsdjtJSUkkJiaek9y2bdsASE1NpaSkhPDw8FolK7yHpwZI9dHDmDPuxJz5gCxFIEQNVNlCt1gsTJ06lRkzZmCaJoMHDyYhIYEFCxbQpk0bEhMTufHGG3nzzTf58ssvAZg+fbpbB/GE66je7l1i19ywGj3vNdAmFBeh133j2HdVCFElpbXWngp++PBhT4U+h7d/TfNkfjo1BfOJO1ADhmPccEulx9UlR11chF4w27HCZNuOGH++C3P2C3BoP8aTr6NC6/6Nz9v/jcH7c/T2/MD3c6xTl4sQZVeQrv7KJVeQ6iOHMJ+6B73mK9SV12LcOQNla+xYq/2kzLQRorqkoItqUaOvdyyx+/4bTu3XNjesxnzyH3AiA+P2RzCuuQlldfQEqqYtUEOvQq/9Gp0ia7ULURUp6KJaVHAo6topThsg1cVFmPNeRc+aCQktMR56CXXRJefGvep6CG+E+f7raLO0znGF8GVS0EW1qT6DT11BOrdOV5DqI6mYT92NXrPc0cVy179Qtorn5KqgYMdm1vv3otd9U+uYQjQEUtBFtZXtQZqfh17831qdw9HFciecyDzTxWKxnD9uzwHQvjN64Tx0bk6t4grREEhBFzWiElqhhtR8gLR8F0urSrtYKoypFMbEaacGSOfWNnUhfJ4UdFFjNR0gLd/Fch3GXTMq7WKpNGbT5qhho9HrvkGn7K5t6kL4NCnoosbKDZAmfXveY8/MYsnE+NsjGNfcWGUXS6Vxr/oDRDQ69UEiA6RC/J4UdFErZQOkn1a8xK4uLsKc+59TXSytHV0snavXxVJpzMBgx1IE+/ei18oAqRC/JwVd1Mr5Bkj1kVTMf92FXvt1rbtYKo3bcwBccJFjpo0MkApRjhR0UWsVDZCaP6xydLFkH69zF0uFMZXCuP5mKDopA6RC/I4UdFEnavT1EBqO+f4b5Lz2NHr2v091sbxY5y6WSmM2bY4aemqA9LdfXBJDiPpICrqoExUcWrbE7slvPnd6F0ulca+a4Bggnf+mDJAKcYpTdiwSDZvqMxiyjhHRrQe5zdq4J2ZgMGrcVPTbz6PXfI0adKVb4grhzaSFLupMKYUx6g8EdOvl3rg9+jsGSBe5/wpS/esuSp++B713h1vjCnE+UtBFvVU208bNA6Rm0reYz98Pv+7C/OBt2VVJeA0p6KJeU/GnBkjXfo3+dZdLY+nSUsyPZqPfeQnadkJN+BMc+BW9KcmlcYWoLinoot5TV02ASJtLB0h1fh7mK4+jv/kMNWQUxt8eRQ0ZCU1boBf/F223uySuEDUhBV3UeyowGDX+j47W8prlTj+/TnNcKMWuragb/4px/V9QVivKsGCMnQzHDqOT3LeJthCVkYIufIJKvBQ6dDk1QJrttPPqrZswn7oLTuZj3PkkRv/Lyx/QpQe06YBe8iG6qMhpcYWoDSnowiecGSAtRC+s+wCp1hpz+ULMVx6H6FiMB/6Natep4rjX3AQnstArv6hzXCHqQgq68BmqScKZJXbrMECqi4vQc15Af/IuqntfjH8+g4pqXHnc9hfCRYnoZZ+g8/NqHVeIupKCLnyKGjUBIqMw59duiV19PBPzufvRP6xCjZmEuvkeVEBglc8zxk6GkwXo5Qtrk7YQTiEFXfiUMwOkv6FX12yAVP/2C+aMOyEtFePW+zFGjkcpVb24Ca1QPQegv/0cfSKrNqkLUWdS0IXPUYn9HAOki6s/QGomfYf53P3g54dx37Oobr1rHvfqG6C0FP3FhzV+rhDOIAVd+JxyA6SfvnfeY7VZivnxHPQ7L0KbDhgPzEQ1bVG7uI3jUAOGOy5yOnq4VucQoi6koAuf5BggvRq9fkWlA6S6IA/zlSfQXy9GDR6J8ffHUKHhdYs7cgJY/dCfvV+n8whRG1LQhc8qGyB9//VzBkgduyrdDTt/Rk2+FWPizShr3RcfVRGNHB8kyWvRB36t8/mEqAkp6MJnqcAgxwDpwRT06q/K7tdbNzmKeUEexj+ewBgw3Llxh4+FkDDMRfOcel4hqiIFXfg0ldgPOnZFL/4v5okszOWLMF95AqJjHP3l7S90fszgENSV18G2zehftjr9/EJURpEiG10AAByQSURBVAq68Gln9iAtIvOu/0N/8g50733qYqEY18UdPMLR3bNwLlprl8UR4mxS0IXPU02aoS4fg5mZjrp6IsbN/6zWxUJ1iukf4Nhv9bdf4KcNLo0lxGlS0EWDoMZMIvqthRij/lDti4XqHLPvUIhrirlwnux7KtxCCrpoEJRhYGkc596YFgvGmMmQdhD9/Sq3xhYNkxR0IVypex9o0Rb9+Xx0SbFbQ+uiInRWultjCs+Sgi6ECymlMK69CbLS0auXuS2uTj+C+eQdmA/fis4+7ra4wrOkoAvhYqpjV8fUyS8/Rp8scHk8/dsvmE/dDTnHoaQY/ZWsANlQVOvSuC1btvDOO+9gmiZDhw5lzJgx5R5/99132b59OwDFxcVkZ2fz7rvvOj1ZIeorY+yNmP+6E/3NYtToiS6LozcnYc76N0Q0wrj9EfTyT9Grl6GHj0FFRrksrvAOVRZ00zSZPXs2Dz74IFFRUdx3330kJibSrFmzsmOmTJlS9vuyZctISUlxSbJC1FeqVTu4pC/668/Qg0agwiOden6tNfqbzxzz7Fu2w/jrg44YIyegf1iFXvYp6vq/ODWm8D5Vdrns3buXuLg4YmNjsVqt9O3bl+Tk5EqPX79+PZdeeqlTkxTCFxhjJkFJEXrpx049ry4tRc9/E/3xHLi4D8ZdM8o+MFTjOFTfoeg1X6GzMpwaV3ifKlvoWVlZREWd+aoWFRXFnj17Kjw2PT2dY8eO0blz5wofX7FiBStWOHZHf/rpp4mOjq5Nzi5htVq9Kp/f8/b8wPtz9Hh+0dHkDBnJyVVf0Wj8FCwxTc45pKY5micLyJ75MMWbkgi+eiKhN05HGeXbaaWTbibj++8IWLmE8JvvrtNL8Ph7WA0NOce6Ly93lvXr19O7d28Mo+KG/7Bhwxg2bFjZ7YwM72kxREdHe1U+v+ft+YH35+gN+enLxsLq5WS++x+MqXec83hNctQnMh3r0hzch7phGkWDRlCUVcFuSYYfqt8wTn6zhKJBo867P2pVvOE9rIqv5xgfH1/pY1V2udhsNjIzM8tuZ2ZmYrPZKjw2KSmJfv361SJFIRoGZYtGDRnp6Nc+tL/W59Gp+xwrRh49jHHbgxiDRpw/7ohxoEAv/ajWMYX3q7Kgt2nThrS0NI4dO4bdbicpKYnExMRzjjt06BD5+fm0b9/eJYkK4SvUlddBYHCtl9fV2/+H+cw/QZsY9zyFuujcv8dzYtoao/oPd2z4kX6kVnGF96uyoFssFqZOncqMGTO444476NOnDwkJCSxYsICNGzeWHbd+/Xr69u3rtnUyhKivVEiYY830n35E791Zo+eaa5ZjvvwYRMdi3Pc8qnmb6scdcR0oA/2ltNJ9VbX60Lt370737t3L3TdhwoRyt8ePH++8rITwcWrYaPR3X2AufA/j7qeqbAhp03Rser3sU7jwYseKkUHBNYsZGYUadCX6uy/QI65DxVTeFyvqJ7lSVAgPUAGBqFF/gD07YNvm8x6rS4rRs2Y65pIPGI7x14dqXMzL4l5xLVit6C8W1Or5wrtJQRfCQ1T/y6BxnGMTDNOs8Bidm4P574fQyWtR196EmjS9TnufqohGqEEj0D+sRh9JrfV5hHeSgi6EhyirH+rqGyA1BZ289pzH9dHDmE/fDfv2ov5yD8YV1zpljEoNvwb8/NBLpJXua6SgC+FBqkd/aNYK/dn7aHtJ2f167w5HMS/Iw7jzSYwezrv6WoVHooaMQievQacddNp5hedJQRfCg5RhYFxzI6QfQa/9BgAzeS3mzIcgOAzjvudQbTs6P+7lY8E/EL3kQ6efW3iOU68UFULUQufu0P5C9Bcfkmfa0R/OgradMG69HxUa7pKQKiwcNfQq9LKP0SPGoZq1dEkc4V7SQhfCw5RSGGNvhJwT5H84C9VzAMY/HndZMS+Le/nVEBiEKa10nyEtdCG8gGrbETVyPCGNbBT0v+KcBbZcEjMkzDEffsmH6AO/oZq3dnnM39NFReDvLxckOom00IXwEsaYSYRce6NbivlpathoCArBXPKB22Kepn/dhXn3TehP3nV7bF8lBV2IBkwFhzq6XrZsQO/f67a4ev9ezJceg8KTjitXT2RW/SRRJSnoQjRwauhoCA7F/Nw9rXR9MAXz3w9DcAjGXTNAm7LvqZNIQReigVNBwY7Fwn5ORqfsdmksffgA5r8fgoBAjDufRLXvjOozBL36K2mlO4EUdCEEashICA3H/Hy+y2LoI4ccxdxidRTzxnGO2CPGSSvdSaSgCyFQgcGoK66BbZtrvKRvdehjaZgzHwTTxLjzCVTsmZUeVeM4VO/B0kp3AinoQggA1KAREBbh9Fa6zjzmKOYlxRj/eALVJOHc2CPHg1kqrfQ6koIuhABOLel7xbWw8yf07m1OOac+nuko5oUFGHc8XukVqapxnKMvfc1y9IkK9kUV1SIFXQhRRg28EiIaOWXGi84+7ijmudkYf38M1eL8uyupkeOh1I7+6tM6x26opKALIcqogADHnqe/bEXv+rnW59G52Y5ifiIT42+PoFpVvdewo5U+WFrpdSAFXQhRjhowHCJtmJ/PR2td4+fr/FzHPPOMoxi3PYRq26n6sUecaqUvl7702pCCLoQoR/n5Owrrnh2w86caPVcX5GO+8AgcOYhx6wOoCy6qWeyYJmfNeJFWek1JQRdCnENdehnYomvUSteFBZgvPQqp+zBuuQ914cW1iz1y3KlW+qJaPb8hk4IuhDiH8vNzDFL+ugu2n38TawBdVIj58uOwbw/GX+5GdelR+9gx8ada6cvQ2cdrfZ6GSAq6EKJCqu9QiIrB/Oz8rXRdXIT5nydh7y7Un+5Ede9T99inW+kyL71GpKALISqkrKda6fv2wM8bKzxGl5Rgvv4U/LIV9X9/w+jR3zmxY+JRvQZJK72GpKALISql+gyBxnEV9qVrewnmm8/Ats2oybdi9Bns3NijZMZLTUlBF0JUSlmtqFET4MCv8NOGsvt1aSnm2zPhpx9RE6dh9L/c+bFj4lG9BkorvQakoAshzkv1GgQx8ZiffYA2TbRZip7zImxOQo3/I8bgEa6LPXIClEgrvbqkoAshzktZLKir/gCpKRR9vwr93n/QP65GXXMjxmVXuzZ2bDyq96lWeo77W+n6RBbm+hVos9TtsWtDCroQokqqZ3+Ia0b2S4+jk75FXXU9xpXXuSd2WSvdvfPSdfZxzOcfQL/7MnrDGrfGri0p6EKIKinDgnH1RCgpRl15naPF7q7Ysaf60lctdVsrXefmODbjOJEJMU3Qn72PLilxS+y6kIIuhKgWlXgp0W8vQo2djFLKvbFHjj/VSl/s8lg6Pw/zxYch/QjGXx/EuGEaZB5Dr17q8th1JQVdCFFtluhYtxdzABXX9FQr/Ut0zgmXxSlbvuDQAYzp96E6dEF1uhg6dkV/+RG6IN9lsZ1BCroQol4400p3TV+6LipyLF+wfy/GzfegOl9S9phx7U2Ql+v168tIQRdC1AuOVvoAl7TSdUkx5qtnLV9wce/ysVu0RfXoj16x2KtXgZSCLoSoN8pa6V87r6Ws7SWYrz8NO39CTbmt0uUL1JhJUFqKXvKh02I7mxR0IUS9oeKaoXr2R69c6pRWetkVr1s3oiZNx+g7tPLYMU1QA65Ar/safeRQnWO7ghR0IUS94piXXlLnVnq5K14n/BFj4BVVxx41Hvz8MRfPq1NsV7FW56AtW7bwzjvvYJomQ4cOZcyYMecck5SUxMcff4xSihYtWvC3v/3N6ckKIYRqclYrffg1qLCIGp9DmyZ63muOK17HTsYYVr0rXlV4I9TlY9BLPkT/9guq9QU1ju1KVRZ00zSZPXs2Dz74IFFRUdx3330kJibSrFmzsmPS0tJYvHgxTzzxBKGhoWRnZ9cqGa01hYWFmKbp9qlRR48epaioyOVxtNYYhkFgYKBHpn8J4QvUyAnoH9egly9CXTelRs/VWqM/fAu97hvUqAkYI8bVLPblY9CrlmF++h7GXTO86u+4yoK+d+9e4uLiiI2NBaBv374kJyeXK+jffvstw4cPJzQ0FICIiJp/YgIUFhbi5+eH1VqtLw5OZbVasVgsbollt9spLCwkKCjILfGE8DWqSTNUjwHolV+ih4+tditda43+5F30yqWoy8eiRk+seezAYNSoCegP3oJtm+GiS6p+kptUWTmzsrKIiooqux0VFcWePXvKHXP48GEAHnroIUzTZNy4cXTr1u2cc61YsYIVK1YA8PTTTxMdHV3u8aNHjxIQEFDzV+Ek7vogsVqtKKXOef1VPacmx3uCt+fo7fmB9+foTfnZJ99MZvIaAtcuJ+zG6WX3ny/HvA9mkf/1IoJGXEfYn+6odetaj72BzO++QH3+PraBl6GMmg1Huup9dEoFM02TtLQ0HnnkEbKysnjkkUd4/vnnCQkJKXfcsGHDGDZsWNntjIyMco8XFRW5rZX8e1arFbvd7rZ4RUVF57z+84mOjq7R8Z7g7Tl6e37g/Tl6VX6Boage/SlY+gmF/YeXtdIry9Fc9gl64VzUpZdRdPUkijMz6xTeHD0R/fbzpC/9FKN3zTb3qMv7GB8fX+ljVX6s2Gw2Ms964ZmZmdhstnOOSUxMxGq1EhMTQ5MmTUhLS6tVskIIUV1q1AQoLkJ/ff41XswVnzmKec+BqMnTa9yirjB24qXQvDV6sfcs3FXlq2rTpg1paWkcO3YMu91OUlISiYmJ5Y7p2bMn27dvByAnJ4e0tLSyPvf6JDs7m3fffbfGz5s8eXKtB4KFELWnmiQ4ruBc+SU6N6fCY8zVX6EXzIbufVFT/44ynNMLoAzDsSSAFy3cVWVBt1gsTJ06lRkzZnDHHXfQp08fEhISWLBgARs3OjaO7dq1K2FhYdxxxx089thjTJo0ibCwMJcn72w5OTnMnTv3nPur6oqZN29erQeChRB1U9ZK/+bceelm0rfo/74GXXpg/PlOlJO7dL1t4S6lf7/zqxudHkw9raCggODgYADMD99GH0xxajyV0ArjD3+u8DGr1cqf//xnvv76a1q3bo2fnx8BAQFERESwd+9e1q1bx9SpUzl8+DBFRUX88Y9/ZNKkSQD06tWLZcuWkZ+fz6RJk+jZsycbN24kLi6OOXPmVDib5ezXWh1e1XdZCW/P0dvzA+/P0VvzM996Dv1zMsZTs2jcqjUZGRmYyWvRb8+EDhdh3PYQys/fJbH1/r2YT/4DNWI8xthJ1XqOx/rQG5L777+fFi1a8M033/Dggw+ydetWHn/8cdatWwfAzJkz+eqrr1i6dClz5swhK+vcRXpSUlK46aabWLlyJeHh4Sxd6h1fxYTwZb9vpestP6BnzYS2HTBufcBlxRzOXrjrM48v3OX+Cd/VVFlL2p26detG8+bNy27PmTOHZcuWAY5vFykpKecMECckJNC5c2cAunTpwsGDB92XsBANlIpvjkq8FP3dUk5e0BnzzWehRVuM2x9GBQS6Pv6YSejNSegvPkRNml71E1xEWujncXaXSFJSEmvXrmXJkiWsWLGCzp07V3hl6dnz6C0WC6Wl9WNzWSHqO0crvZCcFx+D+OYYf38UFVj9bs06xY5pghowHL3Wswt3SUE/S0hICHl5eRU+lpubS0REBEFBQezdu5fNmze7OTshxPmo+OaoAcOxtuuE8ffHUcGh7o0/aoLHF+7y2i4XT7DZbPTo0YMhQ4YQGBhY7kquQYMGMW/ePAYOHEibNm3o3r27BzMVQlRE3XALtujoctfOuC22Fyzc5bWzXNzN3VeKyiwX9/P2/MD7c/T2/MCzOerCAsz7b3Z0+dz5ZKVLC8gsFyGE8HKnF+7il62w3f3dslLQhRDCidSA4dA4DvPT99Cm6dbYUtCFEMKJlNXPsf9o6j70j6vdGlsKuhBCOJmnFu6Sgi6EEE7mqYW7pKALIYQLeGLhLinoddCuXTtPpyCE8GLGtTdBXi7663NXgnRJPLdEEUKIBqhs4a5v3LNwl9deKTpr41FSjhc69ZytGgXyp8TKN97417/+RXx8PFOmTAEcqytaLBaSkpLIzs7Gbrdzzz33MHz4cKfmJYTwXe5cuEta6GcZPXo0S5YsKbu9ZMkSxo0bx+zZs1m+fDkff/wxjz/+OB68uFYIUc+4c+Eur22hn68l7SqdO3cmIyODI0eOkJmZSUREBDExMTz66KNs2LABpRRHjhwhPT2dmJgYt+cnhKif1KgJ6KTvMBfPwzLtXpfF8dqC7imjRo3iyy+/5NixY4wePZqFCxeSmZnJsmXL8PPzo1evXhUumyuEEJUpt3BXym44a+E/Z5Iul98ZPXo0n332GV9++SWjRo0iNzeX6Oho/Pz8WL9+PampqZ5OUQhRD6nLx0BYhGNJABd120pB/50LLriA/Px84uLiiI2N5ZprruGnn35i6NChfPLJJ7Rt29bTKQoh6qGzF+4q/t8Gl8SQLpcKfPvtt2W/22y2cgOlZ9uzZ4+7UhJC+AA1YDh622aU1TWlVwq6EEK4ibL6Ybn9Yfyjo8EFa7ZLl4sQQvgIryroDWl+d0N6rUII9/Cqgm4Yhlu3gfMUu92OYXjVWy+E8AFe1YceGBhIYWEhRUVFle7F5yoBAQFumV+utcYwDAIDA10eSwjRsHhVQVdKERQU5JHY9WHzWyGEOB/53i+EED5CCroQQvgIKehCCOEjlJb5c0II4ROkhX7Kvfe6bklLZ/D2/MD7c/T2/MD7c/T2/KBh5ygFXQghfIQUdCGE8BGWRx999FFPJ+EtWrdu7ekUzsvb8wPvz9Hb8wPvz9Hb84OGm6MMigohhI+QLhchhPARUtCFEMJHeNVaLu6WkZHBq6++yokTJ1BKMWzYMEaMGOHptM5hmib33nsvNpvNK6dk5efn88Ybb3Dw4EGUUtxyyy20b9/e02mV88UXX/Ddd9+hlCIhIYHp06fj7+/v0Zxee+01Nm/eTEREBDNnzgQgLy+PF154gfT0dBo3bswdd9xBaGio1+Q3b948Nm3ahNVqJTY2lunTpxMSEuKR/CrL8bQlS5Ywb948Zs2aRXh4uFflt2zZMpYvX45hGHTv3p1JkyY5J6BuwLKysvSvv/6qtda6oKBA33777frgwYMezupcS5Ys0S+++KJ+6qmnPJ1KhV555RW9YsUKrbXWJSUlOi8vz8MZlZeZmamnT5+ui4qKtNZaz5w5U69cudKzSWmtt2/frn/99Vf9j3/8o+y+efPm6UWLFmmttV60aJGeN2+ep9KrML8tW7Zou92utXbk6sn8tK44R621Tk9P108++aS+5ZZbdHZ2toeyqzi/rVu36scff1wXFxdrrbU+ceKE0+I16C6XRo0alY00BwUF0bRpU7KysjycVXmZmZls3ryZoUOHejqVChUUFLBz506GDBkCgNVq9WiLrTKmaVJcXExpaSnFxcU0atTI0ynRqVOnc1rfycnJDBw4EICBAweSnJzsidSAivPr2rUrFosFgPbt23v876WiHAHee+89brjhBrcvw/17FeX39ddfc/XVV+Pn5wdARESE0+I16C6Xsx07doyUlBTatm3r6VTKeffdd5k0aRInT570dCoVOnbsGOHh4bz22mvs37+f1q1bM2XKFK9a791ms3HVVVdxyy234O/vT9euXenataun06pQdnZ22YdNZGQk2dnZHs6oct999x19+/b1dBrnSE5Oxmaz0bJlS0+nUqG0tDR27drFhx9+iJ+fH5MnT3Za3WnQLfTTCgsLmTlzJlOmTCE4ONjT6ZTZtGkTERERXj2ntrS0lJSUFC6//HKeffZZAgICWLx4safTKicvL4/k5GReffVV3nzzTQoLC1mzZo2n06qSUsrjLczKLFy4EIvFQv/+/T2dSjlFRUUsWrSICRMmeDqVSpmmSV5eHjNmzGDy5Mm88MILTtuSssEXdLvdzsyZM+nfvz+9evXydDrl/PLLL2zcuJFbb72VF198kW3btvHyyy97Oq1yoqKiiIqKol27dgD07t2blJQUD2dV3tatW4mJiSE8PByr1UqvXr3YvXu3p9OqUEREBMePHwfg+PHjHhvMO59Vq1axadMmbr/9dq/7wDl69CjHjh3j7rvv5tZbbyUzM5N//vOfnDhxwtOplbHZbPTs2ROlFG3btsUwDHJzc51y7gbd5aK15o033qBp06aMGjXK0+mcY+LEiUycOBGA7du3s2TJEm6//XYPZ1VeZGQkUVFRHD58mPj4eLZu3UqzZs08nVY50dHR7Nmzh6KiIvz9/dm6dStt2rTxdFoVSkxMZPXq1YwZM4bVq1fTo0cPT6dUzpYtW/jss8947LHHCAgI8HQ652jevDmzZs0qu33rrbfy1FNPedUHY48ePdi+fTudO3fm8OHD2O12wsLCnHLuBn2l6K5du3j44Ydp3rx5WUvj+uuvp3v37h7O7FynC7o3Tlvct28fb7zxBna7nZiYGKZPn+6xqXaV+eijj0hKSsJisdCyZUumTZtWNijlKS+++CI7duwgNzeXiIgIxo8fT48ePXjhhRfIyMjw+LTFivJbtGgRdru9LKd27drxl7/8xSP5VZbj6QF68HxBryi/AQMGlI05Wa1WJk+eTOfOnZ0Sr0EXdCGE8CUNvg9dCCF8hRR0IYTwEVLQhRDCR0hBF0IIHyEFXQghfIQUdCHq4NixY4wfP57S0lJPpyKEFHQhhPAVUtCFEMJHNOhL/4VvysrKYs6cOezcuZPAwEBGjhzJiBEj+Oijjzh48CCGYfC///2PJk2acMstt5StypeamsqsWbPYt28fNpuNiRMnkpiYCEBxcTEffvghP/zwA/n5+TRv3pyHHnqoLObatWtZsGABxcXFjBw5kmuuucYTL100cNJCFz7FNE2eeeYZWrZsyZtvvsnDDz/M0qVL2bJlCwAbN26kT58+zJkzh379+vHcc89ht9ux2+0888wzdOnShVmzZjF16lRefvllDh8+DMDcuXP57bffePLJJ3nnnXeYNGlSuYWpdu3axUsvvcRDDz3EJ598Qmpqqkdev2jYpKALn/Lrr7+Sk5PDddddV7ZN2tChQ0lKSgKgdevW9O7dG6vVyqhRoygpKWHPnj3s2bOHwsJCxowZg9VqpXPnznTv3p1169ZhmiYrV65kypQp2Gw2DMPgggsuKLcWzLhx4/D396dly5a0aNGC/fv3e+otEA2YdLkIn5Kens7x48eZMmVK2X2madKxY0eio6OJiooqu98wDKKiosqWq42OjsYwzrRxGjduTFZWFrm5uZSUlBAXF1dp3MjIyLLfAwICKCwsdOKrEqJ6pKALnxIdHU1MTEyF68Z/9NFHZGZmlt02TZPMzMyyHYIyMjIwTbOsqGdkZNCkSRPCwsLw8/PjyJEjXrsLjhAgXS7Cx7Rt25agoCAWL15McXExpmly4MAB9u7dC8Bvv/3Ghg0bKC0tZenSpfj5+dGuXTvatWtHQEAAn3/+OXa7ne3bt7Np0yb69euHYRgMHjyYuXPnkpWVhWma7N69m5KSEg+/WiHKk+Vzhc/Jyspi7ty5bN++HbvdTnx8PBMmTGDXrl3lZrnExcUxbdq0si3+Dh48WG6Wy/XXX0/Pnj0BxyyX+fPn8/3331NYWEjLli154IEHOHHiBH/961/54IMPyjZPfvTRR+nfv7/XbuwtfJcUdNFgfPTRRxw5csTrdn0Swlmky0UIIXyEFHQhhPAR0uUihBA+QlroQgjhI6SgCyGEj5CCLoQQPkIKuhBC+Agp6EII4SP+H1yyjenPc+ZxAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "8SNXU2CKwwdU", + "outputId": "c4ba4191-0a03-4cff-8a2c-afc478a1658e" + }, + "source": [ + "_ = plt.plot(lr_history[:2*iterations_per_epoch])" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxVdf7H8df3ey8uLC4XBNKhJhFnCi0TTKRFFKopmxlqrKlfy7g0Zm6R2ZQyU7ZQtiiOYjtZmc1YU1RTTTZItkgUZtjYKtrGiJJcUxRL4Hx/f9xivAmieOHcA5/n49GjLpx7eJ/T1Tfn+z2LMsYYhBBCiB9ouwMIIYQILlIMQggh/EgxCCGE8CPFIIQQwo8UgxBCCD9SDEIIIfy47Q4QKFu2bGnV+6Kioti+fXuA07QfJ+d3cnZwdn4nZwdn5w+m7H379m3y63LEIIQQwo8UgxBCCD9SDEIIIfxIMQghhPAjxSCEEMLPIZ2VVFZWxtKlS7Esi/T0dDIzM/2+X1dXR15eHps3byYiIoKsrCyio6MBKCgooKioCK0148ePZ8iQIQDce++9rFu3jp49ezJ//vzGde3evZvc3Fy++eYb+vTpwzXXXEN4eHigtlcIIUQLWjxisCyL/Px85syZQ25uLmvWrKGiosJvmaKiIsLCwli8eDFjxoxh+fLlAFRUVFBcXMyCBQvIzs4mPz8fy7IASEtLY86cOQf8vOeee47BgwezaNEiBg8ezHPPPReI7RRCCHGIWiyG8vJyYmNjiYmJwe12k5qaSmlpqd8ya9euJS0tDYCUlBQ2bNiAMYbS0lJSU1MJCQkhOjqa2NhYysvLATj++OObPBIoLS1l5MiRAIwcOfKAnxVIVslr7F6Rj/Xy01ivPodV9CLWGysxa9/CfLwe89VmjPcbTN2+NssghBCtYSq+wHp+OaZmZ8DX3eJQktfrJTIysvF1ZGQkGzdubHYZl8tFaGgoNTU1eL1eEhISGpfzeDx4vd6D/rydO3fSu3dvAHr16sXOnU1vdGFhIYWFhQDMmzePqKioljblADvKStjz3tsHfL2pB1ToXh50n1hcP/7TNw730f1xxx2LDrNvqMvtdrdq24OBk7ODs/M7OTs4O3+gsu/9eB27XlyB51eZuAO8L4L6ymelFEqpJr+XkZFBRkZG4+tWXUk4eTbRHg/bt22D+jqor4e6fbB3D+yugT27MLtrYNe3GO831FdXUV/+Cbz7pm/5H3mioO8xqJ8noPr/AvoPRIVFHH6eVgimqygPl5Ozg7PzOzk7ODt/oLJbu2oA2LFjB6prWKvW0dyVzy0Wg8fjobq6uvF1dXU1Ho+nyWUiIyNpaGigtraWiIiIA97r9XoPeO9P9ezZkx07dtC7d2927NhBjx49Wop4RJTWqJAQCAnZ76v/a9+maslYFlRXwZavMFu+8v37688xH76PMb45FGL6oeJ/CcediDruRFTP3m26HUIIESgtFkN8fDyVlZVUVVXh8XgoLi5mxowZfsskJSWxevVqBg4cSElJCYmJiSilSE5OZtGiRZx77rns2LGDyspKBgwYcNCfl5yczOuvv05mZiavv/46w4YNO7ItbANKa+gTC31iUSee3Ph1891e+LIcs/lTzKZPMOvfheJVvqGpfsf4CmJQEvxiEMod0uz6hRDCTi0Wg8vlYsKECeTk5GBZFqNGjSIuLo4VK1YQHx9PcnIyo0ePJi8vj+nTpxMeHk5WVhYAcXFxjBgxgpkzZ6K1ZuLEiWjtm+9euHAhH330ETU1NUyePJkLL7yQ0aNHk5mZSW5uLkVFRY2nqzqF6tYdfjEY9YvBwA9HFl9/jvm4zDeZ/formMIXoHsYanAy6qThMCjJ9z4hhAgSyhjT1Fyr4zjh7qpm3/fw8XrM+yW+o4ndu8AdAicko1NG+Uoi5PCOJGSs1T5Ozu/k7ODs/AGbY3jndczD89G33ouK/Vmr1tHqOQYROKpLVzjxZNSJJ2MaGmDTx5h1b2PefQNr3dsQGo4adioqJQ3ij2t24l0IIdqSFINNlMsFAwehBg7CXDABPirDlKzGvF2Eef0VOCoOlXY2KmUUKrR1ZxwIIURrSDEEAeVyweAk1OAkzHe1mLVrfPMRf3sQ88xjqOEjfSVxdLzdUYUQQSfwIwtSDEFGdQtFnXoGnHoG5styzOp/Yd5ZjXnzVRg4CH3Web65CC33PxRCtA0phiCmjhmA+sN0zAXjMW8VYla9gLX4Vt8w01nnoU4eaXdEIUQHJL92OoAKDUefmYnOeRA18RpwuTCPLsKa80dq/7nCd7aTEEIEiBwxOIhyu1EpozDD0+CjMqx//YOaR/4KzyxDnTMWddqZqJAudscUQjicFIMDKaUg8SRciSfRY+tX7Fh2n2+i+l/PoM65AHXaGXJltRCi1WQoyeG6DBqKnnU7euatEBWDefJ+rBunYpW+RQe5dlEI0c7kiKEDUErBcSeif3kCfPg+1jOPYh68C/PvgegLJqASjrc7ohAi0NrwFz8phg5EKQWDhqKPPxHz9mrMc8uw7roBhqSgf/cHVGw/uyMKIRxAhpI6IKVd6FPS0bc9gMq8FD5ejzV3Otazj2G+/87ueEKIQGqDW+dIMXRgqmtX9JgL0bffjzr5dMy/nsH6yxTfo0tl/kEI0Qwphk5A9eiNnpCFvn4ehEdgPXAXVu6NmMoKu6MJIYKQFEMnogYcj85egLp4EnxRjnXzDKznn8TU1bX8ZiFEpyHF0Mkolws9+lz0bfehkk/BvPh3rFuzMJs+sTuaECJISDF0UqpHL/QV16Jn3ATf78W683qsvz3oezypEKJTk2Lo5NTgJPTNeai0czCvvYR10zTMhnV2xxJC2EiKQaC6haL/70r0n+ZB125Yf52L9cS9cmqrEEGt7c4slGIQjdSA49B/yUWdmYl5YyXWzTNk7kGIYNcGTwCWYhB+VEgX9AUT0NfmgGVh3XkDVsEyTL2cuSREZyHFIJqkfjEIfdMiVOpozMtPY90+C/Pfr+yOJYRoB1IMolmqeyh63Az01DnwrRcrZybWGyvlqmkhOjgpBtEiNSQFfdMiSDges2wJ5sG7MbV77I4lhGgjUgzikKievdFXz0WdfzlmXTHWLVdjNn9qdywhRBuQYhCHTGmNPnus77RWwLrrBqyVz2Isy+ZkQohAkmIQh03F/xJ940I4cTjmH49iLb4Vs6fG7lhCdC5tONUnxSBaRYWGoydfj7pksu95D7deg/lyk92xhOiE5HkMIogopdBp56D/dIfvmod5f8J66992xxJCHCEpBnHEVP9foP+S6ztr6bHFWI/nYer22R1LCNFKUgwiIFRET3TWXNQ5F2DefBXrzhsw27fZHUsI0QpSDCJglHahz7sMPTUbqiqxbpuJ+Xi93bGEEIdJikEEnBoyHP3n+dCjF9bCm7Bee0mulhbCQaQYRJtQ0X3Rs++GwcmYJx/APHGv3IhPCIeQYhBtRnUPRU+Z45t3eGMlVu6NmJqddscSomNow6Nw96EsVFZWxtKlS7Esi/T0dDIzM/2+X1dXR15eHps3byYiIoKsrCyio6MBKCgooKioCK0148ePZ8iQIQdd53/+8x+eeOIJLMuiW7duTJ06ldjY2EBus2hHSmvUeZdh9T3ad8ZSzrXoadmonx1rdzQhOgZlw3UMlmWRn5/PnDlzyM3NZc2aNVRUVPgtU1RURFhYGIsXL2bMmDEsX74cgIqKCoqLi1mwYAHZ2dnk5+djWdZB1/nwww8zffp07r77bk499VSeeeaZgG+0aH96+Ejf9Q4NDVjzrsese9vuSEKIZrRYDOXl5cTGxhITE4Pb7SY1NZXS0lK/ZdauXUtaWhoAKSkpbNiwAWMMpaWlpKamEhISQnR0NLGxsZSXl7e4zr17fQ+kr62tpXfv3gHcXGEn9fMEdPZ86HcM1v3z2PP8kzIpLUQQanEoyev1EhkZ2fg6MjKSjRs3NruMy+UiNDSUmpoavF4vCQkJjct5PB68Xm/jeppa5+TJk7njjjvo0qUL3bt3Jycnp8lchYWFFBYWAjBv3jyioqIOaYN/yu12t/q9wcBx+aOiMLffz85Ft7L70Ty6V/6XiD9eg3Id0qhmUHHcvt+Pk7ODs/MHKvveiAh2Ab1798Yd4H0RdH8aX3rpJWbPnk1CQgIvvPACjz/+OJMnTz5guYyMDDIyMhpfb9++vVU/LyoqqtXvDQZOzW/+MIPQmL7UFjzB3v9+hb7yOlS3ULtjHRan7ntwdnZwdv5AZbdqfDeu3LFjB8rdtVXr6Nu3b5Nfb3EoyePxUF1d3fi6uroaj8fT7DINDQ3U1tYSERFxwHu9Xi8ej6fZde7atYsvv/yy8SgjNTWVTz+Ve/53REprIi6fgrpsKnz0vu9Kaa8z/6AL0dG0WAzx8fFUVlZSVVVFfX09xcXFJCcn+y2TlJTE6tWrASgpKSExMRGlFMnJyRQXF1NXV0dVVRWVlZUMGDCg2XWGhYVRW1vLli1bAPjggw/o169f4LdaBA19+lnoGTfB9m1Yd8zCfCV3aBXCbi0OJblcLiZMmEBOTg6WZTFq1Cji4uJYsWIF8fHxJCcnM3r0aPLy8pg+fTrh4eFkZWUBEBcXx4gRI5g5cyZaayZOnIjWvi5qap0AV155JfPnz0drTVhYGFdddVUbbr4IBirxJPT1d2ItvgXrrtnoSdehThhmdywhglzbnbihTAc5LeTHo4zD5eSxSnB2/p9mN99WYy2+DSo+R102FX3qGTama1lH2vdO4+T8AZtjKF6FWfpX9O0Povq07lqvVs8xCNFeVK9I9HW3w3En+i6Ge+kpOZ1VCBtIMYigorp1R0/7MyolDfPcE5i/PYCxGuyOJUSnEnSnqwqh3CEwPgt69sasLMDs+hY9cSYqpIvd0YToFKQYRFBSWqPGjsfq6cE8lY9Vsws9NRsVGmZ3NCE6PBlKEkFNn/Fb1BXXwqZPsO6ejfm2uuU3CSGOiBSDCHp6+Ej0jBvhm22+G/BtrWj5TUKIVpNiEI6gjh/iO2Np3/dYd82WC+GEaENSDMIx1DHx6OvvhJAuWPdkYzZ+ZHckIezThmdySzEIR1ExfdHXz4OevbEW3ojZ8J7dkYSwlx0P6hEi2ChPH/R1d0BMP6y8HMzat+yOJESHIsUgHEn16IWelQPHJmA9eA/Wm6/aHUmIDkOKQTiWCg1HZ90CiUMwj+dhvfqc3ZGE6BCkGISjqa5dfRe+JZ2CefoRrOeXy/2VhDhCcuWzcDzlDoFJs2BZKObFFVC7B35/BUrL7z1CtIYUg+gQlHbB5dOgeyjm389D3T64dIqUgxCtIMUgOgylFFwwAUK6Yl5+CurrYNwMX2kI0eG03ZCpFIPoUJRSqPMuxQoJwTy/HOrrYcI1KLd81EUH1QbXMcifFtEh6XN/7yuHfzyKqa/zPS7UHWJ3LCEcQQZgRYelzzofddEf4f0SrHvvwNTtszuSEI4gxSA6NJ3+a9SlU+A/a7HybsN8/73dkYQIelIMosPTI3+FGnc1fLwea9HNmO/22h1JiKAmxSA6BX1KOmriTCj/CGvhTZjaPXZHEiJoSTGITkMPH4me9Cf4YiPWgr9g9tTYHUmIoCTFIDoVlZSKvmo2/PcLrAU3SjkI52rDW79IMYhOR514MnrKHNjypZSD6ADkeQxCBIQanLxfOciwkhD7k2IQnZavHLJhy1dSDkLsR4pBdGpqcNIP5fC1lIMQP5BiEJ2eGpyEnjpHykGIH0gxCAGoQfuVw/w/Y3bvsjuSELaRYhDiB43lUFnhO3KQchCdlBSDEPvxlUO2lIMIfnIdgxDtRw0aKuUgOjUpBiGa4FcO86UcRBAL/PVtUgxCNEcNGoqe9mfYKuUgOpdDeoJbWVkZS5cuxbIs0tPTyczM9Pt+XV0deXl5bN68mYiICLKysoiOjgagoKCAoqIitNaMHz+eIUOGHHSdxhj+/ve/U1JSgtaaM844g3POOSeQ2yzEIVOJJ6Gn/Rkr7zas3BvRM29DhYXbHUuINtXiEYNlWeTn5zNnzhxyc3NZs2YNFRUVfssUFRURFhbG4sWLGTNmDMuXLwegoqKC4uJiFixYQHZ2Nvn5+ViWddB1rl69murqanJzc8nNzeWUU05pg80W4tCpxJN+uH3GV1i5N2Jqd9sdSYg21WIxlJeXExsbS0xMDG63m9TUVEpLS/2WWbt2LWlpaQCkpKSwYcMGjDGUlpaSmppKSEgI0dHRxMbGUl5eftB1vvrqq4wdOxatfdF69uwZ4E0W4vCpwUnoybOh4gushXMxe2vtjiREm2lxKMnr9RIZGdn4OjIyko0bNza7jMvlIjQ0lJqaGrxeLwkJCY3LeTwevF5v43qaWue2bdsoLi7m3XffpUePHowfP56jjjrqgFyFhYUUFhYCMG/ePKKiog55o/fndrtb/d5g4OT8jsuefjbfhYex8+5sXPfmoG9e5Kz8+3Hcvv8JJ+cPVPa9ERHswvf3qivA++KQ5hjaU11dHSEhIcybN4933nmH++67j1tuueWA5TIyMsjIyGh8vX379lb9vKioqFa/Nxg4Ob8js8cfj/7jddQ9eBfbb86iYUo2qms3u1MdNkfu+/04OX+gsls1vlu3eL1eFK5WraNv375Nfr3FoSSPx0N1dXXj6+rqajweT7PLNDQ0UFtbS0RExAHv9Xq9eDyeg64zMjKS4cOHA3DyySfz5ZdfHuo2CtEuVFIq6oprqfvkP1iLb8V8/73dkYQIqBaLIT4+nsrKSqqqqqivr6e4uJjk5GS/ZZKSkli9ejUAJSUlJCYmopQiOTmZ4uJi6urqqKqqorKykgEDBhx0ncOGDWPDhg0AfPTRR802mhB20sNOo8fVN8JnH2ItuQ2zT8pB2CXwFzK0OJTkcrmYMGECOTk5WJbFqFGjiIuLY8WKFcTHx5OcnMzo0aPJy8tj+vTphIeHk5WVBUBcXBwjRoxg5syZaK2ZOHFi46RyU+sEyMzMZNGiRbz00kt069aNK6+8MuAbLUQgdD/9TGq+/Rbz6F+x7r0dPTUbFdLF7lhCHDFlTBvecKMdbdmypVXvc/JYJTg7v5Ozw//yW2/9G/PYYhicjL5qNiokxO5oLeoo+96JAjbH8OarmMfz0Hc+gvK0bvK51XMMQoiD06eegbpsCvxnLdYDd2Lq6+yOJMQRkWIQIgD06b9CXTIZ1r+L9eDdmPp6uyMJ0WpSDEIEiE47B3XRJHi/BOvhezANDXZHEqJVpBiECCCdfi7qwonwXjEmf4GUg3CkoLvATQin02f8FstqwPzjUXC5YPzVKN26C5CEaFYbnjckxSBEG9BnnY9VX4957glQGsbNQGk5QBdtQNlwHYMQonX0mAuxLAvzwpO+I4fLpko5CEeQYhCiDelfX4TVUI956SnQLrj0KlQb/IYnRCBJMQjRxtRvLwGrAfOvZ8Cl4eIrpRxEUJNiEKKNKaXgvMuhwcK8WuA7cvj9FVIOImhJMQjRDpRSMHYcNNRjVv3TN+cwdryUgwhKUgxCtBOlFPz+Ct+w0qvP+c5W+t0fpBxE0JFiEKIdKaXg4ivBGMzKZ0H7hpmkHMThk+sYhOgwGsvBMr4JaaUh81IpB9E6bfCxkWIQwgZKa7hkMhgL8/LTvouUfnuJlIMIClIMQthEaQ2XTvENK730FGiN+s3/2R1LCCkGIeyktIbLpvqOHP75dywU+jcX2x1LdHJSDELYTGkNl08HA+aff8PSCn3uRXbHEp2YFIMQQUBpDX+Y5jtyeP5JLKXRYy60O5bopKQYhAgSSrtg3AzfnMNzT2AphT7nArtjiU5IikGIIKK07/kNWAZTsMx35HD27+yOJYJR213GIMUgRLBR2gUTsgCDefYx35zDWefbHUt0IlIMQgQh5XLBhGt8w0r/eNQ3rHTmeXbHEkFJHtQjRKehXC6YOBMsC/P0Ut+w0hm/tTuW6ASkGIQIYsrlgiuuxWAwT+X7jhwyfmN3LNHByXMGhQhyyu1GXzELho7ArHgYa9WLdkcSHZwUgxAOoNxu9B+vg5NSMH9/EKtIykG0HSkGIRxCud3oSdfBkOGYvz2IteqfdkcSHZQUgxAOotwh6Cv/9MORw0NYrxbYHUl0QFIMQjiMcoegJ/0JlXSK72yll5+2O5Kwg5EH9Qgh9qPcbvjjLHC5fVdINzSgfy033uuU2uAZHlIMQjiU7zqHLHC5MC88idVQj5KH/YgAkGIQwsEab7zndvse9lNfD7/7g5SDOCJSDEI4XOOT4FwuzMpnoaEeLpwo5SBaTYpBiA5AaQ3/N9k351D4gq8cLprk+7oQh+mQiqGsrIylS5diWRbp6elkZmb6fb+uro68vDw2b95MREQEWVlZREdHA1BQUEBRURFaa8aPH8+QIUMOaZ2PPPIIr732GsuWLQvEdgrR4Sml4PdX+I4cXn0OGhrgkqukHMRha/ETY1kW+fn5zJkzh9zcXNasWUNFRYXfMkVFRYSFhbF48WLGjBnD8uXLAaioqKC4uJgFCxaQnZ1Nfn4+lmW1uM5NmzaxZ8+eAG+qEB2fUgo1djzq7LGYN1ZiHl+MsRrsjiUcpsViKC8vJzY2lpiYGNxuN6mpqZSWlvots3btWtLS0gBISUlhw4YNGGMoLS0lNTWVkJAQoqOjiY2Npby8/KDrtCyLJ554gksvvTTwWytEJ6CUQp13GercizBrVmEeWYhpkHLoeGy8jsHr9RIZGdn4OjIyko0bNza7jMvlIjQ0lJqaGrxeLwkJCY3LeTwevF5v43qaWucrr7xCUlISvXv3PmiuwsJCCgsLAZg3bx5RUVEtbUqT3G53q98bDJyc38nZwQH5J85gT8+e7F7+AF0w9Lz2FlRIF8AB2Vvg5PyByl4bHk4N4ImMxNXLc+TB9hNUk89er5e3336buXPntrhsRkYGGRkZja+3b9/eqp8ZFRXV6vcGAyfnd3J2cEj+tDGoBsP3f3+QqrlZ6ClzUF27OSP7QTg5f6CyW7t3A+D1VqPqrVato2/fvk1+vcWhJI/HQ3V1dePr6upqPB5Ps8s0NDRQW1tLRETEAe/1er14PJ5m1/nFF1+wdetWZsyYwdSpU9m3bx/Tp08/vC0VQvjR6eeixl0NH3+AlXsjpna33ZFEkGuxGOLj46msrKSqqor6+nqKi4tJTk72WyYpKYnVq1cDUFJSQmJiIkopkpOTKS4upq6ujqqqKiorKxkwYECz6xw6dCgPPfQQS5YsYcmSJXTp0oXFixe3yYYL0ZnoU9LRV14HX5Rjzf8z1s4ddkcSQazFoSSXy8WECRPIycnBsixGjRpFXFwcK1asID4+nuTkZEaPHk1eXh7Tp08nPDycrKwsAOLi4hgxYgQzZ85Ea83EiRPRP5w619Q6hRBtRyWdgu7SDeu+O/BmT8FcPRfVO7LlN4pORxnThrfoa0dbtmxp1fucPFYJzs7v5Ozg3Pzmsw2YvNswYRHombei+sTaHemwOXXfQwDnGFa/jFl+P3r+Y6geBz9ZpzmtnmMQQnQsauAget+yGPbWYt15A2bLV3ZHEkFGikGITihkwHHo624HDNbdszFfltsdSRyuNhzrkWIQopNS/Y5B/+kO6Nod655szMfr7Y4kgoQUgxCdmIrui77+ToiMxlp0M2btW3ZHEoct8HfRlWIQopNTvSPR190BPx+I9eDdWK+9bHckYTMpBiEEKiwcfc3NcMIwzJP3Yz2/nA5ywqJoBSkGIQQAqktX9FWzUaeegXlxBWbZErn5XicVVPdKEkLYS7lccPk06NEb8/JTmJpd6EmzGm++JzoHOWIQQvhRSqHPuxR10SRY/w7Wwpvk/kqdjBSDEKJJOv1c1BXXwqZPse6ajfF+Y3ck4aft5oCkGIQQzdInn46++ibwfoN1+3WYrzbZHUm0AykGIcRBqeNO9F3r4NK+I4f/rLU7ktifkusYhBA2UP2OQc++G2L6YS2+DWv1v+yOJNqQFIMQ4pCoXpG++ysNGopZfh/WP5ZirNY9OUwENykGIcQhU926o6dmo9LOxqwswDx4N2bf93bHEgEm1zEIIQ6Lcrng/yZDn1jM00sx31b7niXdo5fd0USAyBGDEOKwKaXQZ56Hnnw9fLUZ6/ZZmIrP7Y4lAkSKQQjRairpFN8N+BrqseZdj3m/xO5InUcb3stKikEIcUTUsQno7PnQ92ise2/HenGF3IDP4aQYhBBHTPWKRM/KQQ0fiXl+OeahezDfy6R0+wj8dQwy+SyECAjVpStMnAn9jsEULMNUVfrOYOodaXc0cZjkiEEIETBKKfTZY9FTs2Hrf7FyZmLKP7Y7ljhMUgxCiIBTJ56Mnn0XdOmKdc8crKIXZd7BQaQYhBBtQvU7Bv3nBZA4FPO3BzH5CzDff2d3LHEIpBiEEG1GhYb75hl+ewnm3Tew7rgOU7XF7liiBVIMQog2pbRGn/t79Iyb4Fsv1m3XYta/a3cscRBSDEKIdqEGDfUNLUUfhZV3G1bBMnmm9JFowykbKQYhRLtRUTHo6+ehTjsT8/LTWPfMkSfDHanAX8YgxSCEaF8qpAv68mm+x4Z+/QXWzVdjyuRWGsFEikEIYQs9fCT6L7kQFYO15Hasvz2IqauzO5ZAikEIYSMV0xd9w12o9F9jil7EmncdZpuctWQ3KQYhhK1USAj6oj/6rpau/gbr1iysN1bKBXE2kmIQQgQFNWQ4+sa/Qv9fYJYtwVp8K2bnDrtjdUpSDEKIoKE8Ueism1G/vwI++QBr7jTMe8V2x+p0pBiEEEFFaY3O+I1vYjoyBuv+eViP5GJq99gdLci03VDbId12u6ysjKVLl2JZFunp6WRmZvp9v66ujry8PDZv3kxERARZWVlER0cDUFBQQFFREVprxo8fz5AhQw66zkWLFrFp0ybcbjfx8fFMmjQJt1vuDi5EZ6OOikPfcBfmpRWYl5/GfPof9KVTUIOT7Y7W4bV4xGBZFvn5+cyZM4fc3FzWrFlDRUWF3zJFRUWEhYWxePFixowZw/LlywGoqKiguLiYBQsWkJ2dTX5+PpZlHXSdp556KgsXLuSee+5h3759FBUVtcFmCyGcQLnd6N9egr7+TujaHWvRLVgPzceSuYf/UYG/wgiKZFAAAA0PSURBVK3FYigvLyc2NpaYmBjcbjepqamUlpb6LbN27VrS0tIASElJYcOGDRhjKC0tJTU1lZCQEKKjo4mNjaW8vPyg6xw6dChKKZRSDBgwgOrq6oBvtBDCWVT/X6D/shD164sw761h+4xLsEpWy5lLbaTFMRqv10tk5P+ewBQZGcnGjRubXcblchEaGkpNTQ1er5eEhITG5TweD16vt3E9B1tnfX09b775JuPGjWsyV2FhIYWFhQDMmzePqKioljalSW63u9XvDQZOzu/k7ODs/I7NPmEG9elj2HXfndTlLyCk7G16XHkdrj6xdic7ZIHa97VhYdQAkZ5IdESPIw+2n6AdvH/44Yc57rjjOO6445r8fkZGBhkZGY2vt2/f3qqfExUV1er3BgMn53dydnB2fidnJ6wnkTn38c3Tj7GvYBnbp12MOucC1JnnoUJC7E7XokDte2uPbzK+2luN+n5fq9bRt2/fJr/e4lCSx+PxG86prq7G4/E0u0xDQwO1tbVEREQc8F6v14vH42lxnU8//TS7du3i8ssvP8TNE0J0Jsrl8p25dMsSGDQU89wTWHOnYza8Z3e0DqHFYoiPj6eyspKqqirq6+spLi4mOdn/rICkpCRWr14NQElJCYmJiSilSE5Opri4mLq6OqqqqqisrGTAgAEHXeeqVatYv349WVlZaC1n0wohmqcio3FdNRt99VwArL/eTMO9t2Oqq+wN5nAtDiW5XC4mTJhATk4OlmUxatQo4uLiWLFiBfHx8SQnJzN69Gjy8vKYPn064eHhZGVlARAXF8eIESOYOXMmWmsmTpzY+Jd9U+sEeOihh+jTpw/Z2dkADB8+nLFjx7bV9gshOgA1aCh67mLMv5/DvPQU1o1TUBmZqF+dj+oeane8ttGGE+/KdJBp/S1bWnfjLUePteLs/E7ODs7O7+TscPD8pvobzLOPYd59AyJ6on59Eeq0s1BBcj1UwOYYCl/ArHgYvXA5KiyiVeto9RyDEEI4iYrsg/7jLHT2fDgqDvPkA1g3TcOsK+6gp7facB2DEEI4kfp5AnpWDnraX8DlwrpvHtYd12E2rOugBRE4wXFsJYQQbUApBScOQw8aillT6Jt/+OtciP8l+tcXw/FDfMsIP1IMQogOT7lcqNPPwqSOxqxZhXn5KayFN/kK4tyLIPEkKYj9SDEIIToN5Q5BjfwVJjXddwTx8tO+I4if/dx3gdyw04JmktpOMscghOh0VEgIOu1sdM4DqHEzoKEB80gu1pxJWCsLOv0tvqUahRCdlgoJQZ2SgUlNhw3rsFY+i/nHUsw//45KGYk6/Veoo/vbHbMZNj+PQQghOjKlFAxOwjU4CfNlOWbVi5jiIszrr8CxA1Ejz0Yln4rq2tXuqO1ChpKEEGI/6pgB6AlZ6Lsf9T1idG8t5tG/Yl03DuuxxZhPPsBYlt0x/6cN5szliEEIIZqgwsJRGb/BpP8aNn6IeevfmNK3MG/9G3pFok4+HTX8dIjr3+HOaJJiEEKIg1BKwcBBqIGDMJd8j/ngXUzJasyqFzCvFkBkNOrEk1FDhkNCYoc4q8n5WyCEEO1Ede2KGnYaDDsNU7MLU1aCWf8u5s1XMUUvQmgYKnEo/PIE1C9PgD6xjjyakGIQQohWUBE9UKedCaedifn+O/ioDFP2DubD96H0Td85Q5HRqF8O9h1J/HwgHNXP5tSHRopBCCGOkOraDU5KQZ2U4rsP09b/Yj5Z75uofv8dWLPKVxRdu+NNOA6r3zHQ7+eoo34GsT9Ddetu9yb4kWIQQogAUkrBUT/z/aU/aozvDKaqLZjPN8Lnn2EqPses+ifU1//vSgRPH4jth/L0gd5R4IlCeaKgRy8IDff907Vbuw1LSTEIIUQbUlr7jgpifwYjRhEZFcU3W7fCN1uh8itMZQVs+RpTtQXz3y9h5w6gicvXXC7oHgruEHC54fu9bZZZikEIIdqZcrvhqJ/5jix+8j1TXwc7qmHHdqjZhandDbW7Yc9u2FsLDfVQXwcNDdArErqHBTyfFIMQQgQR5Q6BPrG+f2iT69daJFc+CyGE8CPFIIQQwo8UgxBCCD9SDEIIIfxIMQghhPAjxSCEEMKPFIMQQgg/UgxCCCH8KGNM2z04VAghhON0+iOGG264we4IR8TJ+Z2cHZyd38nZwdn5nZC90xeDEEIIf1IMQggh/Ljmzp071+4Qduvfv7/dEY6Ik/M7OTs4O7+Ts4Oz8wd7dpl8FkII4UeGkoQQQviRYhBCCOGnUz+op6ysjKVLl2JZFunp6WRmZtod6QBTp06lW7duaK1xuVzMmzeP3bt3k5ubyzfffEOfPn245pprCA8PxxjD0qVLef/99+natStTpkxp97HMe++9l3Xr1tGzZ0/mz58P0Kq8q1ev5tlnnwXg/PPPJy0tzZbsTz31FKtWraJHjx4AXHzxxQwdOhSAgoICioqK0Fozfvx4hgwZAtj3udq+fTtLlizh22+/RSlFRkYG55xzjiP2f3PZnbL/9+3bx0033UR9fT0NDQ2kpKRw4YUXUlVVxcKFC6mpqaF///5Mnz4dt9tNXV0deXl5bN68mYiICLKysoiOjj7odrUr00k1NDSYadOmma1bt5q6ujoza9Ys8/XXX9sd6wBTpkwxO3fu9PvasmXLTEFBgTHGmIKCArNs2TJjjDHvvfeeycnJMZZlmU8//dTMnj273fN++OGHZtOmTWbmzJmtzltTU2OmTp1qampq/P7bjuwrVqwwzz///AHLfv3112bWrFlm3759Ztu2bWbatGmmoaHB1s+V1+s1mzZtMsYYU1tba2bMmGG+/vprR+z/5rI7Zf9blmX27t1rjDGmrq7OzJ4923z66adm/vz55q233jLGGPPAAw+YlStXGmOMeeWVV8wDDzxgjDHmrbfeMgsWLDjodrW3TjuUVF5eTmxsLDExMbjdblJTUyktLbU71iEpLS1l5MiRAIwcObIx99q1azn99NNRSjFw4ED27NnDjh072jXb8ccfT3h4+BHlLSsr44QTTiA8PJzw8HBOOOEEysrKbMnenNLSUlJTUwkJCSE6OprY2FjKy8tt/Vz17t278Tf+7t27069fP7xeryP2f3PZmxNs+18pRbdu3QBoaGigoaEBpRQffvghKSkpAKSlpfnt+x+PwlJSUtiwYQPGmGa3q7112qEkr9dLZGRk4+vIyEg2btxoY6Lm5eTkAHDGGWeQkZHBzp076d27NwC9evVi586dgG+boqKiGt8XGRmJ1+ttXNYuh5v3p/9vPB7PQf+SaGsrV67kjTfeoH///lx++eWEh4fj9XpJSEhoMmMwfK6qqqr4/PPPGTBggOP2//7ZP/nkE8fsf8uyuP7669m6dStnnXUWMTExhIaG4nK5Dsi4/z52uVyEhoZSU1Nz0O1qT522GJzi1ltvxePxsHPnTm677Tb69u3r932lFErZ8bjw1nFa3jPPPJOxY8cCsGLFCh5//HGmTJlic6qD++6775g/fz7jxo0jNDTU73vBvv9/mt1J+19rzd13382ePXu455572LJli92RWq3TDiV5PB6qq6sbX1dXV+PxeGxM1LQfM/Xs2ZNhw4ZRXl5Oz549G4eIduzY0Tgx5/F42L59e+N7g2WbDjfvT//feL1e27ajV69eaK3RWpOens6mTZuAAz8/P2a0+3NVX1/P/PnzOe200xg+fDjgnP3fVHan7X+AsLAwEhMT+eyzz6itraWhocEv40/zNzQ0UFtbS0RERNB89jttMcTHx1NZWUlVVRX19fUUFxeTnJxsdyw/3333HXv37m387w8++ICjjz6a5ORkXn/9dQBef/11hg0bBkBycjJvvPEGxhg+++wzQkNDbR9G+jHX4eQdMmQI69evZ/fu3ezevZv169fbc2YG+M3RvPvuu8TFxTVmLy4upq6ujqqqKiorKxkwYICtnytjDPfffz/9+vXj3HPPbfy6E/Z/c9mdsv937drFnj17AN8ZSh988AH9+vUjMTGRkpISwHem149ZkpKSWL16NQAlJSUkJiailGp2u9pbp77yed26dTz22GNYlsWoUaM4//zz7Y7kZ9u2bdxzzz2A77eKU089lfPPP5+amhpyc3PZvn37Aacf5ufns379erp06cKUKVOIj49v18wLFy7ko48+oqamhp49e3LhhRcybNiww85bVFREQUEB4DtdctSoUbZk//DDD/niiy9QStGnTx8mTZrUWLbPPvssr732Glprxo0bx0knnQTY97n65JNPuPHGGzn66KMbh4suvvhiEhISgn7/N5d9zZo1jtj/X375JUuWLMGyLIwxjBgxgrFjx7Jt2zYWLlzI7t27OfbYY5k+fTohISHs27ePvLw8Pv/8c8LDw8nKyiImJuag29WeOnUxCCGEOFCnHUoSQgjRNCkGIYQQfqQYhBBC+JFiEEII4UeKQQghhB8pBiGEEH6kGIQQQvj5f/eGqvJ09q1oAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Yy8lkzpGwwdV" + }, + "source": [ + "As expected, the learning rate is updated in accordance with cosine annealing schedule." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "t_4cZUXgwwdV" + }, + "source": [ + "The training process was terminated after _16 epochs_. Now we're going to restore the best weights saved during training, and apply the model to the validation subset of the data to see the final model's performance:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "b9WGkwZ7wwdV", + "outputId": "90510a76-d643-4682-a7b3-aaf17de15494" + }, + "source": [ + "net.load_state_dict(best_weights)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 41 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "lI_DqmEIwwda" + }, + "source": [ + "# groud_truth, predictions = [], []\n", + "\n", + "# with torch.no_grad():\n", + "# for batch in batch_generator(*datasets['val'], shuffle=False, bs=bs):\n", + "# x_batch, y_batch = [b.to(device) for b in batch]\n", + "# outputs = net(x_batch[:, 1], x_batch[:, 0], minmax)\n", + "# groud_truth.extend(y_batch.tolist())\n", + "# predictions.extend(outputs.tolist())\n", + "\n", + "# groud_truth = np.asarray(groud_truth).ravel()\n", + "# predictions = np.asarray(predictions).ravel()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "YdXslUMBwwda" + }, + "source": [ + "# final_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n", + "# print(f'Final RMSE: {final_loss:.4f}')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "hB9t-ARGwwdb" + }, + "source": [ + "with open('best.weights', 'wb') as file:\n", + " pickle.dump(best_weights, file)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8Qcj-GoNwwdb" + }, + "source": [ + "### Embeddings Visualization\n", + "\n", + "Finally, we can create a couple of visualizations to show how various movies are encoded in embeddings space. Again, we're repeting the approach shown in the original post and apply the [Principal Components Analysis](http://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html) to reduce the dimentionality of embeddings and show some of them with bar plots." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OqwMsqWHwwdc" + }, + "source": [ + "Loading previously saved weights:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "JKlP4S1zwwdc", + "outputId": "ed020588-98cd-4d38-9ba8-4fee5dcd31b1" + }, + "source": [ + "with open('best.weights', 'rb') as file:\n", + " best_weights = pickle.load(file)\n", + "net.load_state_dict(best_weights)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 45 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "omN9TxzFwwdd" + }, + "source": [ + "def to_numpy(tensor):\n", + " return tensor.cpu().numpy()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_8x4DFcbwwdd" + }, + "source": [ + "Creating the mappings between original users's and movies's IDs, and new contiguous values:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6TOmrBbbg9WA", + "outputId": "6b34ad31-b1a9-4dbe-b12f-040dcb3a2c2b" + }, + "source": [ + "maps.keys()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "dict_keys(['ITEMID_TO_IDX', 'IDX_TO_ITEMID'])" + ] + }, + "metadata": {}, + "execution_count": 49 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qdHe7Nekwwdd", + "scrolled": true + }, + "source": [ + "user_id_map = umap['USERID_TO_IDX']\n", + "movie_id_map = imap['ITEMID_TO_IDX']\n", + "embed_to_original = imap['IDX_TO_ITEMID']\n", + "\n", + "popular_movies = ratings_df.groupby('ITEMID').ITEMID.count().sort_values(ascending=False).values[:1000]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L7cJFbDXwwde" + }, + "source": [ + "Reducing the dimensionality of movie embeddings vectors:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "A0My7Epuwwde", + "outputId": "44f167aa-656f-4fbc-ee20-898672fd5ee4" + }, + "source": [ + "embed = to_numpy(net.m.weight.data)\n", + "pca = PCA(n_components=5)\n", + "components = pca.fit(embed[popular_movies].T).components_\n", + "components.shape" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(5, 1000)" + ] + }, + "metadata": {}, + "execution_count": 52 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MeHZJsE3wwdf" + }, + "source": [ + "Finally, creating a joined data frame with projected embeddings and movies they represent:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "g4_xKa2p7bIO", + "outputId": "1f7539cf-8346-4f83-9a33-9d9abcd739e2" + }, + "source": [ + "movies = movies_df[['ITEMID','TITLE']].dropna()\n", + "movies.shape" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(1682, 2)" + ] + }, + "metadata": {}, + "execution_count": 53 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "YkGwvIUqwwdf", + "outputId": "c99192b1-7bd9-41da-efff-dd2e5a379e5a" + }, + "source": [ + "components_df = pd.DataFrame(components.T, columns=[f'fc{i}' for i in range(pca.n_components_)])\n", + "components_df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
fc0fc1fc2fc3fc4
0-0.013240-0.0391030.061148-0.050387-0.020236
1-0.020302-0.040640-0.0052810.0136270.022263
2-0.046928-0.006252-0.027646-0.0124380.033915
30.068046-0.001613-0.0422350.006097-0.029498
4-0.056585-0.006064-0.015407-0.017673-0.018524
\n", + "
" + ], + "text/plain": [ + " fc0 fc1 fc2 fc3 fc4\n", + "0 -0.013240 -0.039103 0.061148 -0.050387 -0.020236\n", + "1 -0.020302 -0.040640 -0.005281 0.013627 0.022263\n", + "2 -0.046928 -0.006252 -0.027646 -0.012438 0.033915\n", + "3 0.068046 -0.001613 -0.042235 0.006097 -0.029498\n", + "4 -0.056585 -0.006064 -0.015407 -0.017673 -0.018524" + ] + }, + "metadata": {}, + "execution_count": 54 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "chcYgtkw8ij6", + "outputId": "4e996939-45d5-4031-ca55-5cef20b87a6a" + }, + "source": [ + "components_df.shape" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(1000, 5)" + ] + }, + "metadata": {}, + "execution_count": 55 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 173 + }, + "id": "lTu7ZbLL8NaO", + "outputId": "820a8b21-2a52-4c36-983b-cc0c9123efff" + }, + "source": [ + "movie_ids = [embed_to_original[idx] for idx in components_df.index]\n", + "meta = movies.set_index('ITEMID')\n", + "components_df['ITEMID'] = movie_ids\n", + "components_df['TITLE'] = meta.reindex(movie_ids).TITLE.values\n", + "components_df.sample(4)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
fc0fc1fc2fc3fc4ITEMIDTITLE
667-0.0117020.0078130.044108-0.036211-0.002407667Audrey Rose (1977)
703-0.0073030.0268040.0735950.0660340.041713703Widows' Peak (1994)
879-0.0255450.039012-0.001066-0.0167790.006389879Peacemaker, The (1997)
197-0.0386260.002753-0.045264-0.0338070.004753197Graduate, The (1967)
\n", + "
" + ], + "text/plain": [ + " fc0 fc1 fc2 ... fc4 ITEMID TITLE\n", + "667 -0.011702 0.007813 0.044108 ... -0.002407 667 Audrey Rose (1977)\n", + "703 -0.007303 0.026804 0.073595 ... 0.041713 703 Widows' Peak (1994)\n", + "879 -0.025545 0.039012 -0.001066 ... 0.006389 879 Peacemaker, The (1997)\n", + "197 -0.038626 0.002753 -0.045264 ... 0.004753 197 Graduate, The (1967)\n", + "\n", + "[4 rows x 7 columns]" + ] + }, + "metadata": {}, + "execution_count": 60 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rW-l-0Izwwdg" + }, + "source": [ + "def plot_components(components, component, ascending=False):\n", + " fig, ax = plt.subplots(figsize=(18, 12))\n", + " \n", + " subset = components.sort_values(by=component, ascending=ascending).iloc[:12]\n", + " columns = components_df.columns\n", + " features = columns[columns.str.startswith('fc')].tolist()\n", + " \n", + " fc = subset[features]\n", + " labels = ['\\n'.join(wrap(t, width=10)) for t in subset.TITLE]\n", + " \n", + " fc.plot(ax=ax, kind='bar')\n", + " y_ticks = [f'{t:2.2f}' for t in ax.get_yticks()]\n", + " ax.set_xticklabels(labels, rotation=0, fontsize=14)\n", + " ax.set_yticklabels(y_ticks, fontsize=14)\n", + " ax.legend(loc='best', fontsize=14)\n", + " \n", + " plot_title = f\"Movies with {['highest', 'lowest'][ascending]} '{component}' component values\" \n", + " ax.set_title(plot_title, fontsize=20)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 704 + }, + "id": "WpJkrTX9-Asp", + "outputId": "6724cabe-23d6-4ffe-8a21-1c3ba6cb3066" + }, + "source": [ + "plot_components(components_df, 'fc0', ascending=False)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAAMFCAYAAABDA0wqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde1yUZf7/8fcwI6ggMEqjKQieT5snQAwtPKBUnovS1S0NO9A3Navtt9JhZctSt93Nylr9WnksTVrLY2lSVmqaluX5UEqGmngAz2LA9fujL7NODIjD6Ki9no9Hj+K6r/u+P3PNfU/Mm/u+bosxxggAAAAAAOAy8/N1AQAAAAAA4PeJUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAwDVqyJAhslgsysrK8nUpFebJa1mxYoUsFovS09MrvP+srCxZLBYNGTKk3OtMmzZNFotF06ZNu+z7vtrMnj1bbdq0UbVq1WSxWDRy5EhflwR4ncViUadOnXxdBgBccQglAMADFotFFotFfn5++uGHH0rt17lzZ2ffin45havfw5f1K5GnYVdxSLNixQqX9i+//FKDBg3SiRMn9NBDD2n06NG65ZZbPKrtzJkzGj16tJo0aaLKlSvL4XDorrvu0rZt29z25/i59qSnp7s9zgAAVy5CCQDwkM1mkzFGb775ptvlu3bt0ooVK2Sz2S5zZb8aO3astm3bpjp16vhk/950Nb6Wfv36adu2berXr5+vS7miLV68WMYYzZgxQ3//+9+Vnp7uUSiRn5+vbt266dlnn1VwcLAeeeQRJSYm6v3331dMTIzWrl17CaoHAAAVRSgBAB6qWbOmYmJiNHXqVBUUFJRY/sYbb0iSevXqdblLkyRdf/31atq0qSpVquST/XvT1fhaQkJC1LRpU4WEhPi6lCva/v37JUm1a9eu0Hb+9a9/adWqVUpOTtbatWs1fvx4vfPOO3rvvfd0+vRppaSkqKioyBslAwAALyKUAIAKuP/++/Xzzz9r0aJFLu2//PKLpk2bpvj4eDVv3rzU9Xft2qV77rlHderUkb+/v2rXrq177rlHu3btcumXmpoqi8Wi+fPnu93O2rVrZbFYlJyc7Gwr6zL7tWvXKjk5WbVq1ZK/v78iIiL04IMPOr8gnm/37t164IEH1LBhQ1WpUkXVq1fXDTfcoNTUVB05cqSs4ZH065dNd1c4REZGymKx6LnnnnNp//DDD2WxWPTXv/611NeSnp6uevXqSZKmT5/uvEWmtNtkvv32W/Xo0UOhoaGqWrWqEhIStHr16gvW7k5WVpYGDBigsLAwVa5cWTExMSXef6nsOSWWLl2qDh06KDAwUNWrV1ffvn21ffv2C94aUd59F5s9e7Y6d+6s0NBQVa5cWc2aNdOYMWOUn59fou8XX3yhXr16KTw8XAEBAapVq5bat2+vv/3tb84+FotF06dPlyTVq1fPOeZRUVFlD5obxeMzderUEts7//VnZ2drxIgRatSokfP4a9eunctxY4zRpEmTJEl///vf5ef3319v+vTpo5tuuklbt27VZ599dtF1/tayZcvUq1cvORwOBQQEKCIiQn369NHy5ctd+hUVFWnSpEmKjY1VUFCQAgMDFRsbq3//+99uw5Hi+QYOHjyolJQU1axZU4GBgYqPj9cXX3whSTp16pSeeOIJRUZGKiAgQC1atFBGRkaJbZ1/7C1evFjx8fEKDAyU3W5XcnJyic+XYgcOHNDDDz+sqKgo+fv767rrrtPtt9+ur7/+usx9fPrpp+rUqZOqVaum4OBg9ejRo9RbZk6fPq2xY8eqdevWCgwMVFBQkG688UbNnj27RN/z54UpzzkcFRXlPF7Pv3XOYrG4raXYnDlzZLFY9Oijj7pdnp+fL7vdruuvv94ZQB87dkwvvviiunTpovDwcOd49e7dW19++WWZ+ztfWed8WfPiHD16VGlpaWrWrJmqVKmikJAQde3aVcuWLSvR99y5c3rllVfUtm1b2e12Va1aVVFRUW6PWwDwBUIJAKiAP/7xjwoMDHReFVFswYIFysnJ0f3331/quuvWrVNMTIxmzZql2NhY/fnPf1b79u01a9YsxcTEaN26dc6+gwcPliTNmDHD7baKvyiW5/74t956Sx06dNCHH36ozp07a+TIkYqJidEbb7yhmJgY7d2719n3wIEDio2N1dSpU9WiRQuNGDFCd999t+rVq6eZM2fqwIEDF9xfly5dtH//fm3fvt3Z9v333zv3k5mZ6dK/+OeuXbuWus1OnTrpkUcekSS1atVKo0ePdv7TunVrl77r169XfHy8zp49q/vuu089e/bUypUr1bVrV+3YseOC9Z/vxx9/VLt27ZSVlaW7775b/fv31+bNm9WnTx99+umn5drGnDlzdOutt2rDhg2688479eCDDyo3N1c33nhjmfM0XOy+U1JSNHDgQH3//fe644479PDDD6t69ep65plndMstt7hc3fPRRx+pU6dOznF5/PHH1bdvXwUEBOj111939hs9erRatWolSXrkkUecY+7JxJStW7cudXuhoaGSfn3vWrVqpVdffVW1a9fWiBEjNGjQIFWrVs3li9oPP/ygvXv3qnHjxs6w6ny33nqrJOmTTz656DrPN3r0aCUlJWnFihVKSkrS448/rq5du2rbtm2aNWuWS9+7775bDz30kA4ePKj77rtPDzzwgA4dOqT/+Z//0d133+12+3l5eerQoYM2bNigP/7xj7rjjju0fv16JSUl6bvvvlPXrl01f/589ezZU4MHD9bevXvVv39/rVmzxu325s2bp759+yo8PFyPPPKIbrzxRv3nP/9R+/btSxz7e/bsUUxMjF5//XU1aNBAjz/+uJKSkpyhRmnh16JFi9S9e3cFBwcrNTVVN910k5YsWaKEhAQdPny4xOvr2LGjnnzySVmtVqWkpGjw4ME6dOiQBg4cqKefftrtPsp7Do8cOVIJCQmSfv3MPP9zoSx9+/ZVSEiI3nnnHbdXvc2fP195eXkaNGiQ83a8bdu26amnnpKfn5969Oihxx57TN26ddMnn3yim2++WR999FGZ+6yIH3/8UdHR0Ro3bpyuu+46paamqn///tq2bZtuueUWTZkyxaX/kCFD9Mgjj+iXX37RPffcoxEjRujmm2/Wpk2bLmmdAFBuBgBw0SSZOnXqGGOMGTp0qLFareann35yLk9KSjLBwcHm1KlT5qmnnjKSzNSpU53Li4qKTNOmTY0kM2vWLJdtz5kzx0gyTZo0MYWFhc72xo0bG39/f3PkyBGX/mfPnjV2u904HA7zyy+/ONsHDx5sJJk9e/Y423bs2GEqVapkGjRoYLKzs122s3z5cuPn52f69u3rbHvllVeMJDNhwoQSY3Dy5Elz+vTpC47Vm2++aSSZiRMnOtsmTZpkJJlu3boZf39/c+rUKeey1q1bmypVqpj8/PwyX8uePXuMJDN48GC3+/3000+NpBJjf/7+H3rooQvWf/6+JJn09HSXZR999JGRZG699VaX9qlTp5bY9/Hjx01oaKjx9/c33377rUv/v/zlL859uHudnuy7X79+Jd6j0aNHl3hPb7/9diOpRE3GGHPo0CGXn929FxVR2vby8/NNVFSUkWTefvvtEuudf74tWrTISDI9e/Z0u4+MjAwjydx1110e17l06VIjydSrV6/EufPbet555x0jybRp08acOHHC2X7y5EkTHR3t9jUVv8cPPvigy3k/Y8YMI8nY7XbTs2dPc+bMGeeyzz//3EhyOWeN+e/7L8ksXLjQZdmECROMJNOlSxeX9u7duxtJZsyYMS7tq1atMlar1VSvXt3ltRTvw2q1muXLl7usM2rUKCPJjB8/3qW9+L3+bfuZM2dMUlKSsVgsZsOGDc52T87h4uP7008/NRfjgQcecDtexhhz2223GUlm48aNzra8vLwS54Yxvx4H119/vWnatGmJZZJMQkKCS1tZ51Px6x89erRLe0JCgrFYLGb27Nku7bm5uaZVq1amcuXK5ueff3bWabFYTHR0tCkoKCixj8OHD5doA4DLjVACADxwfiixZs0aI8n87W9/M8YYk5WVZfz8/Jy/LLsLJVauXGkkmRtvvNHt9jt27Ggkmc8++8zZ9vzzz5f4cm/Mf79wPfrooy7t7n7ZHTlypJFkFi1a5Ha/ffv2NVar1Rw/ftwY899QYvLkyeUYFfeysrKcX5CL3XnnnaZmzZpm4cKFRpJZunSpMebXX5AtFovp1q3bBV9LeUOJDh06lFh27tw5Y7PZTHR0dLleQ/G+IiMj3f5iX7duXVOjRg2XNnehxMyZM40kc++995bYxokTJ0xoaGipr/Ni9t26dWtjs9lMbm5uif4FBQWmRo0aJjY21tlWHErs2LGj1DEodrlCiffee89IMr17977gNt5++20jyQwaNMjt8mXLlhlJpnv37h7X2bNnTyPJzJs374J9ExMTXY7r8y1fvtxIMp07d3Zpl2SqVq3qPPeKFRQUGJvNZiSZH374ocT2oqKiTFRUlEtb8bH32+CheHsNGjQwkkxWVpYx5tcv0pJM3bp1zblz50qs86c//clIMtOnTy+xD3djvnv3biPJ3HHHHc62w4cPG6vVamJiYkr0N8aYb7/91kgyTzzxhLPNk3PY01Bi1apVRpJJTk52aT9w4ICxWq2mTZs25d7W8OHDjSTz448/urR7I5QoHqff1lnsgw8+MJLMa6+9Zowx5tixY0aSiY+PN0VFReV+DQBwOflmSngAuIbExcXphhtu0FtvvaWnn35ab7zxhoqKisq8deObb76R9OutDe506dJFK1eu1IYNG3TzzTdLku655x4988wzmj59uh5++GFn34u5daP4XufPPvvM5faQYjk5OSosLNTOnTsVHR2t3r1768knn9TDDz+spUuXKikpSR06dFDz5s0veJ92scjISNWvX18rVqxQUVGR83F9iYmJSkhIkM1mU2Zmprp3765PP/1UxphSx8UTMTExJdoqVaqkmjVrKjc396K21bp1a1mt1hLtERER5bqPfMOGDZKkjh07llgWFBSk1q1bl/oow/Lu+/Tp0/ruu+8UFhamCRMmuN1WQECAyz3/gwYN0rx58xQXF6f+/furc+fO6tChg8LDwy/4mi6V4lsSim+98LU1a9bIYrGU68kg33zzjfz8/NSpU6cSyxISEmS1Wp3HwvkaN26satWqubRZrVbVrFlTp06dUv369UusU6dOnVKfLFJ8K8Nvt9exY0f98MMP2rBhgyIjI5213HTTTW4nk+3SpYtmzZqlDRs26J577nFZ5u78ioiIkCSX82vdunUqLCwsdY6EX375RZLczkXhzXO4NPHx8WrcuLEWLlyo3Nxc2e12SdLbb7+twsJCt5+vq1at0ssvv6wvv/xSOTk5OnfunMvyffv2qW7dul6pr1jxuX7s2DG343jo0CFJ/x3H4OBg9erVSwsXLlTr1q11xx136KabblJcXJyqVq3q1doAwFOEEgDgBffff79GjBihDz/8UFOnTlV0dLTatGlTav9jx45J+vWpEu4Ut+fl5TnbwsPD1bVrV3388cfatm2bmjVrppycHH300Udq3bq1WrZsecE6iyemfPHFF8vsd/LkSUm/BgpfffWV0tPT9dFHH2nevHmSfv3S8ec//1kjRoy44D6lX+eHmDJlir755htVqlRJhw4dUteuXVWtWjXFxsY655Eoz3wSF6t4foLfstlsKiws9Nq2yvNkh+L3vWbNmm6Xl9Z+MfvOzc2VMUaHDh1ymaSyLLfffrsWLVqkf/7zn3rrrbc0efJkSVJ0dLTGjh2rbt26lWs73lR87JfnMbDFTzgpHt/fKm4vbQzLW4/dbleVKlUu2PfYsWOqXr26/P39Syyz2WwKCwtTTk5OiWWlPanFZrOVuczdPAhS6cdTrVq1nHWe/++L+Twq5m5Mi+ddOP/8Kv7sWbdundtAtFjxZ8+F9lG8n4s9h8syePBgPfXUU5ozZ44eeughSb+GvpUqVdLAgQNd+r7//vtKTk5W5cqV1a1bNzVo0ECBgYHy8/PTihUr9Nlnn7mdULaiisfx448/1scff1xqv/PH8d1333U+jaZ4fo3KlSsrOTlZ//jHP8r83AGAy4GJLgHAC+6++25VqVJFqamp2rdvnx544IEy+xd/wfj555/dLi+eQPK3X0SKJ7wsvjri7bffVkFBgbP9Qs7/8mZ+vYXP7T/n/4W1WbNmevfdd3XkyBGtX79e48aNU1FRkR555BG9+eab5dpv8ZUPy5cvLxE8dOnSRRs2bNDRo0eVmZmpkJAQtW3btlzbvdoEBwdLkg4ePOh2eWntF6P4PW7Tpk2Z77ExxmW9Hj166JNPPlFubq4yMzP16KOPasuWLerZs6e2bt1a4bouVvEX0X379l2wb5MmTSRJO3fudLu8+GkTjRs3rlA9ubm5OnPmzAX7hoSE6OjRo86//p+voKBAhw8fdh4Ll1Jpx1Px507xseLp59HFKF730UcfLfOYLO+EsZfC3XffLT8/P+fn64YNG7Rp0ybddtttCgsLc+n7zDPPyN/fX+vXr9cHH3ygf/7zn3r22WeVnp7uPB7Lo/hJMe6CJXchUPE4vvzyy2WOY/FTbSSpSpUqSk9P186dO7V3717NmjVLHTt21KxZs1ye2AQAvkIoAQBeEBoaquTkZGVnZyswMFB//OMfy+xffBVFaZfqF/9i/tsv57fffruCg4M1a9YsFRUVafr06bLZbCX+ilea9u3bS5LzEYMXw2azKTo6Wn/5y1+cj+/74IMPyrVuly5dZLFYlJmZqU8++UT169d3Pkaya9euKioq0owZM7Rr1y516tTJ7W0Kv1Xcx5t/Kb3Uit/3lStXllh28uRJffvttxXeR1BQkFq0aKEtW7bo6NGjF71+YGCgunTpon/961968sknde7cOX344YfO5Zdr3IuP1fP3XZoGDRqobt262rlzp/bs2VNiefE2KnJbUPv27WWMKdfTCtq0aaOioiJ9/vnnJZZ9/vnnKiwsvCzBm7tHoBYWFjqPv+Lj8fzj0t2X49I+jy5Gu3bt5Ofn59Fnz8WoyPEZERGhLl26aO3atdqxY4cznHAX+n7//fdq3ry5mjVr5tJeVFTk9vwuTfFtIj/99FOJZevXry/RVpHPcOnX1zho0CAtXbpUDRs21MqVK8v1aGcAuJQIJQDAS8aMGaP3339fS5cuLXFf+G916NBBTZo00cqVK/Xee++5LHvvvff0xRdfqHHjxiXmHqhSpYruuusu7du3Ty+99JK+++473XbbbXI4HOWqcdiwYapUqZIeffRRt39VPnfunMsvu19//bXbS+KL/wJb3nuSHQ6HWrRooVWrVunzzz93uT0jPj5elStX1tixYyWV/4uj3W6XxWJxeYTpla5Pnz4KCQnR22+/re+++85l2ZgxY9z+ZdQTjz32mM6dO6eUlBS328zNzXXOayL9+kXZ3ZdRd+9zjRo1JOmSj3uvXr0UFRWlBQsWOEOw82VnZzv/22KxKDU1VZL0//7f/3O5nWX+/Pn64osv1Lx5c7dzLJTX8OHDJUmPP/6426s3zm9LSUmRJKWlpen06dPO9tOnT2vUqFGSpKFDh3pcS3l98sknJR7lOXHiRP3www/q3LmzIiMjJf16a1i3bt2UlZVVYh6StWvX6p133pHdble/fv08rsXhcGjQoEFav369nnvuObehwQ8//OA2VLoYFT0+i+eOePPNNzV79myFhYWpZ8+eJfpFRUVp165d2r9/v7PNGKP09PSLurKoXbt2klTiMZ6bNm3Syy+/XKJ/TEyMbrrpJs2bN09vvfWW221u2rTJeXvQoUOHtGnTphJ9Tp06pZMnT8pms7m9zQgALifmlAAAL6lbt265JzWzWCyaPn26unXrpv79+6tPnz5q2rSpduzYoQ8++EDVqlXTjBkznJf2nm/w4MF64403lJaW5vy5vJo2baq33npLKSkpatGihW655RY1btxYv/zyi/bu3asvvvhC1113nbZv3y5JmjlzpiZPnqyOHTuqQYMGstvt+uGHH7Rw4UIFBARo5MiR5d53165dtXnzZud/FwsICFCHDh0uej6JoKAgxcXF6YsvvtCgQYPUuHFjWa1W9e7du1zza/hCcHCwXnvtNd19992Kj4/XXXfdpeuvv16rV6/Wd999p4SEBH322Wdu3/eLkZKSoq+//lqvv/66GjRooKSkJNWtW1dHjx7Vnj179Pnnn+vee+/VpEmTJEkjRozQvn371KFDB0VFRcnf319ff/21PvnkE0VGRmrAgAHObXft2lUvvvii7r//ft1xxx2qVq2aQkNDNWzYsArV/Fv+/v7KyMhQ9+7dNXDgQE2ePFnt27fX2bNntW3bNmVmZroEKY899pgWLVqk9957T3Fxceratav27t2rjIwMVa1aVW+99VaFxrV79+56+umnNWbMGDVr1kx9+/ZVRESEDh48qJUrV6p9+/aaNm2aJGngwIGaP3++5s6dqxYtWqhv376yWCz64IMPtGfPHvXv31+DBg2q6BBdUK9evdSvXz/169dPDRs21LfffqsPP/xQ1atX1+uvv+7Sd9KkSerQoYOeeOIJLVu2TDExMfrpp5+UkZEhPz8/TZ069YJh64VMnDhRu3bt0l//+lfNnDlTHTt2VM2aNbV//35t27ZN69at0+zZs1WvXj2P99G5c2f5+fkpLS1Nmzdvdl6J8PTTT5dr/X79+ik4OFgTJkzQL7/8ouHDh7ud/PPRRx9Vamqq2rRpozvuuEOVKlXSqlWrtHXrVufEkuXRp08fNWrUSLNnz1Z2drbi4uK0d+9ezZ8/X3369NHcuXNLrPPOO++oS5cuGjp0qF555RXFxcUpNDRU2dnZ2rhxozZv3qwvv/xSDodD+/btU5s2bXTDDTeoZcuWioiI0PHjx7Vo0SL9/PPPGjFiRIXfVwCosEv+fA8AuAbpvEeCXoi7R4IW2759u/nTn/5katWqZWw2m6lVq5YZNGiQ2b59e5nbbNiwoZFkqlevbvLz8932KetRcxs3bjSDBw82devWNf7+/sZut5sWLVqYBx54wGRmZjr7rVmzxqSmppqWLVsau91uKleubBo0aGCGDBliNm3aVK7XX2zBggVGkrFYLObgwYMuy1544QUjydSsWfOiXsuuXbtMz549TfXq1Y3FYnEZZ3eP0ztfZGSkiYyMLFftF3r8aEJCgvnt/1LdPRK02JIlS8yNN95oqlSpYkJDQ03v3r3Ntm3bTI8ePYwkl0d5erLvYgsXLjQ9evQw1113nalUqZKpWbOmiY2NNU899ZTZtm2bs9+7775rBgwYYBo2bGgCAwNNtWrVTIsWLcyTTz5pcnJySmz3n//8p2natKnx9/d3Pq7UUxd6xOiPP/5oHnroIRMVFWUqVapkqlevbtq1a2eef/75En1PnTplnnnmGdOwYUPj7+9vwsLCTHJystmyZYvH9f3W4sWLTVJSkrHb7cbf39+Eh4ebvn37upw3xhhTWFhoXnvtNRMdHW2qVKliqlSpYtq2bWsmTpxoCgsLS2xXbh4XWaysY/VCx97ChQtN+/btTdWqVU1ISIi5/fbbS330a3Z2tklNTTV169Y1lSpVMjVq1DB9+vQxX331VYm+ZR3fZb2e/Px88+qrr5obb7zRBAcHG39/fxMREWG6dOliXnrpJXP48GFnX0/P4ZkzZ5pWrVqZypUrG0mlnh+lGTp0qHO99evXl9pv6tSpplWrVqZq1aqmRo0apm/fvmbjxo2lPpa0tDHZu3evueuuu5yfsTExMeY///lPma//+PHj5vnnnzdt27Y1gYGBpnLlyiYqKsrcdtttZvLkyebkyZPGGGNyc3PN3/72N9O5c2dTu3Zt4+/vb2rVqmUSEhLMO++8w2NCAVwRLMb8ZqYrAABw2RUWFqp+/fo6d+6cc2JBwBPTpk3Tvffeq6lTp5brUcEAAPgSc0oAAHAZ5eXlucwzIP16L/qYMWO0d+/eCt23DwAAcLVhTgkAAC6jNWvWqH///urevbuioqJ08uRJrVmzRt9++60iIiKUnp7u6xIBAAAuG0IJAAAuoyZNmqhnz55atWqVlixZooKCAoWHh2vEiBF68skny/0kFQAAgGsBc0oAAAAAAACfYE4JAAAAAADgE4QSAAAAAADAJ66pOSX279/v6xIuKCwsTIcPH/Z1GdcMxtN7GEvvYjy9i/H0HsbSuxhP72I8vYex9C7G07sYT++6Gsazdu3apS7jSgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+cU09fQMAAAAAgEvll19+0blz5yRJFovFx9X86uDBg8rPz/fZ/o0x8vPzU+XKlT0aE0IJAAAAAAAu4OzZs5KkqlWrXjGBhCTZbDZZrVaf1lBQUKCzZ8+qSpUqF70ut28AAAAAAHABhYWFHl8NcK2z2WwqKiryaF1CCQAAAAAALoAwomyejg+hBAAAAAAA8IkKzSmxdOlSLViwQHl5eQoPD9eQIUPUrFmzUvtv3bpV06dPV3Z2tux2u3r37q3u3bs7lxcVFWnu3Ln64osvlJeXp9DQUN1000268847fX6PDAAAAAAA8C6PQ4nVq1dr2rRpGjp0qJo2baply5bphRde0EsvvaSwsLAS/XNycjR27Fh17txZw4cP1/bt2/Xmm28qODhY7du3lyR98MEHWrp0qR5++GHVrVtXe/fu1WuvvSabzabk5GTPXyUAAAAAAJdA4f29L+v+rFMWXPQ6RUVFGjVqlBYvXqy8vDxlZGQoPj7+ElR38Ty+fWPRokVKSEhQYmKiwsPDlZKSIrvdrmXLlrntv2zZMtntdqWkpCg8PFyJiYlKSEjQwoULnX127typ6OhoxcTEyOFwKCYmRtHR0fr+++89LRMAAAAAgN+1zMxMzZ07V9OmTdOGDRsUExNTZv+8vDwNHz5cTZs2VdOmTTV8+HAdO3bsktTmUShRUFCg3bt3q1WrVi7tLVu21I4dO9yus2vXLrVs2dKlrVWrVtq9e7cKCgokSU2bNtWWLWUpeXoAACAASURBVFu0b98+SVJ2dra2bNmiNm3aeFImAAAAAAC/e1lZWXI4HIqNjZXD4ZC/v3+Z/YcNG6bNmzdr1qxZmjVrljZv3qwRI0Zckto8un3j+PHjKioqUkhIiEt7aGioNm3a5HadvLw83XDDDS5tISEhKiws1IkTJ2S329WnTx+dOXNGjz32mPz8/FRYWKjbb79dSUlJbre5fPlyLV++XJI0btw4t7eNXGlsNttVUefVgvH0HsbSuxhP72I8vYex9C7G07sYT+9hLL2L8fSuq3U8Dx48KJut5Ffowstch7sa3LUVGzFihN59911JUp06dRQREaF169Zp0qRJmj59uvbt26caNWooOTlZTz/9tHbu3KlPP/1UCxcuVLt27SRJ//jHP9S7d29lZWWpYcOGbvcTEBDg0ftaoYkuvW316tX6/PPPNWLECEVERCgrK0tTp06Vw+FQly5dSvRPTExUYmKi8+fDhw9fznI9EhYWdlXUebVgPL2HsfQuxtO7GE/vYSy9i/H0LsbTexhL72I8vetqHc/8/Pwr4gEMxXcaFLPZbCXazpeenq7atWtrzpw5WrJkiaxWq8aMGaMZM2Zo9OjRiouL05EjR7R582YVFBToq6++UmBgoNq0aePcbtu2bVW1alWtXbtWUVFRbveTn59f6vtau3btUuvzKJQIDg6Wn59fiXtKip+Y4U5oaKjy8vJc2o4dOyar1apq1apJkmbNmqVevXqpQ4cOkqS6devq0KFDev/9992GEgAAAAAAoHTBwcEKCgqS1WqVw+HQqVOnNGXKFKWnp2vAgAGSpHr16jnnmcjJyVGNGjVksVic27BYLAoLC1NOTo7X6/NoTgmbzab69etr48aNLu2bNm1SkyZN3K7TqFGjErd2bNy4UfXr13deapKfny8/P9eS/Pz8ZIzxpEwAAAAAAHCenTt3Kj8/Xx07dvR1KZIq8PSNnj17asWKFcrMzFR2dramTp2qo0ePqlu3bpKkiRMnauLEic7+3bt319GjRzVt2jRlZ2crMzNTK1asUK9evZx9oqOj9cEHH+ibb75RTk6OvvrqKy1atMh5HwsAAAAAALh0HA6Hjhw54nJxgDFGhw8flsPh8Pr+PJ5TIj4+XidOnNC8efOUm5uriIgIpaWl6brrrpNUcn4Hh8OhtLQ0TZ8+3fl40HvvvVft27d39klJSdG7776rN954Q8eOHZPdblfXrl2VnJzsaZkAAAAAAOD/NGrUSAEBAVq5cqXq169fYnl0dLROnTql9evXKzY2VpK0fv16nT59WtHR0V6vp0ITXSYlJZX6ZIz09PQSbc2bN9f48eNL3V6VKlU0ZMgQDRkypCJlAQAAAAAAN4KCgjR06FCNGzdOAQEBiouLU25urjZu3KjBgwerUaNG6ty5s0aNGuX8/j5q1CglJiaW+uSNiriinr4BAAAAAMDVxDplga9LuGhpaWkKCQnRhAkTdODAAYWFhbncoTBx4kQ988wzGjRokKRfp2MYM2bMJanFYq6hWST379/v6xIu6Gp9/M2VivH0HsbSuxhP72I8vYex9C7G07sYT+9hLL2L8fSuq3U8T58+rapVq/q6jBIu9EjQy6Ws8SnrkaAeT3QJAAAAAABQEYQSAAAAAADAJwglAAAAAACATxBKAAAAAAAAn+DpG2UovL93mcuvxllWfYnx9J4LjaXEeAIAAAC48nGlBAAAAAAA8AmulADwu8dVPN7FeHoPV0V5F8emdzGe3sO57l2MJ3B14UoJAAAAAADgE4QSAAAAAADAJ7h9AwAAAAAAD/V5e/tl3d/8QU0vep2ioiKNGjVKixcvVl5enjIyMhQfH38Jqrt4hBIAAAAAAFzDMjMzNXfuXGVkZCgyMlKhoaFl9n/55Zf1ySefaMuWLTpz5oz27dt3yWrj9g0AAAAAAK5hWVlZcjgcio2NlcPhkL+/f5n9z507p1tvvVX33XffJa+NKyUAAAAAALhGjRw5UhkZGZKkOnXqKDw8XGvWrNHkyZM1c+ZM7d+/X9WrV1dycrLS0tIkSU888YQkadGiRZe8PkIJAAAAAACuUc8++6zCw8M1Z84cLVmyRFarVePGjdOMGTM0evRoxcXF6ciRI9q8ebNP6iOUAAAAAADgGhUcHKygoCBZrVY5HA6dOnVKU6ZMUXp6ugYMGCBJqlevnmJiYnxSH3NKAAAAAADwO7Fz507l5+erY8eOvi5FEqEEAAAAAADwEUIJAAAAAAB+Jxo1aqSAgACtXLnS16VIYk4JAAAAAAB+N4KCgjR06FCNGzdOAQEBiouLU25urjZu3KjBgwdLkvbt26fc3FxlZ2dLknMSzHr16ikwMNCr9RBKAAAAAADcKry/d5nLrVMWXKZKrlzzBzX1dQkXLS0tTSEhIZowYYIOHDigsLAwJScnO5e/+OKLzseISlJSUpIkKSMjQ/Hx8V6thVACAAAAAIBrWGpqqlJTU50/+/n5adiwYRo2bJjb/hMmTNCECRMuS23MKQEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8wubrAgAAAAAAuFotfDfvsu6vV//Qi16nqKhIo0aN0uLFi5WXl6eMjAzFx8dfguouHqEEAAAAAADXsMzMTM2dO1cZGRmKjIxUaGjpwcZPP/2kCRMmaPXq1crJyZHD4VDv3r01cuRIValSxeu1EUoAAAAAAHANy8rKksPhUGxs7AX7fv/99yosLNTYsWNVr1497dq1S3/5y1+Um5urv//9716vjTklAAAAAAC4Ro0cOVLp6enat2+f6tSpo7i4OBljNGnSJHXo0EH16tVTdHS0xo4dK0nq3LmzJkyYoE6dOikyMlKJiYkaPny4Fi9efEnq40oJAAAAAACuUc8++6zCw8M1Z84cLVmyRFarVePGjdOMGTM0evRoxcXF6ciRI9q8eXOp2zh58mSZt3xUBKEEAAAAAADXqODgYAUFBclqtcrhcOjUqVOaMmWK0tPTNWDAAElSvXr1FBMT43b97OxsTZo0ScOHD78k9XH7BgAAAAAAvxM7d+5Ufn6+OnbseMG+hw4d0qBBg3TzzTfrgQceuCT1EEoAAAAAAAAXOTk5uvPOO9WkSRO98sorslgsl2Q/hBIAAAAAAPxONGrUSAEBAVq5cmWpfQ4ePKjk5GQ1atRIr7/+umy2SzfzA3NKAAAAAADwOxEUFKShQ4dq3LhxCggIUFxcnHJzc7Vx40YNHjxYP//8s5KTk1WrVi2lp6fr6NGjznVr1Kghq9Xq1XoIJQAAAAAA8FCv/uV/KoXJ2lXmcktUowpWUz5paWkKCQnRhAkTdODAAYWFhSk5OVmS9Nlnn2nPnj3as2eP2rVr57LemjVrFBER4dVaCCUAAAAAALiGpaamKjU11fmzn5+fhg0bpmHDhpXo279/f/Xv3/+y1cacEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnbL4uAAAAAACAq9Urr7zixa19eMEeI0aMuOitFhUVadSoUVq8eLHy8vKUkZGh+Ph4Twr0OkIJAAAAAACuYZmZmZo7d64yMjIUGRmp0NDQUvsWFRUpJSVFW7Zs0ZEjRxQSEqKOHTvqySef1PXXX+/12rh9AwAAAACAa1hWVpYcDodiY2PlcDjk7+9fZv8OHTpo0qRJ+vzzz/W///u/+vHHH3Xfffddktq4UgIAAAAAgGvUyJEjlZGRIUmqU6eOwsPDtWbNGk2ePFkzZ87U/v37Vb16dSUnJystLU1+fn66//77neuHh4dr2LBhuvfee3X27FlVrlzZq/URSgAAAAAAcI169tlnFR4erjlz5mjJkiWyWq0aN26cZsyYodGjRysuLk5HjhzR5s2b3a6fm5urefPmqU2bNl4PJCRCCQAAAAAArlnBwcEKCgqS1WqVw+HQqVOnNGXKFKWnp2vAgAGSpHr16ikmJsZlveeff15Tp07VmTNn1LZtW82YMeOS1MecEgAAAAAA/E7s3LlT+fn56tixY5n9HnroIS1dulSzZ8+W1WrV8OHDZYzxej1cKQEAAAAAAFxUr15d1atXV4MGDdSwYUPFxsbqq6++UlxcnFf3w5USAAAAAAD8TjRq1EgBAQFauXJludcpvkIiPz/f6/VwpQQAAAAAAL8TQUFBGjp0qMaNG6eAgADFxcUpNzdXGzdu1ODBg7V+/Xpt3rxZsbGxCgkJUVZWll588UVFRESoXbt2Xq+HUAIAAAAAAA+NGDGi3H1N1q4yl1uiGlWwmvJJS0tTSEiIJkyYoAMHDigsLEzJycmSpMqVK2vRokV68cUXdebMGTkcDnXq1En//ve/efoGAAAAAAC4OKmpqUpNTXX+7Ofnp2HDhmnYsGEl+v7hD3/Qe++9d9lqq1AosXTpUi1YsEB5eXkKDw/XkCFD1KxZs1L7b926VdOnT1d2drbsdrt69+6t7t27u/TJzc3V22+/rQ0bNujs2bNyOBy6//771bx584qUCgAAAAAArjAehxKrV6/WtGnTNHToUDVt2lTLli3TCy+8oJdeeklhYWEl+ufk5Gjs2LHq3Lmzhg8fru3bt+vNN99UcHCw2rdvL0k6deqUnnnmGTVt2lRpaWkKDg7WwYMHFRwc7PkrBAAAAAAAVySPQ4lFixYpISFBiYmJkqSUlBR9++23WrZsmQYOHFii/7Jly2S325WSkiJJCg8P1/fff6+FCxc6Q4n58+fLbre7XELicDg8LREAAAAAAFzBPAolCgoKtHv3bvXq1culvWXLltqxY4fbdXbt2qWWLVu6tLVq1UqfffaZCgoKZLPZtG7dOrVu3VovvfSStmzZIrvdrq5duyopKUkWi8WTUgEAAAAAwBXKo1Di+PHjKioqUkhIiEt7aGioNm3a5HadvLw83XDDDS5tISEhKiws1IkTJ2S325WTk6Nly5apR48e6tu3r7KysvTWW29Jkm655ZYS21y+fLmWL18uSRo3bpzb20Yq4uAFlnuyP5vN5vU6rxaMp/dcaCylix/P3+tYShyb3sZ4eg/nundxbHoX4+k9nOvexXh6F+f6fx08eFA2W8WeFfHLBZZ7uv2K1uUNAQEBnh0Pl6AWjxUVFalBgwbO2z/q1aunAwcOaOnSpW5DicTEROftI5J0+PDhy1arp/sLCwu77HVeLRhP77rYcWEsS8ex6V2Mp3dxrnsPx6Z3MZ7exbnuXYyn9/yezvX8/HxZrdZLuo+CgoKLXsdms3m0nrfl5+eX+r7Wrl271PX8PNlZcHCw/Pz8dOzYMZf2vLw8hYaGul0nNDRUeXl5Lm3Hjh2T1WpVtWrVJEl2u13h4eEufcLDw6/KAxYAAAAAAJTNo1DCZrOpfv362rhxo0v7pk2b1KRJE7frNGrUqMStHRs3blT9+vWdl5o0adJE+/fvd+mzf//+q/LSHgAAAAAAUDaPQglJ6tmzp1asWKHMzExlZ2dr6tSpOnr0qLp16yZJmjhxoiZOnOjs3717dx09elTTpk1Tdna2MjMztWLFCpfJMnv06KFdu3Zp3rx5+vnnn/Xll1/qww8/VFJSUgVeIgAAAAAAuBJ5PKdEfHy8Tpw4oXnz5ik3N1cRERFKS0vTddddJ6nkvUUOh0NpaWmaPn268/Gg9957r/NxoJLUsGFDPfHEE5o9e7b+85//KCwsTP379yeUAAAAAABckRzfp3lvY99fuEtOw7EXvdmioiKNGjVKixcvVl5enjIyMhQfH+9Bgd5XoYkuk5KSSg0M0tPTS7Q1b95c48ePL3Obbdu2Vdu2bStSFgAAAAAA+D+ZmZmaO3euMjIyFBkZWepckL919uxZ9ezZU9u2bdOSJUvUqlUrr9fm8e0bAAAAAADgypeVlSWHw6HY2Fg5HA75+/uXa73nnntO119//SWtjVACAAAAAIBr1MiRI5Wenq59+/apTp06iouLkzFGkyZNUocOHVSvXj1FR0dr7FjX20KWLl2q1atX669//eslra9Ct28AAAAAAIAr17PPPqvw8HDNmTNHS5YskdVq1bhx4zRjxgyNHj1acXFxOnLkiDZv3uxcZ//+/UpLS9PMmTNVuXLlS1ofoQQAAAAAANeo4OBgBQUFyWq1yuFw6NSpU5oyZYrS09M1YMAASVK9evUUExMjSSosLNTw4cP1wAMPqEWLFvrpp58uaX3cvgEAAAAAwO/Ezp07lZ+fr44dO7pd/sorr6hSpUp68MEHL0s9XCkBAAAAAAAkSatWrdLatWsVGRnp0t6rVy/17t1bEydO9Or+CCUAAAAAAPidaNSokQICArRy5UrVr1+/xPJ//etfOn36tPPngwcPauDAgXr11VcVGxvr9XoIJQAAAAAA+J0ICgrS0KFDNW7cOAUEBCguLk65ubnauHGjBg8erLp167r0DwwMlCRFRUWpdu3aXq+HUAIAAAAAAA/lNBx74U7/x2TtKnO5JapRBaspn7S0NIWEhGjChAk6cOCAwsLClJycfFn2/VuEEgAAAAAAXMNSU1OVmprq/NnPz0/Dhg3TsGHDLrhuRESE9u3bd8lq4+kbAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAADABRhjfF3CFc3T8SGUAAAAAACgHAgm3KvIuBBKAAAAAABwAZUrV9apU6cIJtzIz8+Xv7+/R+vavFwLAAAAAADXHKvVqipVquj06dOSJIvFctHbKPphZ5nL/Rx1LnqbAQEBys/Pv+j1vMUYI6vVqkqVKnm0PqEEAAAAAADlYLVaFRgY6PH6hXMml739rj0uepthYWE6fPiwpyX5HLdvAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCUIJAAAAAADgE4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCVtFVl66dKkWLFigvLw8hYeHa8iQIWrWrFmp/bdu3arp06crOztbdrtdvXv3Vvfu3d32ff/99zV79mwlJSVp6NChFSkTAAAAAABcgTy+UmL16tWaNm2a+vXrp/Hjx6tJkyZ64YUXdPjwYbf9c3JyNHbsWDVp0kTjx49X3759NXXqVK1Zs6ZE3507d2r58uWKjIz0tDwAAAAAAHCF8ziUWLRokRISEpSYmKjw8HClpKTIbrdr2bJlbvsvW7ZMdrtdKSkpCg8PV2JiohISErRw4UKXfqdPn9arr76qhx56SIGBgZ6WBwAAAAAArnAehRIFBQXavXu3WrVq5dLesmVL7dixw+06u3btUsuWLV3aWrVqpd27d6ugoMDZNnnyZMXFxekPf/iDJ6UBAAAAAICrhEdzShw/flxFRUUKCQlxaQ8NDdWmTZvcrpOXl6cbbrjBpS0kJESFhYU6ceKE7Ha7li9frp9//lnDhw8vVx3Lly/X8uXLJUnjxo1TWFiYB6+mdAcvsNyT/dlsNq/XebVgPL3nQmMpXfx4/l7HUuLY9DbG03s4172LY9O7GE/v4Vz3LsbTuzjXvYvxLKlCE1160/79+zV79mw999xzstnKV1ZiYqISExOdP5c2n8Wl4sn+wsLCLnudVwvG07sudlwYy9JxbHoX4+ldnOvew7HpXYynd3Guexfj6T2c6951rY5n7dq1S13mUSgRHBwsPz8/HTt2zKU9Ly9PoaGhbtcJDQ1VXl6eS9uxY8dktVpVrVo1fffddzpx4oQee+wx5/KioiJt27ZNH3/8sWbOnKlKlSp5Ui4AAAAAALgCeRRK2Gw21a9fXxs3btSNN97obN+0aZPi4uLcrtOoUSOtW7fOpW3jxo2qX7++bDabYmNj9Y9//MNl+b///W/VqlVL/fr1K/fVEwAAAAAA4Org8Tf9nj176tVXX1XDhg3VpEkTffzxxzp69Ki6desmSZo4caIkadiwYZKk7t27a+nSpZo2bZoSExO1Y8cOrVixQo888ogkKTAwsMTTNgICAhQUFKS6det6WiYAAAAAALhCeRxKxMfH68SJE5o3b55yc3MVERGhtLQ0XXfddZJK3gvjcDiUlpam6dOnOx8Peu+996p9+/YVewUAAAAAAOCqVKF7IpKSkpSUlOR2WXp6eom25s2ba/z48eXevrttAAAAAACAa4OfrwsAAAAAAAC/T4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCUIJAAAAAADgE4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJQgkAAAAAAOAThBIAAAAAAMAnCCUAAAAAAIBPEEoAAAAAAACfIJQAAAAAAAA+QSgBAAAAAAB8glACAAAAAAD4BKEEAAAAAADwCUIJAAAAAADgE4QSAAAAAADAJwglAAAAAACATxBKAAAAAAAAnyCUAAAAAAAAPkEoAQAAAAAAfIJQAgAAAAAA+AShBAAAAAAA8AlCCQAAAAAA4BOEEgAAAAAAwCcIJQAAAAAAgE8QSgAAAAAAAJ8glAAAAAAAAD5BKAEAAAAAAHyCUAIAAAAAAPgEoQQAAAAAAPAJW0VWXrp0qRYsWKC8vDyFh4dryJAhatasWan9t27dqunTpys7O1t2u129e/dW9+7dncvff/99ffXVV9q/f79sNpsaNWqkgQMHqm7duhUpEwAAAAAAXIE8vlJi9erVmjZtmvr166fx48erSZMmeuGFF3T48GG3/XNycjR27Fg1adJE48ePV9++fTV16lStWbPG2Wfr1q3q3r27nnvuOY0ePVpWq1XPPfecTp486WmZAAAAAADgCuVxKLFo0SIlJCQoMTFR4eHhSklJkd1u17Jly9z2X7Zsmex2u1JSUhQeHq7ExEQlJCRo4cKFzj5PPfWUOnfurLp166pu3boaPny4jh8/ru3bt3taJgAAAAAAuEJ5FEoUFBRo9+7datWqlUt7y5YttWPHDrfr7Nq1Sy1btnRpa9WqlXbv3q2CggK365w5c0bGGAUFBXlSJgAAAAAAuIJ5NKfE8ePHVVRUpJCQEJf20NBQbdq0ye06eXl5/5+9O4+3qq73x/9ikIAUDsrsAf0igwJCGfo9maAZDg/MIW4dHMAEynKMvDbw8GsaV0O8XTUTLczQmxIOOaRYGSQ30Yt+r6UIKmg5fOEGhHqOoiAy/P7w576eGA7nsHGBPp+PBw9da6+19me99xpf+7PXyf77719nXNu2bbNu3bq88cYbadeu3UbzTJ06NXvvvXd69+69yWXOnDkzM2fOTJJcdtllad++fWNWZ7OW1fN6Y96vefPmZW/nzkI9y6e+WiYNr+dHtZaJbbPc1LN87OvlZdssL/UsH/t6ealnednXy0s9N7ZND7rcnm666aYsXLgwEyZMSNOmm+7QMXTo0AwdOrQ0vLnnWWwvjXm/9u3bf+Dt3FmoZ3k1tC5quXm2zfJSz/Kyr5ePbbO81LO87OvlpZ7lY18vrw9rPbt27brZ1xr18402bdqkadOmqa2trTO+pqYmFRUVm5ynoqIiNTU1dcbV1tamWbNm2W233eqMv/HGG/Pwww/ne9/7Xjp16tSYJgIAAAA7uEaFEs2bN0+PHj0yb968OuOfeuqp9OnTZ5Pz9OrVa6OfdsybNy89evRI8+b/02Fj6tSppUBizz33bEzzAAAAgJ1Ao//6xuc///nMnj07s2bNyuLFizN16tS8+uqrOeKII5Ik11xzTa655prS9EceeWReffXV3HjjjVm8eHFmzZqV2bNn59hjjy1N87Of/SyzZ8/ON77xjey6666pqalJTU1NVq9evQ2rCAAAAOyIGv1MiYMPPjhvvPFG7rzzzrz22mvp1q1bxo8fnw4dOiTZ+LcwHTt2zPjx43PTTTeV/jzo6NGjU1VVVZrmvT8nOmHChDrzfvGLX0x1dXVjmwoAAADsgLbpQZdHHXVUjjrqqE2+dvHFF280rm/fvpk0adJml3fbbbdtS3MAAACAnUijf74BAAAAsC2EEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAqqE3eQAAIABJREFUAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIYQSAAAAQCGEEgAAAEAhhBIAAABAIZoX3QAAAACgPK6++uotvn7uued+QC3ZOnpKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIXYpgdd/u53v8uvf/3r1NTUpLKyMqeddlr222+/zU7/9NNP56abbsrixYvTrl27HHfccTnyyCO3aZkAAADAzqnRPSUeeeSR3HjjjfnCF76QSZMmpU+fPvnBD36QFStWbHL65cuXZ+LEienTp08mTZqUE044IVOnTs3cuXMbvUwAAABg59XoUOK+++7LoYcemqFDh6aysjJjxoxJu3bt8sADD2xy+gceeCDt2rXLmDFjUllZmaFDh+bQQw/Nvffe2+hlAgAAADuvRoUSa9euzV//+tcMHDiwzvgBAwZk4cKFm5znueeey4ABA+qMGzhwYP76179m7dq1jVomAAAAsPNq1DMlXn/99axfvz5t27atM76ioiJPPfXUJuepqanJ/vvvX2dc27Zts27durzxxhvZsGFDg5c5c+bMzJw5M0ly2WWXpX379o1Znc2765EtvvyZH82pdxFfad75H8bU1Bl6Ydm/b3H+S4b9rd73WF91fb3T7BC2sZ4b1zIpdz0/LLVMGlPPhtUy+ejU077eQDvBvp7sJPW0r5fXDrCvJ+r5fo6d/78PYF9PXCe9n2NnA3wA+/ros3pucf6mc79a73t8VOq5NddJEyZM2OIy6qvnB13LbXrQZdGGDh2aoUOHloY/qs+e+Kiu9/agluWlnuWlnuWlnuWjluWlnuWlnuWjluWlnv+jvlp0LMMyPkq2tZ7bo5Zdu3bd7GuNCiXatGmTpk2bpra2ts74mpqaVFRUbHKeioqK1NTUTXBqa2vTrFmz7LbbbknS4GUCAAAAO69GPVOiefPm6dGjR+bNm1dn/FNPPZU+ffpscp5evXpt9DOMefPmpUePHmnevHmjlgkAAADsvBr91zc+//nPZ/bs2Zk1a1YWL16cqVOn5tVXX80RRxyRJLnmmmtyzTXXlKY/8sgj8+qrr+bGG2/M4sWLM2vWrMyePTvHHnvsVi8TAAAA+PBo9DMlDj744Lzxxhu5884789prr6Vbt24ZP358OnTokGTj36F07Ngx48ePz0033VT686CjR49OVVXVVi8TAAAA+PDYpgddHnXUUTnqqKM2+drFF1+80bi+fftm0qRJjV4mAAAA8OHR6J9vAAAAAGwLoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQiOZFNwAAAABI7jll3y2+fu+tNR9QSz44ekoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFaF50AwA+Cs4999wtT/D8+A+mIR8S6lk+9dYyUc8GUM/ysq+Xl3qWj30dykdPCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQHnQJAAAAHxHLe04sugl16CkBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFKJ50Q0AAACAzVnec2LRTWA70lMCAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAAChE88bMtGHDhtx+++2ZNWtWVq5cmV69emXs2LHp1q3bFuebO3dubr311ixbtiydOnXKSSedlIMOOihJsnbt2kyfPj1PPPFEli1bllatWqVfv3455ZRT0r59+8Y0EwAAANiBNaqnxD333JP77rsvo0ePzsSJE9OmTZtccsklWbVq1WbnWbRoUa666qoMHjw4l19+eQYPHpwrrrgizz33XJJkzZo1eeGFFzJ8+PBMmjQp3/72t/PKK6/k0ksvzbp16xq3dgAAAMAOq8GhxIYNG3L//ffnhBNOSFVVVbp3756zzz47q1atypw5czY734wZM9KvX78MHz48lZWVGT58ePr165cZM2YkSVq3bp0LL7wwBx98cLp27ZqePXvm9NNPz5IlS7JkyZLGryEAAACwQ2pwKLF8+fLU1NRkwIABpXEtWrTIfvvtl4ULF252vkWLFmXgwIF1xg0cODCLFi3a7DxvvfVWkuTjH/94Q5sJAAAA7OAa/EyJmpqaJElFRUWd8W3bts1rr722xfnatm270TzvLe8frV27Nr/4xS/yqU99Knvssccmp5k5c2ZmzpyZJLnssst2ymdPTJgwYcsTzP1qvcvYGdd7e9nWeqrl/6i3lol6NkC9tXi+DMv4CFHP8tmqOtRTT7X8H+pZXvb18trWeqrl/7Cvl5daNMSm75/fb2erZ72hxEMPPZQpU6aUhsePH79dG5Qk69aty9VXX50333wz3/72tzc73dChQzN06NDS8IoVK7Z728qtvjZ3LMMyPkq2tZ5q+T+2phbqufXs6+WlnuVjXy8v9Swv+3p5uU4qH/t6ealFee2I9ezatetmX6s3lBg0aFB69epVGn7nnXeSvNvz4f0JTG1t7UY9Id6voqIitbW1dcbV1tZu1ONi3bp1+dGPfpSXX345F198cXbbbbf6mggAAADshOp9pkSrVq3SuXPn0r/KyspUVFRk3rx5pWnWrFmTZ599Nn369Nnscnr37l1nniSZN29eevfuXRpeu3Ztrrzyyrz00ku56KKLNgosAAAAgA+PBj9TokmTJhk2bFjuuuuu7LnnnunSpUvuvPPOtGzZMoccckhpugkTJqRnz545+eSTkyTDhg3LRRddlLvvvjsHHnhgHnvssSxYsKD0u/V169bliiuuyF/+8pd85zvfSZMmTUrPm2jdunVatGhRjvUF2C6OHVE3RG3fvv0O2XVuZ/CPtUzUc1vYNstLPctLPctHLctLPeGD0+BQIkmOP/74rFmzJjfccEPefPPN9OzZMxdccEFatWpVmmbZsmV1HlDZp0+fjBs3LtOnT8+tt96azp07Z9y4caWfhrzyyiv5r//6ryTJd7/73Trvd+aZZ+awww5rTFMBAACAHVSjQokmTZqkuro61dXVm51m8uTJG42rqqpKVVXVJqfv2LFjbrvttsY0BwAAANgJ1ftMCQAAAIDtQSgBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFKJRf32DD87ynhOLbsKHinqWl3oCAADbQk8JAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQzYtuAMCO7p5T9i26CR8q6lle6lk+alle6lle6lle6gk7Dj0lAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQjQvugE7s3tO2bfeae69teYDaMmHQ331VMuGUU8AAGBHp6cEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQiOaNmWnDhg25/fbbM2vWrKxcuTK9evXK2LFj061bty3ON3fu3Nx6661ZtmxZOnXqlJNOOikHHXTQJqedMmVKZs6cmZEjR+a4445rTDMBAACAHVijekrcc889ue+++zJ69OhMnDgxbdq0ySWXXJJVq1Ztdp5FixblqquuyuDBg3P55Zdn8ODBueKKK/Lcc89tNO3cuXPz/PPPp127do1pHgAAALATaHAosWHDhtx///054YQTUlVVle7du+fss8/OqlWrMmfOnM3ON2PGjPTr1y/Dhw9PZWVlhg8fnn79+mXGjBl1pvv73/+eqVOn5txzz03z5o3qyAEAAADsBBocSixfvjw1NTUZMGBAaVyLFi2y3377ZeHChZudb9GiRRk4cGCdcQMHDsyiRYtKw+vWrcuPfvSj/NM//VMqKysb2jQAAABgJ9Lgrgg1NTVJkoqKijrj27Ztm9dee22L87Vt23ajed5bXpLcdttt2W233XLkkUduVVtmzpyZmTNnJkkuu+yytG/ffqvm+2DVbPHVHbPNO6ot1zJRz4axbW5PzZs3b1gNn69/ko/yZ6Ke5dPgWib11vOjWstEPcvNvl4+ts3yUs/tSy0a4sN3T1RvKPHQQw9lypQppeHx48dvl4YsWLAgs2fPzr/+679u9TxDhw7N0KFDS8MrVqzYHk3brnbGNu/I1LN81HLbtG/fvkE17LgV03yUPxP1LJ+G1jKpv54f1Vom6llu9vXysW2Wl3puX2pRXjtiPbt27brZ1+oNJQYNGpRevXqVht95550k7/Z8eH8CU1tbu1FPiPerqKhIbW1tnXG1tbWlHhcLFixITU1NTj/99NLr69evzy233JL7778/P/nJT+prKgAAALATqTeUaNWqVVq1alUa3rBhQyoqKjJv3rz07NkzSbJmzZo8++yzGTly5GaX07t378ybN6/On/ecN29eevfunSQ56qijUlVVVWeeSy+9NJ/5zGfq9IYAAAAAPhwa/KDLJk2aZNiwYbnnnnvy6KOP5uWXX861116bli1b5pBDDilNN2HChEybNq00PGzYsMyfPz933313lixZkrvuuisLFizIMccck+Td50t07969zr/mzZunoqJii109AAAAgJ1To/7m5vHHH581a9bkhhtuyJtvvpmePXvmggsuqNOjYtmyZdljjz1Kw3369Mm4ceMyffr03HrrrencuXPGjRtX56chAAAAwEdHo0KJJk2apLq6OtXV1ZudZvLkyRuNq6qq2ugnGluyqWUAAAAAHw4N/vkGAAAAQDkIJQAAAIBCCCUAAACAQjTqmRIAAABwzyn71jvNvbfWfAAtYWelpwQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQCKEEAAAAUAihBAAAAFAIoQQAAABQiOaNmWnDhg25/fbbM2vWrKxcuTK9evXK2LFj061bty3ON3fu3Nx6661ZtmxZOnXqlJNOOikHHXRQnWn++7//O9OmTcv8+fOzdu3a7LnnnjnnnHNSWVnZmKYCAAAAO6hG9ZS45557ct9992X06NGZOHFi2rRpk0suuSSrVq3a7DyLFi3KVVddlcGDB+fyyy/P4MGDc8UVV+S5554rTbN8+fJceOGF6dixY773ve/l3/7t3zJixIi0bNmyMc0EAAAAdmANDiU2bNiQ+++/PyeccEKqqqrSvXv3nH322Vm1alXmzJmz2flmzJiRfv36Zfjw4amsrMzw4cPTr1+/zJgxozTNL3/5ywwcODCnnnpqevTokU6dOuWAAw5I+/btG7d2AAAAwA6rwaHE8uXLU1NTkwEDBpTGtWjRIvvtt18WLly42fkWLVqUgQMH1hk3cODALFq0KEmyfv36PP7446msrMyll16asWPHZvz48XnkkUca2kQAAABgJ9DgZ0rU1NQkSSoqKuqMb9u2bV577bUtzte2bduN5nlvea+//npWr16du+66KyNGjMgpp5yS+fPn5+qrr07Lli1zwAEHbLTMmTNnZubMmUmSyy67bAftUVGzxVd3zDbvqLZcy0Q9G8a2uT01b968YTV8vv5JPsqfiXqWT4NrmdRbz49qLRP1LDf7evnYNstLPbeV687y+fDdE9UbSjz00EOZMmVKaXj8+PHbpSHr169PkgwaNCif//znkyR77713/vKXv+S3v/3tJkOJoUOHZujQoaXhFStWbJe2bU87Y5t3ZOpZPmq5bdq3b9+gGnbcimk+yp+JepZPQ2uZ1F/Pj2otE/UsN/t6+dg2y0s9ty+1KK8dsZ5du3bd7Gv1hhKDBg1Kr169SsPvvPNOknd7Prw/gamtrd2oJ8T7VVRUpLa2ts642traUo+LNm3apFmzZhv9lY0999zTTzgAAADgQ6jeZ0q0atUqnTt3Lv2rrKxMRUVF5s2bV5pmzZo1efbZZ9OnT5/NLqd379515kmSefPmpXfv3kne7RK1zz775L//+7/rTPO3v/0tHTp0aNBKAQAAADu+Bj9TokmTJhk2bFjuuuuu7LnnnunSpUvuvPPOtGzZMoccckhpugkTJqRnz545+eSTkyTDhg3LRRddlLvvvjsHHnhgHnvssSxYsCATJkwozXPcccflyiuvzH777Zf+/ftn/vz5eeSRR/Ktb32rDKsKAADAB+3YEXWfR9iYn8Pw4dXgUCJJjj/++KxZsyY33HBD3nzzzfTs2TMXXHBBWrVqVZpm2bJl2WOPPUrDffr0ybhx4zJ9+vTceuut6dy5c8aNG1fnpyEHHXRQvva1r+Wuu+7K1KlT06VLl5x11lmbfJ4EAAAAsHNrVCjRpEmTVFdXp7q6erPTTJ48eaNxVVVVqaqq2uKyDzvssBx22GGNaRYAAACwE6n3mRIAAAAA24NQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAAChE86Ib8GF37IiKOsPt27fPihUrCmrNzu0fa5mo57awbQIAAEXTUwIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAoRPOiGwAAAADU79gRFRuNa9++fVasWFFAa8pDTwkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBDNGzPThg0bcvvtt2fWrFlZuXJlevXqlbFjx6Zbt25bnG/u3Lm59dZbs2zZsnTq1CknnXRSDjrooNLrq1evzrRp0/LYY4/ljTfeSPv27XPEEUfk85//fGOaCQAAAOzAGtVT4p577sl9992X0aNHZ+LEiWnTpk0uueSSrFq1arPzLFq0KFdddVUGDx6cyy+/PIMHD84VV1yR5557rjTNTTfdlD/96U85++yzc+WVV2b48OGZNm1a/vjHPzammQAAAMAOrMGhxIYNG3L//ffnhBNOSFVVVbp3756zzz47q1atypw5czY734wZM9KvX78MHz48lZWVGT58ePr165cZM2aUplm0aFGGDBmS/v37p2PHjjn00EPTq1evOsEFAAAA8OHQ4FBi+fLlqampyYABA0rjWrRokf322y8LFy7c7HyLFi3KwIED64wbOHBgFi1aVBru06dPHn/88axYsSJJsnDhwrz44ov5xCc+0dBmAgAAADu4Bj9ToqamJklSUVFRZ3zbtm3z2muvbXG+tm3bbjTPe8tLkjFjxmTKlCk588wz06xZsyTJ6NGj86lPfWqTy5w5c2ZmzpyZJLnsssvSvn37hq7OB6558+Y7RTt3FupZPmpZXg2u5/P1T/JR/nzUs3wata/XU8+Pai0T9Sw3+3r52DbLSz3Ly3Vnee3s9aw3lHjooYcyZcqU0vD48eO3W2N+85vfZOHChfn2t7+dDh065JlnnskvfvGLdOzYcZO9JYYOHZqhQ4eWht/rYbEja9++/U7Rzp2FepaPWpZXQ+vZcSum+Sh/PupZPo3Z1+ur50e1lol6lpt9vXxsm+WlnuXlurO8doZ6du3adbOv1RtKDBo0KL169SoNv/POO0ne7fnw/jSmtrZ2o54Q71dRUZHa2to642pra0s9LtasWZNp06blvPPOy6BBg5Ike+21V1588cXce++9fsIBAAAAHzL1hhKtWrVKq1atSsMbNmxIRUVF5s2bl549eyZ5N1B49tlnM3LkyM0up3fv3pk3b16OO+640rh58+ald+/eSZK1a9dm3bp1adq07mMumjZtmvXr1zdsrQB2Mst7Tiy6CR8q6lle6lle6lk+alle6lle6glbp8EPumzSpEmGDRuWe+65J48++mhefvnlXHvttWnZsmUOOeSQ0nQTJkzItGnTSsPDhg3L/Pnzc/fdd2fJkiW56667smDBghxzzDFJktatW6dv376ZNm1aFixYkOXLl2f27Nn5j//4jxx00EFlWFUAAABgR9LgB10myfHHH581a9bkhhtuyJtvvpmePXvmggsuqNOjYtmyZdljjz1Kw3369Mm4ceMyffr03HrrrencuXPGjRtX56ch48aNy7Rp03L11Vdn5cqV6dChQ0aMGJGjjz56G1YRAAAA2BE1KpRo0qRJqqurU11dvdlpJk+evNG4qqqqVFVVbXaeioqKnHnmmY1pEgAAALCTafDPNwAAAADKQSgBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFKLJhg0bNhTdCAAAAOCjR0+JD9h3v/vdopvwoaKe5aOW5aWe5aWe5aOW5aWe5aWe5aOW5aWe5aWe5bWz11MoAQAAABRCKAEAAAAUotnFF198cdGN+Kjp0aNH0U34UFHP8lHL8lLP8lLP8lHL8lLP8lLP8lHL8lLP8lLP8tqZ6+lBlwAAAEAh/HwDAAAAKIRQAgAAACiEUKIAZ511Vn79618X3Qw+BNavX58pU6ZkzJgxqa6uzoIFC7br+1188cW54YYbtut7fFip3cYmT56cyy67rOhmsA2WL1+e6urq/OUvfym6KdvNggULUl1dnddff32Tw1tr9uzZGTVq1PZo4nZVXV2duXPnbtW0jnPsDG677bb88z//c9HN+FD5KJwLdlRbc2559tlnc/755+ekk07Kjvo4SaHE+0yePDnV1dWlf2PHjs1ll12WJUuWFNquxl4A7UjeX9uTTjopZ599dv793/89q1evrnfeza3/znzxs7m2N/Si9c9//nMefPDBfOc738mUKVPSp0+fcjZzI+eff35OPvnk7foeRSrHTfKHYX/dGpMmTcqECRM2+drixYtTXV2dJ5988gNu1Y6tpqYmU6dOzTnnnJOTTz45X/va1/KDH/wgf/rTn4pu2lbZGUOkTbX58ccfz8iRIzN9+vTt9r474pcPv//97zNq1KisXbu2NG7t2rUZOXLkRjdoS5fZCLt7AAAgAElEQVQuTXV1dZ566qlMmTIln/rUpz7o5pbVjn5c3pp96x/DofXr1+faa6/NmDFj8txzz23vJn5g3n+9eOKJJ+aMM87I9ddfn5UrVxbdtJKt3Z7eu1EfMWJEVqxYUee1lStX5pRTTtkhb+Qbc65q3759pkyZkr333vuDa+gOZEe9h3zPjTfemL322is//vGPc/7552/z8rbHMbV52Zb0IbH//vvnnHPOSZK8+uqrufnmm/PDH/4wV155ZcEt2/m9V9u1a9fm2WefzU9+8pO8/fbb+epXv1p003ZaS5cuTbt27bY5jFi7dm2aN6//cLDrrrtu0/vw4XH44Yfnhz/8YZYvX56OHTvWee0Pf/hDOnTokP3337+g1u14li9fngsvvDCtWrXKSSedlL333jvr16/P/Pnzc/311+e6664ruokfCX/84x/zk5/8JCNHjsywYcOKbs4Hql+/fnn77bfz/PPPZ999902SPPfcc2ndunX+9re/5fXXX0+bNm2SJPPnz88uu+ySPn36pEWLFkU2e4eztefL7WnNmjW56qqr8sILL2TChAmprKwstD3l9t714rp167J48eJcd911efPNNzNu3Liim9You+++ex588MF86UtfKo2bM2dO2rZtu1FYUbTGnKve2ycqKioKaPGOo6H3kB/ksWTp0qU56qij0r59+w/k/RpDKPEPdtlll9JOVVFRkWOOOSaTJk3KmjVrSifmW265JY899lhWrFiRioqKfPrTn051dXWdE/ef/vSn3HHHHXnppZfysY99LL1798555523yZP7H//4x9xwww0555xzMmjQoDqvLV++PN///veTJF/5yleSJIceemjOOuusvPPOO7nlllvy8MMP56233sree++dUaNGlS42djTvr+0hhxyS+fPn5//+3/+bPn365De/+U2WLFmSFi1apG/fvjnttNOy++67b3b9k+Tpp5/O008/nd/97ndJkmuuuSYdO3bM008/nZtvvjkvvfRSWrdunc985jMZOXJkace/+OKLU1lZmdatW2fWrFlp0qRJhgwZkpEjR6Zp0x2n89DkyZPzxhtvZMCAAbnnnnuyZs2aHHjggRk7dmw+9rGPZfLkyfmP//iPJO9+g9KhQ4dMnjy53u1iwYIF+f73v5/vfve7uf322/Piiy/m/PPPT79+/fKzn/0sjz76aFq2bJlhw4Zl4cKF2W233XLWWWclebd23bp1y9ixYzNt2rQ8+eSTmTRpUp12/5//83/So0ePjBkzJkny4IMP5te//nWWL1+e9u3b54gjjsiwYcNKta6urs7pp5+eefPm5c9//nPatm2b6urqDBky5IMq9WbV9xn8oy3tr0myYcOGTJs2bbPb3dq1azN9+vTMmTMnK1euTLdu3TJixIh84hOf+IDWeOsdcMABadu2bWbPnp3q6urS+LVr1+ahhx7KUUcdlWeffXaL++I/2rBhQ+677778/ve/z4oVK9KmTZsMGTKk1DunvmPvbbfdlkcffTTDhw/P9OnTU1tbm/79++frX/966WZr/fr1ufPOOzNr1qzU1tamS5cuOfHEE3PggQdu13q91zPqsssuS8uWLUvjKysrM3jw4NLwihUrMnXq1Dz11FNJkgEDBmT06NHZY489tnodn3/++UyfPj0vvPBC1q5dm+7du2fUqFHp3bt36X0aut/ddtttdY43SXLRRRelQ4cOSZK///3vmTZtWhYuXJgOHTpk9OjRGTBgQGn+xYsX5xe/+EWeeeaZtGjRIv37989pp532gV7EzpgxI7fccku+/vWvl9Zz9uzZueGGG/KLX/yiNN17x8if/exnpZo2xMUXX5y///3vufnmm3PzzTcnebd+73nqqady4403Zvny5enZs2fOOOOMOsHef/3Xf+X222/P4sWLU1FRkUMOOSRf+tKXtvnitWvXrmnXrl3mz59f53zQv3///P3vf8+CBQvy6U9/ujS+d+/eadGiRaqrq3PeeeelqqoqSXLHHXfkD3/4Q2pqavLxj388AwcOzNlnn116n/qOc2eddVY++9nPZtmyZXn00Ufz8Y9/PKNGjcrAgQNz/fXX5/HHH0+7du0yduzYDBw4MMm7++1Pf/rTzJ8/PzU1Ndljjz3yuc99Lscee2xpuS+//HJuvPHG/OUvf8n69evTuXPnfPnLX07Hjh03e1zesGFDfv3rX2fmzJl59dVX07lz5xx//PGl7WP58uU5++yzc+6552bWrFlZtGhRRo0alaOPPnqbPott8dZbb+Xyyy9PTU1N/uVf/qXOTcbW7Ncvv/xybrrppjz77LNp0aJFBg0alNGjR6d169ZZsmRJvvnNb2bKlCmpqKjI22+/ndGjR6dfv3654IILkiSzZs3K3XffnR//+MfbbR3ff724xx575OCDD87s2bNLr2/Ncfy9m8Innngia9asSZcuXfLlL385/fv33+j9VqxYkUsuuaS0P27YsGGz5+L6zvObcthhh2X27Nn54he/mCZNmiR5N7w/7LDDcscdd9SZthznuW2xNeeq6urqjBkzJvPnz8+TTz6ZI444IkcffXTOPvvsTJw4Mfvss0/pODp+/PhMnz49ixcvzj777JNvfOMbWbZsWaZOnZqlS5emX79+Oeuss7Lbbrsl2fx+/N7ntiNf42/pHrKmpmazx5L6rpPvu+++zJ49O8uWLUvr1q3zyU9+MqNGjcrHP/7xTbZj5cqVmTRpUlq2bJlTTz211BPuuuuuy3XXXZczzzwzQ4YM2S7H1G0hlNiCVatW5ZFHHkn37t3rhAkf+9jHcsYZZ2T33XfP4sWLc/3116d58+Y58cQTkyRPPPFELr/88pxwwgk588wzs27dujz55JPZ1F9fvf/++3P77bfnO9/5Tvr27bvR6+3bt88///M/59/+7d9yxRVXZNdddy215eabb85//ud/li5o7rvvvlx66aW5+uqr065du+1UlfJp0aJF1q1bl7Vr1+ZLX/pS9txzz7zxxhu55ZZb8qMf/Sjf//73t7j+f/vb39K1a9fSDUubNm3y6quvZuLEiRk8eHDOPPPMLFu2LD/5yU/StGnTnHrqqaX3fuihhzJs2LD8y7/8S1588cVcffXV6dGjRw455JBCarE5zzzzTCoqKnLhhRfmlVdeyZVXXpkuXbrkC1/4QkaPHp0OHTrkwQcfzMSJE0sHka3dLm655Zaceuqp6dy5c1q1apV///d/z9NPP51vfetbadeuXX71q1/lmWeeyUEHHbTJtg0ZMiR33313lixZkj333DNJsmzZsixatCinnXZakmTmzJm57bbbMmbMmPTo0SMvv/xyfvrTn6Z58+Z1LuruuOOOnHzyyTn55JPzhz/8Idddd1369u27QyS6W/oM/tGWttek/u3u2muvzbJly3Luuedmjz32yJ///OdMmjQpEydO3OG6RDZr1iyHHnpo6ULrve3v8ccfz+uvv55DDz005513Xr374vv98pe/zAMPPJBTTz01ffv2zeuvv54XXnih9Hp9x97k3ZuIRx55JOeff37efvvtXHXVVZk+fXpOP/30JO8ec++999589atfTY8ePfLQQw/lhz/8YSZNmrTdarxy5co88cQTGTFiRJ2LvPe8d2Gxfv36XH755WnRokUuuuiiJMnPf/7z/Ou//msmTpxYupitbx1Xr16dIUOG5LTTTkuTJk3y29/+NhMnTszVV19duvBLGrbfHXfccVmyZElWrlxZ+iZo1113zauvvpokmT59ekaOHJmvfOUr+dWvfpWrrroq1157bVq2bJnXXnstF110UT772c9m1KhRWbduXX75y1/m8ssvzyWXXPKBhMHTp0/Pfffdl/PPPz8HHHDAdn2v888/P9/61rfy2c9+NkceeWSd19auXZu77747Z5xxRnbZZZdMnjw5119/femG74knnsiPf/zjnHbaadlvv/2yYsWKXH/99XnnnXc2u980RL9+/bJgwYJ88YtfTPJu+HDIIYdk+fLldUKJp59+OkccccRG88+dOzf33ntvvvGNb6R79+6pra3d6KcDW3N+nTFjRk488cQMHz48v//97zN58uT0798/Bx98cE488cTcdddd+fGPf5xrr702LVq0yPr167P77rvnm9/8Ztq0aZPnn38+U6ZMyW677ZbDDz88SfKjH/0oe+21V37wgx+kWbNmefnll9OiRYstHpenT5+euXPnZuzYsenatWsWLVqUn/70p9l1113rbCe//OUvM2rUqJxxxhlp1qzZNn8OjVVbW5vvf//7adKkSSZMmLDJm9At7derV6/OpZdemn322ScTJ07MypUr89Of/jTXXnttzj///Oy5556pqKjIggUL8pnPfCYLFy5Mq1atsnDhwqxbty7NmjXLggUL0q9fvw9snZctW5YnnniiTt3rO46vXr06F198cdq2bVu6pnnppZc2ufzFixfn0ksvTVVVVU499dQ0adIkV1999WbPxd27d9/ieX5TPvnJT2bWrFmZP39+9t9//7zwwgtZtmxZPv3pT28USpTjPNdYW3uuSt7dzk466aSMGjWqdG7alNtuuy2nnXZaWrdunauvvjpXXXVVdtlll5x++ulp2rRprrjiitx+++2lL7I2tx8n2amu8Td3D/mPx5KtuU5u0qRJTjvttHTs2DErVqzIz3/+8/z85z8vnYvf79VXX82ll16aysrKnHPOOWnatGmmTJmSc845JyeddFIOPvjgtG7dersdU7fFjvO18A7iiSeeyKhRozJq1Kh8+ctfztNPP51zzz23zjRf/OIXs++++6Zjx4454IAD8oUvfCEPP/xw6fVf/epXqaqqyoknnpjKysrstddeOe644zb6ZnX69Om566678r3vfW+TgUSSNG3atNRlvk2bNqmoqEjr1q2zevXqPPDAAznllFNywAEHpLKyMqeffnoqKipKPQd2ZM8//3wefvjh9O/fP4cffngOOOCAdOrUKT179sxXvvKVPPPMM3nllVc2u/6tW7dO8+bN87GPfSwVFRWpqKhI06ZN87vf/S7t2rXLV77ylVRWVuZTn/pUTjnllPz2t7/N22+/XXr/ysrKjBgxIl27ds3BBx+cfv36Zf78+UWVY7Nat26d008/PZWVlRk4cGCqqqpK7WzdunVatmyZpk2bpqKiIm3atGnQdvGlL30pAwcOTKdOndKiRYs8+OCDOeWUUzJgwIB069YtX//617d4s1BZWZn/9b/+Vx566KHSuDlz5qRLly7p2bNnknf3hZEjR6aqqiodO3bMoEGDcsIJJ2zUliFDhmTIkCHp3LlzRowYkWbNmuXpp58uVxm3yZY+g3+0ue31PVva7pYuXZqHH3443/zmN9O3b9906tQpRx99dD75yU9m5syZ239FG+Hwww/PihUrSt/qJ+9++zNw4MDMnDlzq/bF96xevTozZszIySefnMMPPzydO3dO7969c9RRR5Wmqe/Ym7x7Y3/WWWdlr732Su/evTN06NA67bv33ntz7LHH5pBDDknXrl0zYsSI7Lffftv19/9Lly7Nhg0b6u1iPX/+/Lz00ks599xzs88++2SfffbJueeemxdeeKHOOtS3jv3798+QIUNSWVmZPffcM2PGjMkuu+ySP//5z3XeryH7XcuWLdOiRYvSN0EVFRV1vrk/5phjMmjQoHTp0iUnn3xyVq5cmRdffDFJ8sADD2SvvfbKyJEjS+fEs88+O88//3z++te/NrScDTZv3rzceeedOe+887Z7IJG8G9Y0bdo0LVu2LNXqPevWrcvYsWPTs2fP7LXXXjn22GOzYMGC0pcWd911V4499th89rOfTefOndO/f/+ccsop+f3vf7/JLzYaqn///lm0aFHeeeedrFmzJosWLUq/fv3St2/f0oOSlyxZktdee22z3yhXVFRkwIABad++ffbZZ5+Neg1szfl14MCBOeqoo9KlS5dUV1fnnXfeSadOnXLooYemc+fO+ad/+qe8/vrr+X//7/8lSZo3b54RI0akZ8+e6dixYw4++OAcccQRdfb/FStWZMCAAdlzzz3TuXPnHHTQQendu/cWr6Puu+++fP3rX88nPvGJdOzYMYccckg+97nPbXSOOvroo0vnsfd6LRXhpptuyltvvZWLLrpos9+Kb2m/njNnTlavXp1zzjkn3bt3T9++fXP66afnsccey9KlS5Okzrbw9NNPp6qqKrvttlvpuQfPPPPMZq9Zy+W9a/FTTjkl55xzThYvXpzjjz++9Hp9x/E5c+akpqYm3/rWt7Lffvulc+fO+d//+39vtE0/99xzueiii3LEEUfky1/+cpo0aVLvubi+8/ymNGvWLEOGDMmDDz6Y5N3z5Kc//elN9rgsx3musbb2XJUkBx98cD73uc+lU6dOG/2E8/3e+2z22muvHHHEEVm4cGFGjhyZXr16ZZ999smhhx5a5yHtm9uPk+zw1/hbcw/5j8eSrblOPuaYY9K/f/907Ngxffv2zciRI/Of//mfWb9+fZ1lL126NBdeeGH69OmTb3zjG2nevHnpHiF593q2oqIiLVq02C7H1G2lp8Q/2G+//fK1r30tybuJ4QMPPJBLL700l156aenbo7lz52bGjBlZunRpVq9enfXr19fZMF544YXSTww25ze/+U1WrVqViRMnpkuXLg1u57Jly7Ju3bo6zxJo2rRpevXqlcWLFzd4eR+E93bW9evXZ+3atTnwwAMzZsyY/PWvf80dd9yRF198MStXrixdeK1YsaLBJ/8lS5akV69edW6m991336xduzZLly7NXnvtlSSl/76nXbt2qa2t3cY1LL/Kyso667L77rvn+eef3+z0Ddku9tlnn9L/L126NOvWrSuFCcm7NyHdunXbYvsGDx6c3/3ud6UEf86cOaXufa+//npeeeWVTJkyJddff31pnvXr1290cd29e/fS/zdr1ixt2rTZYR5I1tDPYEu2tN298MIL2bBhQ775zW/WmWbt2rWbvDnYEXTp0iV9+/bNgw8+mIEDB+bVV1/Nk08+mXHjxmXOnDlbtS++Z/HixXnnnXe2+ByK+o69ybu9Vd5/cmzXrl1pW3rrrbfy2muvbfQMln333XejG/Zy2tqbycWL/z/2zjysqmp9/B+mwyTDQWYQkAAnFERucv1apJSaXrWMtFQcL1pidUtv2i0TyuFqZVbiFcccSM2BB8RSLjij1jVnQAWVZBBQ4QAOeBjO7w+es38cDigoCOj6PA/PA2uvvfbai7XWu/a73vW+2VhZWWks8Ozs7JDL5WRnZ0vHIR70jlC9m7p161ZSUlJQKBRUVVWhVCq1zi435bir+f9UW2Sp+/aVK1dIS0ur04lvXl6exrzTHHTo0IG7d++ybds2OnXqVK/J65PAwMAAR0dH6W+5XE5FRQV37tyhXbt2XLlyhYyMDGJjY6U8KpVKMv99XCtIb29vysvLuXTpEiqVCnNzc+zt7bG0tCQvLw+FQkFKSgqGhoZ1/l8CAgL45ZdfmD59Oj4+Pvj6+uLv74+BgYGUpyHytWYeIyMjDA0NNfqjehFd876EhAT27dvHjRs3UCqVVFZWSseHoHrhHhUVxcGDB+nevTu9e/eWrPjqQj3nLFiwQCO9drmgKS9bEj8/P06cOMGvv/7KiBEj6szzoHGdk5ODq6srxsbGUp5OnTqho6NDdnY29vb2dO3ald27dwPVljSvvvoqSqWSlJQUzM3NuXXrVrNbSqjX4kqlksTERPLz8yUfMA2ZxzMzM3F1dX3gcYbCwkK+/PJLgoODGTZsmJTeXLK4f//+fPzxxygUCo4cOcInn3xSZ77HlXOPQ2MUn+7u7g3KV3OsW1hYAGiN9Zrj/EHjuLWv8R/0Damm5lzS0HXy+fPniYmJIScnh7t370rfUQqFAisrK6C6f86ZM4fevXtLRyoeRlPPqY+LUErUwtDQEHt7e+lvd3d3xo8fT2JiIm+99RaXLl1i6dKlBAcHM378eExNTTlx4oTGedSG0KlTJ86cOUNycrJkRvm0ox6senp6yOVy9PX1JVPC7t27M336dCwsLCgtLeXzzz/X8BDeFNQ0L6ttfqmjo9Mku1ANxdjYmLt372ql37lzR0PQ1GUm2lT1rEtD31j+7//+j02bNnHp0iX09fXJycmRlBJqIRoaGvpQR5y1z0rr6OhoCeGWoin/Bw/qdyqVCh0dHRYuXKjVHq3Z0Vz//v2Jiori9u3bHDhwgHbt2uHv78+RI0fqvedBpp710dC5tzX2JQcHB2nRX99xqIdRs80e9o6RkZEUFxczfvx4bGxsMDAw4IsvvtCaU5uyrWr2bXVda/btnj171nn8QL1IbU7kcjmzZs0iIiKCL7/8ks8++0za5amrL1ZWVjZbXWpbn6mfr273qqoqgoODpWMUNWmK8+K2trbY2NhIO5NdunQBqhUD7u7upKSkkJKSQufOnev0YWFtbc3SpUs5f/48Z8+eZcOGDWzfvp358+dL5t4Nka91zat1PU9939GjR1m/fr3kG8XExIQ9e/bwv//9T8o7cuRIXnjhBU6dOsWZM2fYtm0boaGhkilyfWXPmjVL68hS7fo1hbxsCvr27UtAQADLli2T+kptHndcq/1L5eXlceXKFbp164ZSqeTIkSOYm5tjZ2fX7NYiNdfikyZNIiIigu3bt2v4L3pczMzMsLGxITk5mf79+0tzQnPJYkdHRzp27Mh3332HpaUlXl5eFBQUaORpaTnXGFlV1/GOuqhLNtSuf835obHjuHbZtZ+pvvYk1vgP+oZU17/mXNKQdfKNGzdYuHAhQUFBjBo1inbt2nH16lW+++47DZmur6+Pj48Pp06d4saNG1qK1do0x5z6uIjjGw1AV1cXpVIJwMWLF7GysiI4OBgPDw8cHBy4ceOGRv6OHTs+1EzI3d2dTz/9lPj4eK3zZLVRD96aE46dnR36+vpcvHhRSquqqiI9Pb3VemFWD1YbGxvpnXJzcyktLWX06NF07doVJycnLW1mXe+vTq+d5uTkRHp6ukb6hQsX0NfXx87Orjle65FwdHSUtPE1uXr1qsYuWmN51H5hb2+Pnp6eRliq+/fvS6az9SGXy/H29ubw4cMcOXIELy8vqZ0tLS2Ry+Xk5+djb2+v9fO0Ul9/fRhubm6oVCoUCoVWW6k14a2RgIAADAwMOHToEPv37+fFF19EX1+/0WPRyckJAwODek1QGzL3PgwTExPkcrnG+FDXqznnzXbt2uHj48PevXvrDIN8584doNoqp7CwUGOhmp+fT1FRUaPqd+HCBQYNGoSfnx8dOnSQ/Do8LnXNuQ2hY8eOZGdnY21trdW3a+7YNidWVlaEh4dz//59vvzyS0pLS4HqD/379+9rKInVx04eh0dtK3d3d3JycuqcM5vKl4Har0Rt3wBqE+fU1NQH7oTLZDL8/PyYMGECCxcuJCsrS2tMNTUXLlzAw8ODQYMG4e7ujr29Pfn5+Vr5HBwcGDx4MJ988gn9+/dn3759QN3zsrOzMwYGBty4cUOrrR+2oG9J+vbtywcffMCOHTs0HKg2BCcnJ65du8a9e/ektIsXL2qY7Kv9SuzcuRM7OzssLCzo2rUrFy9e5OzZs0/Un4Sa4OBgYmNjKSwsbNA87ubmxp9//vlA6wEDAwNmzZpFu3btmDdvnjQPN0QWP6qc79+/PykpKfTr16/O600h5x6Hhsqq5qa+cdxW1vg1qfkNWZuGrJMvX75MRUUFEyZMwMvLC0dHx3rl+bRp0+jcuTMREREPjerSHHPq4yKUErUoLy9HoVCgUCjIzs5m7dq1lJWVSTG6HRwcKCws5PDhw+Tn55OQkKB11uv111/n2LFjkrfZrKws4uPjtc5Re3h48NlnnxEfH8+OHTvqrZONjQ06OjqcPHmSkpISysrKMDIyYsCAAURHR3Py5EnJGY5CodA4g93asba2xsDAgD179pCfn8/JkyfZunWrRp663l+dnpGRQUFBASUlJVRVVTFw4ECKiopYvXo12dnZnDx5kujoaAYNGtRqdjoABgwYQH5+PmvXriUzM5Pc3Fzi4+NJTk7WMCNsLI/aL4yMjOjXrx/R0dGcO3eO7OxsVqxYQVVV1UN3tV944QWOHj1KcnKyRhQBqNayxsbGEh8fT25uLteuXePgwYPExMQ88ju2durrrw/D0dGRvn37snz5co4fP05+fj6XL18mLi6O3377rZlr/ejIZDL69u3Ltm3byM/PlzTojR2LxsbGvPrqq2zevJn9+/eTl5dHRkYGCQkJQMPm3oYwbNgwdu3axZEjR8jNzWXr1q2kpaUxdOjQx2uIhzB58mRUKhWzZ8/m2LFj5ObmkpOTQ0JCghQzvHv37lIc8cuXL3P58mW+//57Onbs2CizYQcHBw4fPkx2djYZGRl89913TRJ2zMbGhqysLHJzcykpKWmwNdvAgQO5e/cuS5cuJT09nfz8fM6ePUtUVJTGx1FzI5fLmTt3LhUVFXzxxReUlJTg6emJoaEhP/30E3l5eRw/frxJ/DLZ2Nhw4cIFCgsLG2VW/cYbb5CcnMzWrVu5du0aOTk5HD9+XIri0RR069aN9PR00tPTNT4wu3btytGjRyVv/nVx4MABkpKSuHbtGgUFBRw4cAA9Pb1HOobaGBwcHLh69SqnTp3i+vXrbN++XcP/iVKpZPXq1aSkpFBQUEB6errGR2pd87KxsTFDhw5l48aN7Nu3j7y8PDIzM0lISGi1fnzU/PWvf+XDDz8kJiaGzZs3N/i+F154AUNDQ5YtW8a1a9dITU1l5cqVPP/88xqbBV27duXw4cNS/7C1tcXc3Jzff/+92f1J1EW3bt1wdnZm586dwMPn8b59+2JhYcFXX31FWloa+fn5nDhxQmvDUCaTMWvWLExMTCTFRENk8aPK+cDAQFavXs2QIUPqvN5Ucu5xaIisai4eNo5b+xr/Yd+QdfGwdbKDgwMqlb9ooLEAACAASURBVIrdu3dTUFDAkSNHpONVtdHV1SUsLAwvLy/Cw8MfqJhojjn1cRHHN2px7tw5yXutsbExjo6OfPjhh9LE7O/vz7Bhw/jxxx9RKpX4+PgwatQoVq9eLZXh5+fHP//5T7Zt20ZcXBzGxsZ4eXlpeeGG/6+YmDdvHlC9IKmNlZUVb775Jlu2bCEqKooXX3yRsLAwxowZAyDFb+7YsSOffvppm4i8ocbc3JywsDA2b97M3r17cXFxYdy4cRpnPOt7/6FDhxIZGclHH32EUqmUQoJ+8sknbNq0iY8//hhTU1P+7//+j7fffrsF31IbOzs7IiIi2Lp1K/Pnz0epVOLk5MSHH35Iz549H6vsR+0X48aNY9WqVSxevBgjIyOGDBlCcXGxxlnhuujduzerV6/m7t279OnTR+NaUFAQhoaG7Nq1i82bNyOTyXB2dm7RcGrNTX39tSFMmzaNnTt3smnTJm7dukW7du3w8PBotT4l1PTv35+EhAQ6deokCSwrK6tGj8XRo0fTrl07duzYwa1bt7C0tJRC2TVk7m0Ir776Kvfu3SM6OhqFQoGjoyMzZsxo9ugmdnZ2LFq0iJiYGKKjoyksLMTMzAxXV1fpDKqOjg4ff/wxa9eulcJtde/enUmTJjXqyMu7777LypUrmTVrltQfm+K88csvv0xqaiqzZ8+mrKxMIyTog7CysuLLL7/kp59+YsGCBSiVSqytrfHx8Xno/NLUWFpaMnfuXL788ksiIiL4/PPPef/999m0aRP79++na9eujBo1imXLlj3Wc0aOHMmqVat47733KC8vb/COtq+vL7Nnz2bHjh3s2rVL+uB/6aWXHqs+NenWrRsVFRW0b99e40O0c+fOKJVKjI2N6z0vbmJiQmxsLBs3bqSyshJnZ2dmzpz5QEd3TcErr7wiedFXqVT07t2boUOHSo4DdXV1uXPnDsuXL6eoqAgzMzP8/PwkPyb1zcujRo3CwsKCXbt2sXr1aoyNjXFzc9Nwqthaef7555kxYwZLliyhsrKSsWPHPvQeQ0NDPv30U3788Uc++eQTjZCgNVErqGorrQ4ePNgilhIAQ4cOZfny5QwfPvyh87iRkRHh4eFs2LCBRYsWUVFRgaOjI+PHj9cqVyaTMXv2bP79738zb948Pvvss4fK4keV87q6ug88htVUcu5xaIisai4aMo5b8xr/Qd+QtY/qqHnYOtnV1ZUJEyYQGxvLli1b6NSpEyEhISxdurTO8nR1dZk+fTrLli0jIiKCuXPn1hlRq7nm1MdBR/UkD9ILBII2Q3l5OdOmTWPYsGHNvossEAgEAoFAIBAInk2EpYRAIACq/Vnk5OTg4eHBvXv3iI2NpaysTMv6QSAQCAQCgUAgEAiaCqGUEAgEEuozbXp6eri5uREREdGiMdkFAoFAIBAIBALB0404viEQCAQCgUAgEAgEAoGgRRDRNwQCgUAgEAgEAoFAIBC0CM+sUiIlJYWRI0c2iUdyQcO4ffs2oaGh5OXltXRVtFiyZAm7du1q6Wo0GNGWT4awsDDi4uJauhoAJCYm8u677zJq1KhGx6Z/0rTm/rlx40bWrl3b0tVoFsLDw1mzZk2zlH3gwAHJ67ZAoEaM9eajNcmftojom62HhsiPuLi4x47eIGjbPBM+JcLDw+nQoQOTJ09u9mcVFBQwffp0rXR/f38+/vjjBpUxcuRIPvroIwICApq6ei1KTEwMPXv2lEKQrVu3josXL5KVlYWlpSWRkZFa9xw9epSYmBiuX7+Oubk5gwYNYtiwYdL1yMhIDh48qHWfoaEhGzdulP5OTU1l/fr1ZGdnI5fLGTZsmEaI1uDgYObOnUtQUBAmJiZN+drNQnO0JcCePXvYu3cvBQUFWFtbM2LECAIDA6XriYmJHDp0iKysLFQqFR07dmTUqFF07txZytOW2jIyMpLS0lJmz55d5/WFCxe2itjXt2/fZs2aNYwbN46AgACMjY1bukoPpKX6Z3h4uEacbTXOzs4sWbIEgOHDh/Pee+8xZMgQ7OzsmvK1G0TtOcvMzAxPT09CQkJwcnJ64vVpKH369HnscMVNjUKhICYmhpMnT3Lr1i0pbN2gQYPw8/Nr6eo9E4ix/mD++9//smHDBtatW4e+fvWSu6KiggkTJmBnZ8c333wj5c3Ly+P9999nzpw5dO/evUmef+DAAdasWaOxHqrN2bNnWbhwIREREXh5eUnpSqWSf/7zn3h7exMaGtok9XmSiL75ZBg5cuQDrwcGBhIaGtrq5MfTivo7dOHChTz33HMtXZ1G8UwoJVqCf/3rXxpx7590LHaAyspKdHV1GxXfvrm4f/8++/btY9asWVKaSqUiMDCQa9eucfbsWa17Tp06xffff8/EiRPx9fUlJyeHqKgoZDKZFL934sSJjBkzRuO+OXPm0KVLF+nvgoICFi5cSL9+/Xjvvfe4cOECa9aswdzcXFL8uLi4YGdnx6FDh6SyWyvN1ZYJCQlER0czdepUPD09ycjIICoqClNTU/z9/YFq5U6fPn3o1KkThoaGxMfHM3/+fBYvXoyDgwPQttryYTwonviT5ObNm1RWVtKrVy/kcnlLV+eBtGT/nDlzJhUVFVK55eXlzJw5k7/+9a9Smrm5OT169CAhIaHFdv67d+/Oe++9B0BhYSGbNm3i66+/5ttvv32k8tRzfXNRUVGBTCZDJpM12zMaS0FBAXPmzMHY2Ji3334bNzc3qqqqOH/+PKtWreI///lPS1fxsamoqJA+ZJ/EfY1FjPWH061bN+7fv09GRoakvE9PT8fExITr169TUlIiyZnz589jYGBAp06dnmgde/TowSuvvEJkZCSLFy+WFPE//fQTKpWqTVpIib755Fi5cqX0+x9//EFUVJRGmlp2tCb5IWidPPVKicjISFJTU0lNTWXv3r0ALFu2TLr+559/snnzZq5du4azszNTpkzB3d1dun7x4kV++uknLl++LE06Y8aMeegOsJmZGZaWllrpdVlBhIWFMXDgQIYNGyaZLqm1qTY2NkRGRvLzzz/z22+/aWjVa2vA1XmGDh3Kjh07KCgoYP369VRVVbFx40b+97//oVQq6dixI+PGjXuiGrRTp04BaAjbSZMmAdUmW3UJiEOHDtGrVy8GDhwIgJ2dHa+99hqxsbEMHDgQHR0dTExMNP4XFy5cID8/X8NaJSEhAblcLj3P2dmZjIwMdu3apfF/8Pf3Jzk5udV/SDdXWx46dIigoCD69u0r5bl8+TKxsbGSsH3//fc1yg0NDeV///sfp0+flpQS0Hba8mHUHJtQPX6nTJnC2bNnOXXqFBYWFowcOZIXX3xRuqewsJANGzZw5swZALy8vJgwYYJG+9Tm5s2brFu3jnPnzgHVi8SJEyfSvn17Dhw4wPLlywGkfr1s2TJsbW2b5Z0fl5bsn+3atdMo9/Dhw9y/f59+/fpppPv7+7N58+YWWwwaGBhI8sHS0pIhQ4awaNEilEolMpmM6Ohofv/9d27evImlpSV//etfGTlypLSoq2+uh2oFxbp16zh06BAA/fv3Z8yYMZLS4tChQ/z666/k5OQgk8no2rUrEyZMwMrKCqg+2hgREcHs2bPZtm0bmZmZzJw5k9LSUg15k5eXx4YNG0hPT6esrAxHR0dGjhxJr169pPcMCwujf//+3Lp1i+TkZIyNjRk8eLDWzuOjoD6m8u9//xsjIyMp3dnZmRdeeAF48Liq2Y4jRoxgy5YtFBcX4+3tzTvvvCN9KFZVVbFp0yb2798PVO/6lZeXk5OTQ3h4OFC3NWZtKyyVSkVcXByJiYkUFhZib2/P8OHDpblDvbv1/vvvk5SUxKVLlwgJCWHAgAHs3LmTpKQkiouLcXBw4K233uIvf/nLA+97EnOvGOsPx9HREblczvnz5yWlREpKCt7e3ty4cYOUlBTpYzUlJQUvLy+Nj7fy8nJWrlxZ7/iJj4/nwIED5OfnY2JiQs+ePQkJCcHU1JSUlBRJdqh3s4ODg+vc2R47dixnzpwhOjqaSZMmSWvmiIgI9PT0+PHHH0lOTubu3bu4ubkREhKi8T4RERGsXr1aGjctvVsr+uaTo+a3jqmpqVYa1G2xExsbS3x8PGVlZfTu3bvONc3+/fuJi4uTrFJeeeUVBg8e3KxK+LbA6dOn2blzJ1lZWQB4eHgwfvx4nJ2dpXXiJ598AkDXrl0lWdXaeer/qxMnTsTLy4uXXnqJlStXsnLlSqytraXrP/30E6NHj2bRokWYmZnxww8/oA5Icu3aNebNm4e/vz9fffUVM2fOJDMzs1l3YBYuXAjA1KlTWblypfR3QykoKODIkSN8+OGHfPXVV+jr67Nw4UIKCwuZPXs2ixcvpkuXLnzxxRcUFRU1xyvUSVpaGu7u7o2y2igvL9eyMJHJZNy6dYsbN27UeU9SUhIdOnTQEETp6en06NFDI5+Pjw9XrlzR0GZ7eHiQkZGBUqlscB1bguZqy/Lyci1NtkwmIyMjQ6OdalJRUUF5ebkkiNS0lbZ8FLZv3y7NCX369OE///kPN2/eBKp3ZyIiIjAwMCA8PJx58+Yhl8v58ssvuX//fp3lVVVVsXjxYoqLi5k7dy5z586lqKiIr776CpVKRZ8+ffjXv/4FwIIFC7TmsNZGa+qfSUlJ+Pr6arWXh4cHhYWFreKs8b179zh69CguLi7S+xkaGvLuu+/y7bffMnnyZJKTk9m5c6fGfbXnenX7HTlyBJVKxbx58wgNDSUxMZFffvlFuq+iooI333yTr776itmzZ1NaWsp3332nVa/o6Gjeeustli5diqenp9b1srIyfH19mTNnDl999RW9e/fm66+/JicnRyPf7t27cXFxYdGiRQwfPpxNmzZx6dKlx2qz27dvc/r0aQYOHKihkFBjamr60HGlpqCggKNHjzJz5kw+++wzMjMz2bJli3R9165dJCUlERoayrx586iqquLIkSONrvOWLVvYt28fkydP5ttvv+X1119n1apVnDx5UiPf5s2bGThwIN9++y1/+ctf+OWXX9i1axdjxozh66+/5vnnn+frr78mMzPzgfc9CcRYbxjdunUjJSVF+jslJYVu3brRtWtXjfTU1FS6deumce/Dxo+Ojg4TJkzgm2++4YMPPiAjI0PyVdCpUycmTJiAoaGhtP6tTyEok8l47733+O9//8uJEydYvnw5w4YNw8vLi02bNnH06FHeffddFi1aRIcOHZg/f/4TXUM2FtE3WzdHjx5ly5YtjBw5kkWLFuHo6Mju3bs18iQmJrJ582ZGjRrFt99+S0hICLGxsSQkJLRQrVsPZWVlDB48mAULFhAeHo6xsTGLFi2ioqKCBQsWANUW+ytXrmTmzJktXNuG89QrJUxMTNDX18fQ0BBLS0ssLS01NGyjRo3C29sbJycn3njjDXJycigsLASqtal9+vRh6NChODg44OnpSWhoKL/99hvFxcUPfO7cuXMJCQmRftLS0hpUX7WW2dTUFEtLy0abj1dUVDB9+nTc3d1xcXHhwoULZGZmMmPGDDw8PLC3t+ett97C1tZW2kl7Ety4caPRZue+vr6cOHGCM2fOUFVVRW5uLvHx8UD1WeLa3L17l2PHjhEUFKSRrlAotLS2FhYWVFZWUlpaKqXJ5XIqKyul/39rpbna0sfHh/3795ORkYFKpeLy5cskJSVptVNNtmzZgpGRkbRDoKattOWj8OKLL/Liiy9ib2/PqFGj0NPTk86PJicno1KpmDZtGq6urjg5OTFlyhTKysr4448/6izv/Pnz/Pnnn7z//vs899xzPPfcc7z//vtcvXqVc+fOIZPJMDMzA6rnh9pzWGujtfTP3NxcUlNTteYDQKpffcrN5ub06dOSbBg/fjypqakaVkjBwcF07twZW1tb/Pz8eP3110lOTtYoo/Zcr6enB1S/28SJE3FycqJPnz4MGzZMakuotpzw8/PDzs4ODw8P/v73v5OWlsatW7c0yn/zzTfx8fHBzs6uTjnk5ubGgAEDcHFxwd7enhEjRuDu7s7x48c18vXo0YNBgwZhb2/Pq6++ir29vWS58Kjk5eWhUqlwdnauN8/DxpWaqqoqwsLCcHV1xcvLi5dfflnj+i+//MLw4cPp06cPTk5OTJgwoU4ryAdRVlZGfHw877zzDr6+vtja2tK3b1+CgoIkC041gwYNIiAgAFtbW9q3b8+uXbsYOnQoffv2xdHRkVGjRtGlSxctB4i173sSiLHeMLy9vbl06RLl5eUolUouXbqkpZTIycmhqKgIb29vjXsfNn6GDBmCt7c3tra2dO3albFjx3Ls2DGqqqrQ19eXLEnV69+6lHhqPDw8eO211/jqq68wMTHhzTffpKysjISEBMaMGYOfn59kUWxpaanVd1sTom+2bn755RcCAwN55ZVXcHR0ZMSIEXh4eGjk2bFjB2PHjpXmNX9/f1577bVW3e+eFAEBAQQEBODg4ICrqyvTpk2joKCAjIwMSV6rLfZrW+60Zp764xsPw9XVVfpdbb5aXFxM+/btuXLlCnl5eRw9elTrvvz8fCwsLOot9/3338fFxUWr7ObGyspKY8F05coVlEqllpPP8vJy8vPzn0id1M9r7HmyoKAg8vLyWLx4MZWVlZLp4rZt2+rUfh86dAiVSqVhSt8Y1PVr7bv7zdWWwcHBKBQK5syZg0qlwsLCgsDAQOLi4ups719++YXExETmzJmjdZyprbTlo1BzXOvp6WFubi5F8bly5QoFBQWMGzdO4x6lUlnveMvOzsbKykrDdNHOzg65XE52draWlU9rp7X0z6SkJORyeZ0OD1u6f3bp0oWpU6cC1bv+CQkJzJ8/n/nz52Ntbc3x48fZvXs3eXl5lJWVUVVVRVVVlUYZted6NZ6enhrt4eXlxdatW7l79y4mJiZcuXKF7du3k5mZye3btyWrgZs3b2p8zD7M5LqsrIzt27fzxx9/oFAoJKupmuMDNGUsVC/EH6bUfxg1LR3qo6HjytraWmP+ksvl0ni+e/cuRUVFGs7/dHV18fDw0FLiPKwu5eXl0g6WmsrKSmxsbDTSara7+vm1fQx07txZMk+v674nhRjrDcPb25vy8nIuXbqESqXC3Nwce3t7LC0tycvLQ6FQkJKSgqGhodaH2cPGz/nz54mJiSEnJ4e7d+9SVVVFRUUFCoXikdadwcHB7Nixg+HDh6Ovr09OTg6VlZUafVBXVxdPT0+ys7MbXf6TQvTN1k1OTg79+/fXSPP09JSsRkpKSrh16xYrV65k1apVUp6qqqoGzf9PO3l5eWzdupWMjAxKSkqkdrl58+YT+95sDp55pYR6d6km6g6vUqno378/f/vb37TyPOyf3r59e8njb010dHS0BlR9Jl810dXVbdB9tbXgVVVVWFhY8MUXX2jlfZIe/M3MzLh9+3aj7tHR0WHs2LGMHj0ahUKBubm5tENQl7fipKQkevfuraUVtLS01LKsKC4uRk9PT9qBBqT6tRbnhvXRXG0pk8mYNm0aU6ZMobi4GLlcTmJiIsbGxlptsnv3brZu3cq//vUvrUUUtJ22fBRqO5DT0dGRPhhVKhVubm784x//0LrvUbTVrcFJbWNpDf2zoqKCgwcPEhQUVOcc39L909DQUEM+uLu7M378eBITE/Hz82Pp0qUEBwczfvx4TE1NOXHihJb3/AfteNZHWVkZ8+fPp3v37kyfPh0LCwtKS0v5/PPPteTJw6LObNy4UbL4cHBwwNDQkGXLlmmVU7v965KBjcXBwQEdHR2ys7N5/vnnG31/zXH1oPHcmPJqv1NlZaX0u/rarFmztEy4a7fPo0b7aYkoQWKsNwxbW1tsbGwkqwi1I24jIyPc3d1JSUkhJSWFzp07a/XHB42fGzdusHDhQoKCghg1ahTt2rXj6tWrfPfddw1aV9aF+nl1tWV91CWnavb/lkD0zbaNeg4ODQ194o5f2wKLFi3CysqK0NBQrKys0NPT46OPPnrkcd9aaL02wE2Ivr5+oxcZAB07diQ7Oxt7e3utn0f1Imtubq5xDk+hUGh9MOvp6WnV19zcnOLiYo2FT+0zpXXh7u5OcXExOjo6Wu/wIEuPpsbNzU3rrHFD0dXVxcrKCn19fZKTk/Hy8tKaxDMyMvjzzz/rNJHz9PTUMhc+e/Ys7u7uGguArKysencfWxPN3Zb6+vq0b98eXV1dkpOT8fPz0zguEB8fz9atW5k9e7ZGKNCatJW2bGo6duxIXl4eZmZmWuOtPqWEs7MzhYWFFBQUSGn5+fkUFRU90Dy9tdLS/RPg999/p7S0VGsnRk1WVhZ6enpau/otia6uLkqlkosXL2JlZUVwcDAeHh44ODg0yrw3PT1dQ06kp6cjl8sxMTEhNzeX0tJSRo8eTdeuXXFycnpkq4ULFy4QGBhIQEAArq6uWFlZPTHru3bt2uHj48PevXspKyvTun7nzp0mGVcmJibI5XKNM/wqlYqMjAyNfObm5lpy/M8//5R+d3Z2xsDAgBs3bmjNC7UtJep6/sWLFzXSL1y40CrmBjHWG47ar4Tan0TN9PPnz9fpT+JhXL58WQov6uXlhaOjo5afh0dd/6qxs7NDX19fow9WVVWRnp4u9UH1/63msxuyPm1ORN9s3Tg5OZGenq6RVvNvS0tL5HI5+fn5dX6DPcuUlpaSk5PD66+/To8ePXB2dubevXuSIlD9XfM4476leCaUEjY2NmRkZFBQUCCZuTSE4cOHk5GRwcqVK7l69Sp5eXn88ccfGqFuGku3bt3Yu3cvly9f5urVqyxfvlzLsY6trS3nzp1DoVBImtSuXbty+/ZtYmJiyMvLY9++ffz2228PfV737t3p1KkTixcv5tSpUxQUFHDp0iV+/vnnBvu5aAp8fX3Jzs7WOHOXl5dHZmYmRUVFVFRUkJmZSWZmpqTpKykpISEhgezsbDIzM1m3bh3Hjh1jwoQJWuUnJibi4OBQp1AfMGAAhYWF/Pjjj2RnZ5OUlMSBAwcYOnSoRr60tDR8fHya9sWbgeZqy9zcXA4dOsT169fJyMhg6dKlZGVl8fbbb0t54uLiiI6O5p133sHR0VFSqt29e1ejjm2lLaHa0aC6vdQ/NT9kGsMLL7yAhYUFixcvJjU1lYKCAlJTU9mwYQPXr1+v857u3bvj6urKDz/8wOXLl7l8+TLff/89HTt21Dpf3BZoyf6pJikpCW9v73rjv6elpdGlS5cW2V2GatNi9djJzs5m7dq1lJWV0atXLxwcHCgsLOTw4cPk5+eTkJCg5U/iQRQVFfHjjz+Sm5vL8ePHiYuLY8iQIUD1UQUDAwP27NlDfn4+J0+eZOvWrY/0Dg4ODvz+++9cuXKFa9eu8cMPPzxRM+TJkyejUqmYPXs2x44dIzc3l5ycHBISEpg5c2aTjatXX32VuLg4jh8/Tm5uLj/++KOWAsLb25tTp05x4sQJcnNzWb9+veT8FqqtEocOHcrGjRvZt2+fNB4SEhJITEx84POHDRvGrl27OHLkCLm5uWzdupW0tDQt+dUSiLHecLp160Z6ejrp6eka65SuXbty9OhRKfJLY3BwcEClUrF7927J8W1tZ4E2NjaUl5dz9uxZSkpK6nW4XB9GRkYMGDCA6OhoTp48SXZ2NqtWrUKhUEhRKuzt7Wnfvj3btm0jNzeXM2fOaDnmfdKIvtm6GTx4MAcPHiQxMZHr168TExOjpewdOXKkFKEjNzeXa9eucfDgQWJiYlqo1q0DU1NTzMzMSEpKIi8vj9TUVFatWiVZ41hYWCCTyThz5kyd6/PWzDNxfGPo0KFERkby0UcfoVQqNUKCPghXV1ciIiLYsmUL4eHhVFVVYWtr+0jmomrGjRvHihUrCA8Px9LSkjFjxmhpc0NCQtiwYQPvvvsuVlZWREZG4uzszN///ndiYmKIiYmhV69evP7662zevPmBz9PR0eGTTz5hy5YtREVFUVxcjKWlJZ06dXpk3wuPgouLCx4eHhphIlesWCE5CAT4+OOPAc1whwcPHpTMlr28vAgPD9c6LnDv3j2Sk5MJDg6u89m2trZ88sknrF+/XgoPOnHiRI1woEqlkt9//51PP/206V66mWiutqyqqpImfz09Pbp168a8efM0zmTv3buXyspKli5dqlGnwMBAKZxtW2pLqF4YqNtLTe/evZkxY0ajyzI0NCQiIoKffvqJJUuWcPfuXeRyOd26ddOKUKJGR0eHjz/+mLVr1xIREQFUKyomTZrUJo9vtGT/hOrd8PPnz/PBBx/UW8fk5OQ6w+I9Kc6dO8eUKVOA6g9WR0dHPvzwQ+ljZdiwYfz4448olUp8fHwYNWoUq1evblDZffv2paqqin/961/o6OhoHEE0NzcnLCyMzZs3s3fvXlxcXBg3bpyWr4OGMH78eFasWMHcuXMxNTVl8ODBlJeXN7qcR8XOzo5FixYRExNDdHQ0hYWFmJmZ4erqytSpU5tsXA0dOhSFQsGKFSuAake3ffv21ZDb/fr1488//5Qicw0cOJDnn39e44No1KhRWFhYsGvXLlavXo2xsTFubm4MHz78gc9/9dVXuXfvHtHR0SgUChwdHZkxYwZubm4NfofmQoz1htOtWzcqKiq0jvZ27twZpVKJsbGxRjj6huDq6sqECROIjY1ly5YtdOrUiZCQEA353KlTJ1555RW+++47SktL6w0J+iDGjBkDwH/+8x/u3LlDx44d+fTTTyVHjfr6+vzjH/9g9erV/POf/8TNzY23336bf//73416TlMi+mbrpk+fPuTn57Nlyxbu37+Pv78/Q4YM4eDBg1KeoKAgDA0N2bVrF5s3b0Ymk+Hs7NzmQ80/Lrq6unz44YesW7eOGTNmYG9vT0hICN988w1QbW0/ceJEtm/fzrZt2+jSpUubCQmqoxIeQwRPiNOnT7Nu3Tq+/fbbVhc9YM+ePZw4cYLPPvuspavSIERbx/7NHQAAIABJREFUClozrbl/njx5ko0bN/L111836ty0QKBmzZo1ZGVltZmFXnMixrqgtSL6pkDQttALF1JV8ISwt7dHpVIhl8vr3TVuKTIzMxkwYICG48vWjGhLQWumNffPq1ev8tJLL2k5HBQIGsqpU6coKSnhpZdeaumqtDhirAtaK6JvCgRtC2EpIRAIBAKBQNBAhKWEQCAQCARNi1BKCAQCgUAgEAgEAoFAIGgRWtchK4FAIBAIBAKBQCAQCATPDEIp8QBu375NaGgoeXl5LV0VLZYsWcKuXbtauhoCgQCIjIxsUU/jgqcXIYealtbcnhs3bmTt2rUtXQ1BKyI8PJw1a9Y8MM+MGTP4+eefn1CNBM8qrXnubIuy6MCBA4SEhGikJSYm8u677zJq1ChpTNeV9rTyTIQEfVRiYmLo2bOnFL5p3bp1XLx4kaysLCwtLYmMjNS65+jRo8TExHD9+nXMzc0ZNGgQw4YN08hz5MgRYmNjuX79OsbGxnTv3p1x48ZhaWkJVHfU5cuXa5W9adMmZDIZAMHBwcydO5egoCBMTEya+tUFAkEdREZGaoSsUhMeHo6rq2sL1EjwtNNScigrK4uff/6Zq1evUlBQUGcowbYoh5qrPffs2cPevXspKCjA2tqaESNGEBgYKF0PDw/XCEeoxtnZmSVLlgAwfPhw3nvvPYYMGYKdnV1TvragFVFTjujp6WFqakqHDh3o3bs3L7/8Mvr6/39pPnPmzMeOznDgwAHWrFkjhbpsasLDw+nQoQOTJ09ulvIFrYOWmjvb4jfRw0K9BgYGEhoaSs+ePaW027dvs2bNGsaNG0dAQADGxsZ1pj0pIiMjsbGxeaJha4VSoh7u37/Pvn37mDVrlpSmUqkIDAzk2rVrnD17VuueU6dO8f333zNx4kR8fX3JyckhKioKmUwmxdW9cOECP/zwAyEhITz//PMoFArWrFnD999/z+effy6VZWhoyA8//KBRvnrwQXUMZjs7Ow4dOvTMx+wVCJ4k3bt357333tNIMzMze+DCsaKiQmOhKRA0hJaUQ/fv38fGxobevXuzZcuWOuvX1uRQc7VnQkIC0dHRTJ06FU9PTzIyMoiKisLU1BR/f3+g+uOyoqJCKre8vJyZM2fy17/+VUozNzenR48eJCQkaO2gCZ4u1HKkqqqKkpISzp8/z7Zt2zh8+DBz5szByMgIgHbt2rVwTQWClp07oe19E61cuVL6/Y8//iAqKkojTSaTST9qbt68SWVlJb169UIulwPV0exqpz3NiFVyPZw6dQqATp06SWmTJk0CIC4urs4BeOjQIXr16sXAgQMBsLOz47XXXiM2NpaBAweio6PDpUuXaN++PX/7298AsLW1ZdCgQXWabKp3rOrD39+f5OTkVjEABYJnBQMDA62xGRkZSWlpKbNnzwaqd46cnJwwNDTk4MGD2NrasnDhQrKzs9m4cSNpaWnIZDK8vb2ZMGHCQ8e64NmkJeWQh4cHHh4eQPUOWX20JTnUXO156NAhgoKC6Nu3r5Tn8uXLxMbGSgvr2h+Xhw8f5v79+/Tr108j3d/fn82bNwulxFNOTTliZWWFm5sbPXr0YNasWcTFxUm7k7WtEIqLi4mKiuLMmTNYWFgQHBz82HU5ffo0O3fuJCsrC6ge++PHj8fZ2VnKs337dvbt24dCocDU1BQfHx+mT59OZGQkqamppKamsnfvXgCWLVuGra3tY9dL0HpoyblTTVv6JqpZV3U42tr1r2nBVNMaZPr06QBMmzZNK23ZsmVUVVWxYcMG0tPTKSsrw9HRkZEjR9KrVy+p7LCwMPr378+tW7dITk7G2NiYwYMHS1Yqy5cvp6SkRFqzAlRVVREWFsaQIUOktUFNfvvtN7Zt28b169eRyWS4uLjw4YcfNun6VSgl6iEtLQ13d3d0dHQafE95eTkGBgYaaTKZjFu3bnHjxg1sbW3p3Lkzmzdv5sSJE/Tq1YvS0lKOHj2qYcIDoFQqmTZtGlVVVbi5uTFq1Cg6duyokcfDw4MdO3agVCo1tG0CgaDlOXz4MC+//DJffPEFKpWKoqIi5s6dS79+/QgJCaGyspLNmzezePFi5s2bh66ucPEj0KSl5VBDaEtyqLnas7y8XOvdZTIZGRkZ9VpJJSUl4evri7W1tUa6h4cHhYWF5OXlSWbSgmcDFxcXfH19+e233+o1mV6+fDk3btxgzpw5GBoasn79egoKCh7ruWVlZQwePBhXV1eUSiU7duxg0aJFfPvtt+jr63P8+HF27drFBx98gIuLC8XFxaSnpwMwceJErl+/jqOjI6NHjwaqLX4ETxctPXc+7d9Effr0wdLSkgULFrBgwQKsra0xMjLSSjM3N+fatWv4+vry1ltvIZPJOHr0KF9//TVff/01Tk5OUpm7d+9m5MiRDBs2jFOnTrFu3To6d+6Ml5cXL7/8Mp9//jlFRUWSBcbZs2dRKBS8+OKLWvVTKBQsXbqU0aNH07t3b8rKyqQ5oCkRq+B6uHHjRqNNZXx9fTlx4gRnzpyhqqqK3Nxc4uPjgep/KICXlxf/+Mc/+OGHHxg9ejR///vfUalUkhYMwNHRkXfffZePP/6YDz74AAMDA+bMmcP169c1nieXy6msrKSwsPAx31YgEDSU06dPExISIv0sWLCgzny2traMGzcOJycnnJ2dSUhIwNXVlbFjx+Ls7IyrqyvTp08nIyODK1euPOG3ELQFWlIONZS2JIeaqz19fHzYv38/GRkZqFQqLl++TFJSEpWVlZSWlmqVmZubS2pqKkFBQVrX1PW7ceNGY19P8BTg7OxMfn5+nddyc3M5deoUU6ZMoXPnznTs2JGwsDCUSuVjPTMgIICAgAAcHBxwdXVl2rRpFBQUkJGRAVSblVtaWtKjRw+sra157rnnpN1oExMT9PX1MTQ0xNLSEktLS6FgfwppybnzWfgmkslkmJmZAdVKPUtLS4yMjLTSdHV1cXNzY8CAAbi4uGBvb8+IESNwd3fn+PHjGmX26NGDQYMGYW9vz6uvvoq9vT3nzp0DqtcATk5OGj7S9u/fj7+/v6RUDAsLk5SjhYWFVFZWEhAQgK2tLS4uLgQFBTW5la+wlKiHurR3DyMoKIi8vDwWL15MZWWlZC6zbds2SbuYnZ3N2rVreeONN/Dx8aGoqIhNmzaxcuVKaUHo5eWFl5eXVG6nTp345z//ya+//iqZS8H/P0/1uAJJIBA0nC5dujB16lTpb5lMxubNm7Xyubu7a/x95coV0tLS6jTLzsvLk0zlBQI1LSmHGkpbkkPN1Z7BwcEoFArmzJmDSqXCwsKCwMBA4uLi6txZTEpKQi6X4+fnp3WtLbWnoOlRqVT17kbn5OSgo6OjIStsbGywsrJ6rGfm5eWxdetWMjIyKCkpoaqqCpVKxc2bN4FqpcUvv/zC9OnT8fHxwdfXF39/f61dcMHTS0vOneKbSJOysjK2b9/OH3/8gUKhoKKigvLyclxcXDTy1Xa+LpfLKS4ulv4OCgpi7969vPbaa9y+fZsTJ04wc+bMOp/p5uZG9+7dmTFjBj169KBHjx4EBAQ0uVWUUErUg5mZGbdv327UPTo6OowdO5bRo0ejUCgwNzeXtFJqT9oxMTF4eHhI53pcXV0xMjLi888/5+2336Z9+/Za5erq6vLcc89pheFR10+YygkETw5DQ8MGmVUbGhpq/K1SqejZsyfjxo3TymthYdFk9RM8PbQmOVQfbUkONVd7ymQypk2bxpQpUyguLkYul5OYmIixsbFWu1RUVHDw4EGCgoLqdI7bltpT0PRkZ2c/1B9DY0zoG8KiRYuwsrIiNDQUKysr9PT0+OijjyTHrNbW1ixdupTz589z9uxZNmzYwPbt25k/f77kkFPwdNMa5k41z/o30caNGyWLXQcHBwwNDVm2bJmGI2VAS77o6OigUqmkv1988UWio6O5cOECV69exdzcHB8fnzqfqaury2effUZ6ejpnzpxh3759/PTTT4SHh+Pm5tZk7yZsrOrBzc2NnJycR7pXV1cXKysr9PX1SU5OxsvLSxok9+/f1zJtU/9ds7PURKVS8eeff2qZyWRlZWFlZSWc5AkEbYCOHTuSnZ2NtbU19vb2Gj9PMsyToO3QmuRQfbQlOdRc7alGX1+f9u3bo6urS3JyMn5+flrt/Pvvv1NaWkr//v3rfE5WVhZ6enpau16Cp59r165x5swZAgIC6rzu5OSESqWSjlVA9dGKxzFXLy0tJScnh9dff50ePXrg7OzMvXv3qKys1Mgnk8nw8/NjwoQJLFy4kKysLC5evAhU9/uqqqpHroOg9dMa5k41z/o30YULFwgMDCQgIABXV1esrKzqPfL1INq1a8fzzz/Pvn372L9/P4GBgQ88eqWjo4OXlxdvvvkmCxcuRC6Xc/To0cd5FS2EpUQ9+Pr6Eh0dTWlpqXSmJy8vj7KyMoqKiqioqCAzMxOoPgOor69PSUkJx48fp2vXrlRUVLB//36OHTtGRESEVK6/vz9RUVEkJCRIZrPr16+nY8eOksOrbdu24enpiYODA/fu3eOXX37h2rVrhIaGatQxLS2tXq2WQCBoXQwcOJCkpCSWLl3K8OHDMTc3Jz8/n2PHjjFu3DihmBBo0ZJyqKKiguzsbKDaHFahUJCZmYmRkZGGpVBbkkPN1Z65ublkZGTg6enJnTt3iI+PJysri7CwMK06JCUl4e3tLe0U1iYtLY0uXbpoWVoJni7Ky8tRKBQaIUFjYmJwd3dn6NChdd7j6OiIr68vK1euZOrUqchkMtavX98gs3qVSiX1bTW6uro4OztjZmZGUlIS1tbWFBYWsnHjRo1d1gMHDlBZWYmnpydGRkYcPXoUPT09HBwcgOojJBkZGRQUFGBkZES7du2EX4mnjJacO8U3kSYODg78/vvv+Pv7o6+vz7Zt2x75yEpQUBALFiygsrKSGTNm1Jvv0qVLnDt3Dh8fHywtLbl69Sq3bt3SiNDTFAilRD24uLjg4eGhEV5mxYoVpKamSnk+/vhjQDP80cGDB9m4cSNQfQ4qPDxc4/zfSy+9xL1799izZw8bNmzAxMQEb29vxowZI+W5c+cOK1euRKFQYGJiQseOHYmIiNAoR6lU8vvvv/Ppp582XyMIBIImw8rKii+//JKffvqJBQsWoFQqsba2xsfHR5zNFdRJS8qhwsJCqWyA/Px8EhMT6dq1K+Hh4UDbk0PN1Z5VVVXEx8eTm5uLnp4e3bp1Y968eVpm+Pn5+Zw/f54PPvig3jomJyfXG3lB8PRw7tw5pkyZgq6uLqampnTo0IE333yTl19+uc5oLWqmTZtGVFQUERERmJubExwcTElJyUOfp1QqNcYzVJvkr1mzhg8//JB169YxY8YM7O3tCQkJ4ZtvvpHymZiYEBsby8aNG6msrMTZ2ZmZM2dK/Xvo0KFERkby0UcfoVQqRUjQp5CWnDvFN5Em48ePZ8WKFcydOxdTU1MGDx5MeXn5I5XVrVs32rdvj7W1db2KcqieAy5evMiePXu4c+cO7du354033qgzUsfjoKNqrK3mM8Tp06dZt24d3377bavT+u7Zs4cTJ07w2WeftXRVBAKBQNBMCDnUtLTm9jx58iQbN27k66+/rtPfhEAgELQUrXnubIuyqDWgVCqZOnUqkyZN4oUXXmjp6qAXrt7yEGhhb2+PSqVCLpdjamra0tXRIDMzkwEDBkhmVAKBQCB4+hByqGlpze159epVXnrpJekIjUAgELQWWvPc2RZlUUuiPjYWFxdHVlYWU6dObRWKJmEpIRAIBAKBQCAQCAQCwVNOQUEB06dPp3379rzzzjutxheHUEoIBAKBQCAQCAQCgUAgaBFa3lZDIBAIBAKBQCAQCAQCwTOJUEoIBAKBQCAQCAQCgUAgaBGEUkIgEDzz3L59m9DQUPLy8lq6KlosWbKEXbt2tXQ1BAKBQNAElJSUsHr1asLCwhg9ejShoaF88cUXnD17tqWrJhAInhBi3alN/cGQBQKB4BkhJiaGnj17Ym9vD8C6deu4ePEiWVlZWFpaEhkZqXXP0aNHiYmJ4fr165ibmzNo0CCGDRumkWfPnj3s3buXgoICrK2tGTFiBIGBgdL1Y8eOERsbS15eHpWVldjb2zNkyBBeeuklKU9wcDBz584lKCgIExOT5mkAgUAgEDwRvvnmG+7fv88777yDvb09xcXFpKamUlpa2tJVazIqKirQ1xefGAJBfTTXurOiooIdO3Zw6NAhioqKsLCwYOjQoQwePBiArKwsfv75Z65evUpBQQHBwcGMHDlSo4yWWneKGUMgEDzT3L9/n3379jFr1iwpTaVSERgYyLVr1+rcvTp16hTff/89EydOxNfXl5ycHKKiopDJZAwaNAiAhIQEoqOjmTp1Kp6enmRkZBAVFYWpqSn+/v4AmJmZMWLECJycnNDT0+PkyZOsWLECc3Nz/Pz8AHBxccHOzo5Dhw5JZQsEAoGg7XHnzh3S0tL47LPP6N69OwA2NjZ4eHhIecLCwhg4cKDGx0Z4eDgdOnRg8uTJUp5+/fqRn5/Pb7/9hqmpKSEhIfj4+LBq1Sr++OMP5HI5kydPljzrp6SkEBERwSeffMKWLVvIzs7mueee44MPPiA/P59169aRl5dHt27dCAsL0wivuH//fuLi4iQF+yuvvMLgwYOlMIIjR45k0qRJnD9/njNnzvDKK68wbty4Zm9PgaAt0lzrToClS5dy69Ytpk6dKik9lUqlxrNtbGzo3bs3W7ZsqbN+LbXuFEoJgUDwTHPq1CkAOnXqJKVNmjQJgLi4uDqFw6FDh+jVqxcDBw4EwM7Ojtdee43Y2FgGDhyIjo4Ohw4dIigoiL59+0p5Ll++TGxsrKSU8Pb21ih38ODBHDx4kAsXLkhKCQB/f3+Sk5OFUkIgEAjaMEZGRhgZGXHixAk6d+6MTCZ75LJ2797NW2+9xYgRI/jvf/9LZGQk3t7e9OnTh7feeouYmBh++OEHli9frvGcn3/+mQkTJmBiYsL333/P0qVLMTAwYMqUKejq6rJkyRK2bdsmycHExER+/vlnJk2ahLu7O9euXSMqKgp9fX0NmbR9+3befvttQkJC0NHRefRGEgiecppr3XnmzBnOnTvHDz/8gLm5OQC2trYa5Xh4eEhK0JiYmHrr2BLrTuFTQiAQPNOkpaXh7u7eqEVUeXk5BgYGGmkymYxbt25x48YNKU/tBadMJiMjI4OKigqtMlUqFefOnSM3N5cuXbpoXPPw8CAjI0ND2y0QCASCtoWenh7Tpk3j8OHDTJw4kU8//ZQNGzaQnp7e6LJ8fHwYOHAgDg4OjBw5kvLycuzs7AgMDMTe3p433niDkpISsrKyNO4bNWoUXbp0wdXVlVdeeYWLFy8yduxYPD09ee655wgMDCQlJUXKv2PHDsaOHUtAQAC2trb4+/vz2muvsXfvXo1y+/TpQ1BQEHZ2dlofQgKB4P/TXOvO//3vf3h4eBAfH88777zD+++/z9q1aykrK2t0HVti3SksJQQCwTPNjRs3kMvljbrH19eXH3/8kTNnztC9e3fy8vKIj48HQKFQYGtri4+PD/v37+f555/nueee48qVKyQlJVFZWUlpaan0zLt37zJ16lQqKirQ1dVl8uTJ9OzZU+N5crmcyspKCgsLpfOHAoFAIGh7BAQE4Ofnx4ULF7h06RKnT58mPj5esnpoKK6urtLvRkZGGBoa4uLiIqVZWloCUFxcXO99FhYWAFr3qe8pKSnh1q1brFy5klWrVkl5qqqqUKlUGuW6u7s3uO4CwbNMc6078/PzuXDhAvr6+syYMYM7d+6wbt06ioqKmDFjRqOe1xLrTqGUEAgEzzR1WTQ8jKCgIPLy8li8eDGVlZUYGxszePBgtm3bJmm+g4ODUSgUzJkzB5VKhYWFBYGBgcTFxWlox42MjPjqq68oKyvj3LlzrF+/HhsbG+m8MSDVT1hKCAQCQdtHJpPRo0cPevToQXBwMCtWrGDbtm0MGzaszt3TyspKrTQ9PT2ttLqcS9ZWHtS8T/2s2vep76mqqgIgNDRUw9S8LoyMjB54XSAQVNNc6071uP3ggw8kB5WTJk1i/vz5KBQKSVHZEFpi3SmUEgKB4JnGzMyM27dvN+oeHR0dxo4dy+jRo1EoFJibm3Pu3Dmg+pwfVE/o06ZNY8qUKRQXFyOXy0lMTMTY2Fg66wegq6sraaHd3NzIyckhJiZGQymhrl/N+wQCgUDwdODs7ExVVRVKpRJzc3OKioqka0qlkpycHNzc3J54vSwtLZHL5eTn52tEjhIIBI9Oc607LS0tsbKy0oiY4eTkBMDNmzcbpZRoiXWnUEoIBIJnGjc3Nw4ePPhI9+rq6mJlZQVAcnIyXl5eWhO4vr4+7du3l/L4+flJHsvroqqqivLyco20rKwsrKysGiVQBAKBQNC6KC0tZcmSJfTr1w9XV1eMjY0lB8je3t6YmJjg7e3N/v378ff3x9zcnJ07d9ZpKfGkGDlyJGvXrsXExAQ/Pz8qKiq4evUqhYWFvP766y1WL4GgrdJc687OnTtz/PhxysrKJMul69evA9VRfhpDS6w7hVJCIBA80/j6+hIdHU1paakUAi0vL4+ysjKKioqoqKggMzMTqN7N0tfXp6SkhOPHj9O1a1cqKirYv38/x44dIyIiQio3NzeXjIwMPD09uXPnDvHx8WRlZREWFibl2blzJx4eHtjZ2VFeXs6pU6ckB2g1SUtLk8K6CQQCgaBtYmRkhKenJ7/++it5eXmUl5djZWVF3759eeONNwB47bXXKCgoYPHixRgZGTFixAgNy4knTVBQEIaGhuzatYvNmzcjk8lwdnYW0aAEgkekudadffv2ZceOHf+vvbsLiWrr4zj+05mmxJy0RHsRidCJNGoyg6JCagKngoqoiLKLBAus6EbqwgKDbopIEIu80UDspgsxvDBpEoUpCUl7QQ0mEF8mU6YmX0BzdJ6LcHhsPD3P6djZneP3A3Oz9lpr//e+GJf/+e+1dffuXR09elSjo6O6f/++tm7dGto/JhAIqLe3V9K3Kiy/36+uri4tWrRoxt4RRqw7I4LfP2wGAPNMYWGhdu7cGVpkFRUVqb29PaxfaWmpEhISNDQ0pBs3bqi7u1uSZLPZdPz4caWmpob69vb2qqSkRF6vVyaTSenp6crJydHKlStDfR48eKDm5mb5fD5ZLBatWrVKTqcz9BpR6dsfjby8PBUWFspms/2qWwAAAIC/wa9Yd0rffhArLy9XZ2enoqOjtWXLFp08eVJRUVGSpIGBAZ0/fz7sPGlpaSoqKpJk3LqTpASAea+trU0VFRUqLi7+4aMVRqirq1NLS4uuXLlidCgAAAD4i1h3hvu97gIAGMButys7O1s+n8/oUMKYzWbl5uYaHQYAAADmAOvOcFRKAAAAAAAAQ1ApAQAAAAAADEFSAgAAAAAAGIKkBAAAAGCgkZER5eXlqb+/3+hQwlRWVqq8vNzoMAD8i5mNDgAAAACYz6qrq7Vp0yYtX75cklRRUaF3796pp6dHsbGxunPnTtiYZ8+eqbq6Wh8+fJDVapXT6dSBAwdm9Kmrq9Pjx481MDCg+Ph4HT58WFlZWaHjf/QqwqSkJN2+fVuSdPDgQV24cEH79+9XYmLiXF42AEgiKQEAAAAYZnx8XE+fPtXly5dDbcFgUFlZWeru7tbr16/DxrS2tqqkpESnT5+W3W5XX1+fysrKZLFY5HQ6JUn19fWqqqrS2bNnlZqaKo/Ho7KyMkVHRyszM1OSVFBQoEAgEJp3YmJCBQUF2rZtW6jNarVqw4YNqq+v16lTp37VbQAwj/H4BgAAAGCQ1tZWSdLatWtDbbm5udq7d69WrFgx65impiZt3rxZ2dnZSkxMVEZGhg4dOqSamhpNv1ivqalJDodDO3bsUGJiorZv3649e/aopqYmNM/ixYsVGxsb+nR2dmp8fFy7du2acb7MzEy53e65vnQAkERSAgAAADBMR0eH1qxZo4iIiP97zMTEhBYsWDCjzWKxyOfzaXBwMNTHYrGE9fF4PDOqI/6by+WS3W5XfHz8jPaUlBR9+vTpt9zzAsA/H0kJAAAAwCCDg4OKi4v7U2PsdrtaWlr06tUrTU1Nyev1qra2VpLk9/slSRs3blRDQ4M8Ho+CwaDev38vl8ulyclJDQ8Ph83p9XrV3t4uh8MRdmw6vumEBwDMJfaUAAAAAAwyW0XD/+JwONTf36+bN29qcnJSUVFR2rdvnx4+fBiquDhy5Ij8fr+uXr2qYDCoJUuWKCsrS48ePZq1KsPlcikuLk4ZGRlhx6bj+/r1609cIQD8GEkJAAAAwCAxMTEaGRn5U2MiIiKUk5OjEydOyO/3y2q16s2bN5IUekOGxWJRfn6+zpw5oy9fviguLk5PnjxRVFSUrFbrjPkCgYAaGxvlcDhkMpnCzjcd3/fjAGAu8PgGAAAAYJDVq1err6/vp8ZGRkZq6dKlMpvNcrvdstlsYYkDs9msZcuWKTIyUm63WxkZGYqMnPkvwIsXLzQ8PKzdu3fPep6enh6ZTCYlJyf/VJwA8CNUSgAAAAAGsdvtqqqq0vDwsGJiYiRJ/f39Ghsb0+fPnxUIBNTV1SVJSkpKktls1tDQkJqbm5WWlqZAIKCGhgY9f/5c165dC83r9Xrl8XiUmpqq0dFR1dbWqqenR+fOnQuLweVyaf369aEqi+91dHRo3bp1Wrhw4dzfAADzHkkJAAAAwCDJyclKSUmR2+2W0+mUJN27d0/t7e2hPpcuXZIklZaWKiEhQZLU2NioyspKSZLNZlNRUZFSUlJCY6amplRbWyuv1yuTyaT09HRdv349NH7ax48f9fbtW128ePEPY3S73Tp27NjcXDAAfCciOP0yYwAAAAB/u7a2NlVUVKi4uDjs0QqjvXz5UpWVlbp169as+011BXLuAAAAa0lEQVQAwF/1e33rAQAAAPOM3W5Xdna2fD6f0aGEGRsbU35+PgkJAL8MlRIAAAAAAMAQVEoAAAAAAABDkJQAAAAAAACGICkBAAAAAAAMQVICAAAAAAAYgqQEAAAAAAAwBEkJAAAAAABgiP8Ay6vDS8y6EY0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 690 + }, + "id": "VCnP_uOT-ApN", + "outputId": "af46a636-923f-4a9b-81e1-418a70319878" + }, + "source": [ + "plot_components(components_df, 'fc0', ascending=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAAL1CAYAAADw5l6HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZQV5Z038G9DAyICzSIYZYuyqLxRCS4oGFQQJyqoqGFcIkiirzGOS0YzGscBjVGJM1l8jTETFUmicRsVMYkLKFFj1GhU3FAScQETiUATXJCt3j883WPbDULTWmg+n3NyYj/1VNVTv1t1D/d7q55bURRFEQAAAICPWbOyBwAAAAD8YxJKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClEEoA/IMYN25cKioq8tJLL5U9lA3WmGOZOXNmKioqMnHixA3e/0svvZSKioqMGzdug7f1j2TOnDk55JBDssUWW6SioiJVVVVlDwma3KfpvRbg4yCUAGgCFRUVqaioSLNmzfLnP/95jf323nvv2r5XX331xzfAfwCCgqa1ISFOQ6/DqlWrcvDBB+fXv/51DjzwwEyYMCFnnnlmo8d3++23Z6+99kr79u2z2WabZbfddsuUKVMa7FvzIZFPD9c7wKeHUAKgiVRWVqYoilx55ZUNLp8zZ05mzpyZysrKj3lk77nwwgvz3HPPZauttipl/03p03Qs/yjmzp2bZ599NmPHjs0VV1yRiRMnNjqUuPTSSzNy5Mg8/fTTOfroo3Pcccfltddey7hx43L66ac38cgBgI+SUAKgiXTt2jU777xzJk+enJUrV9ZbfsUVVyRJRo4c+XEPLUnymc98Jttuu21atGhRyv6b0qfpWP5RvPbaa0mSLbfccoO289JLL+X0009Px44d8+ijj+ZHP/pRvv/972fWrFnZZptt8l//9V/5/e9/3xRDBgA+BkIJgCZ03HHH5a9//Wtuv/32Ou0rVqzI1VdfnT322CPbb7/9GtefM2dOjjnmmGy11VZp2bJlttxyyxxzzDGZM2dOnX4nnHBCKioqMnXq1Aa38/DDD6eioiKHHXZYbdvannN++OGHc9hhh2WLLbZIy5Yt07179/zf//t/az9Ivt+LL76Y448/Pr17907r1q3TsWPHfO5zn8sJJ5yQhQsXrq08Sd77UNrQHQ49e/ZMRUVFvv3tb9dp/81vfpOKior8x3/8xxqPZeLEifnsZz+bJJkyZUrtIzJrekzmiSeeyAEHHJCqqqpsuummGTp0aB588MEPHfu6+Mtf/pKvf/3r6dWrV1q2bJnNN988o0ePzmOPPVan35133pmKioqcffbZddrvvffe2rG/+uqrdZaNGTMmFRUVefHFF+u0z549O+PGjUv37t3TsmXLdO3aNUceeWSef/75euN7/fXXc/rpp6dfv35p06ZNqqqq0q9fv4wbN652u+PGjcvee++dJDn33HPr1HPmzJnrXZOKiooMHTq03vbe/2jIqlWrcvnll2fw4MFp3759Wrdund69e+erX/1qnfP/qquuyrvvvpuTTjopvXr1qm3v0KFDvvWtbyVJLr/88vUe4wfNmzcvJ598cvr06VN7nu+66671zs8keeyxx3LooYemS5cuadWqVXr27JkTTzwxf/nLX+r1rTl3586dm0svvTTbb799Ntlkk/Tq1SsXXHBBiqJIktx4443Zdddd06ZNm3Tp0iUnnXRS3nnnnXrbq6ioyF577ZXXXnstX/7yl9OlS5e0bt06AwcOzLXXXtvgsa1evTqXX355dtlll2y22WZp06ZNdtlll/z4xz/O6tWr17iPN954I8cff3w+85nPpFWrVunfv38mT568xhreeeed2X///dO5c+e0atUq22yzTc4444xUV1fX69urV6/06tUrb731Vs4444z06NEjrVq1Su/evTNp0qTauiTrf73XWLZsWaqqqtKlS5cGg+Mk+drXvpaKioo67+G33nprjj766PTt2zdt2rRJmzZtMnDgwFxyySUN1qshH/Y4VM3xN+SXv/xl9t5771RVVWWTTTbJdtttl/PPPz/vvvtuvb73339/Ro4cmW7duqVVq1bZYostMmjQoJx77rnrNE6AMpRzDzHAp9QRRxyRb3zjG7niiity8MEH17bfdtttWbBgQSZNmpQ//elPDa77hz/8IcOHD8/SpUszatSobL/99pk9e3Z+8YtfZOrUqZk+fXp22WWXJMnYsWPzk5/8JD/72c9y0EEH1dtWzbP16/K89VVXXZXjjz8+rVq1yqhRo9K9e/fMmTMnV1xxRaZNm5aHHnooPXr0SPLeB+5ddtklf//737P//vvn0EMPzbJlyzJ37tz8/Oc/z0knnZROnTqtdX/77LNPrrnmmsyePTvbbrttkuRPf/pTXnnllSTJjBkzcs4559T2nzFjRpJk2LBha9zmXnvtlerq6vzwhz/MjjvuWKf2O+20U52+jz76aL773e9m9913z1e/+tW88sor+Z//+Z8MGzYsTzzxRPr16/ehNVuTuXPnZsiQIXnttdeyzz775Igjjsirr76aG2+8Mb/61a/yP//zPznwwAOTJHvuuWdatmyZGTNm5Dvf+U69463575rXsCiK3HvvvenVq1e23nrr2j533HFHRo8enRUrVmTkyJHp3bt35s2bl5tvvjm/+tWvcu+99+bzn/98kuTtt9/O4MGD8+c//zn77rtvRo4cmaIo8vLLL2fq1Kk57LDDsvXWW9fWb8qUKRk6dGj22muv2v2t6YPT2kyYMCEvvfRSve3V/P/y5ctz4IEH5u6770737t1z5JFHpl27dnnppZdyyy23ZMiQIenTp0+S5J577kmS/NM//VO9/Xzxi1+s06exHn300ey3335ZtGhRvvCFL2T06NF5++238+yzz2bixIl1zs/bb789hx56aIqiyGGHHZaePXvmsccey49//ONMnTo1DzzwQO0H6Pc7/fTTM3PmzIwcOTIjRozIbbfdlrPPPjvLly9Px44dc+aZZ+bggw/Onnvumbvvvjs/+tGPsmrVqvz4xz+ut63Fixdnjz32SFVVVY499thUV1fnhhtuyFFHHZX58+fnjDPOqNP/y1/+cq699tp07949X/3qV1NRUZFbbrklJ554Yh544IFcc8019fZRXV2dwYMHp2XLljnssMPy7rvv5sYbb8z48ePTrFmzjB07tk7/c889NxMnTkzHjh1z4IEHpkuXLpk1a1b+8z//M7/+9a/z+9//Pu3atauzzooVK7Lffvvltddeyxe/+MVUVlbm1ltvzZlnnplly5ZlwoQJSdbven+/TTbZJGPGjMl///d/5ze/+U29u9befffdXH/99enatWud8+vMM89Ms2bNsttuu2WrrbbKkiVLcs899+SUU07JH/7wh/z85z9f4z431Pjx4zN58uR069Ythx56aKqqqvLQQw/lnHPOyYwZM3L33XfXPhJ4xx135IADDki7du0yatSobLXVVlm0aFGee+65XHbZZbX1A9joFABssCTFVlttVRRFUXzlK18pmjdvXrz66qu1y/fbb7+iXbt2xVtvvVWcffbZRZJi8uTJtctXr15dbLvttkWS4he/+EWdbV933XVFkqJfv37FqlWratv79u1btGzZsli4cGGd/suWLSs6dOhQdOnSpVixYkVt+9ixY4skxdy5c2vbnn/++aJFixbFNttsU8ybN6/OdqZPn140a9asOPjgg2vbLrnkkiJJ8YMf/KBeDd58883i7bff/tBaXXnllUWS4tJLL61tu/zyy4skxb777lu0bNmyeOutt2qX7bTTTkXr1q2Ld999d63HMnfu3CJJMXbs2Ab3e++99xZJ6tX+/fv/2te+9qHjX9u+RowYUSQpzj///Drtv/vd74rmzZsXHTt2LJYuXVrbvueeexbNmzcvqqura9sGDRpUDBgwoOjUqVNx9NFH17Y/8cQTRZJi/PjxtW2LFi0qqqqqik6dOhXPPPNMnX0+9dRTRZs2bYoBAwbUtt12221FkuLUU0+td0zvvvtu8fe//73275p6TZgwYZ1q8mHWtr2zzjqrSFKMHDmyWLZsWZ1ly5YtKxYsWFD7d+fOnYskxRtvvNHgftq0aVMkqXMOrY9333236NWrV5GkuOaaa+otf/91vXTp0qJjx45Fs2bNivvuu69Ov4suuqj2nH6/mnO3Z8+eda65xYsXF506dSo23XTTonPnzsWzzz5bu2zZsmXFdtttV7Rs2bJ4/fXX62yv5pw+/PDD67w/vPjii0WHDh2KFi1aFH/+859r26+99toiSTFgwIA65+Kbb75ZDBw4sMHjrtnHV77ylWLlypW17c8880zRvHnzYrvttqvT/5577imSFLvvvnuxePHiOssmT57c4DnYs2fPIknxxS9+sc77yOuvv160b9++aN++fbF8+fLa9g+73tfkwQcfLJIUhx56aL1lN9xwQ5Gk+MY3vlGn/U9/+lO9vqtWrSqOOeaYIknx0EMP1VnW0PvTh11PPXv2LHr27FmnraZWhxxySL331gkTJtR7Lx49enSRpHjiiSfqbf9vf/tbg/sF2Bh4fAOgiR133HFZtWpVrrrqqiTJyy+/nLvvvjtHHXVUNt100wbXefDBBzN79uzsvvvuOeqoo+osGzNmTIYMGZLnn38+DzzwQG372LFjs3z58vzyl7+s03/atGlZvHhxjjrqqA+dVPPHP/5xVqxYkR/+8If1HqkYNmxYRo0alWnTpmXp0qV1lrVu3brettq0adNg+wfV3PHwwTsCunbtmpNPPjnLly+vPc6FCxfmySefzJAhQ9KyZcsP3fa6GDx4cL07SMaPH5/Kyso88sgjjd7uvHnzctddd6VHjx755je/WWfZHnvskSOOOCKLFi3KzTffXNs+bNiwrFq1Kr/97W+TJEuXLs2jjz6afffdN3vvvXedb/wbumPkZz/7Waqrq3PuuefWeyzo//yf/5Pjjjsujz/+eJ599tk6yxp6nVq2bJm2bds28ugbb9WqVbnsssvSunXrXH755WnVqlWd5a1atcrmm29e+/eSJUuSJO3bt29wezXtNf3W17Rp0/LSSy9l1KhROfLII+st79atW+1/T506NYsWLcqYMWOy55571un3r//6r+nVq1fuvvvu2ruA3u+cc86pc81VVVVl1KhRefvtt/O1r30t2223Xe2yVq1aZcyYMVm+fHmee+65ettq3rx5Jk2alGbN/vefdZ/97Gdz8sknZ8WKFXW+ya95X7rooouy2Wab1ba3adMmkyZNSvK/89+836abbprvfe97ad68eW3b9ttvn8GDB+e5557Lm2++Wdt+ySWXJEl++tOf1vvZ13HjxmWnnXZq8G6MmnXff3526dIlBx10UJYsWdLg40jra/fdd0/fvn0zbdq0LFq0qM6ymjvMPnjXxzbbbFNvO82aNcspp5yS5L3HVD4KP/zhD1NZWZmrrrqq3jV7zjnnpFOnTg3WsaHru3Pnzh/JGAGagsc3AJrYbrvtls997nO56qqr8u///u+54oorsnr16hx33HFrXOePf/xjkvcebWjIPvvskwceeCCPP/54vvCFLyRJjjnmmJxzzjmZMmVKvv71r9f2XZ9HN2omBPztb3+bP/zhD/WWL1iwIKtWrcoLL7yQgQMHZtSoUfnWt76Vr3/967nzzjuz3377ZfDgwdl+++3X+ScXe/bsma233jozZ87M6tWra+cpGD58eIYOHZrKysrMmDEjI0aMyL333puiKNZYl8bYeeed67W1aNEiXbt2zeLFixu93ccffzzJe49lNDQB5z777JNf/OIXefzxx3PMMcfUtk2cODEzZszIqFGj8tvf/jYrV67MsGHD0qtXr9x000157rnnst1229UGFO+vRc3r9+STTzb4rPoLL7yQJHnuueey/fbbZ+jQodlqq61y0UUX5Y9//GP233//DB48ODvttFOdD5sfp9mzZ2fJkiXZbbfdNngSzKbw0EMPJfnfR0HWZm3XbWVlZb7whS/kpZdeyuOPP177CFSNhs7DmuMfOHBgvWU1Aca8efPqLevRo0eDj4jstddeOffcc2vPzZoxN2vWrM4jOTWGDh2a5s2b1+lfo0+fPvUet0iS7t27J3nvEZKakOP3v/99WrRokRtvvDE33nhjvXWWL1+ev/3tb1m4cGGdx73at2+f3r17r3UfTWHs2LE5++yzc9111+XEE09M8t5cK3feeWcGDBiQHXbYoU7/hQsX5uKLL86vf/3rvPjii3nrrbfqLJ8/f36TjOv93n777Tz55JPp3LlzfvCDHzTYp1WrVnVCqqOOOio333xzdtttt4wZMyZ77713Bg8eXCdIA9gYCSUAPgLHHXdcTj755PzmN7/J5MmTM3DgwAwYMGCN/Wu+1f3MZz7T4PKa9vdPENetW7cMGzYsd999d+0H1wULFuSOO+7ITjvtVO8f1g2pmZjy4osvXmu/mm9Be/bsmUceeSQTJ07MHXfcUfutf/fu3XP66afn5JNP/tB9Ju992//Tn/40f/zjH9OiRYv87W9/y7Bhw9K2bdvssssutXcFrMt8Euvrg9/c1qisrMyqVasavd3GvIaDBg1KmzZt6hxvy5YtM2TIkNq5G2bMmJE+ffrkvvvuy/bbb58tttiidv2a1++nP/3pWsdW8/q1a9cuDz30UCZMmJDbbrut9hvezp0758QTT8y///u/f+y/aFJTj3X9edf27dvnjTfeyJIlSxqcv+TD7qRoyvE05jWv0dD4au5sWtuyFStW1FvWtWvXBvdfc668/66RJUuWpGPHjg3eeVRZWZnOnTtnwYIF9Zat7bpJUufaWbhwYVauXPmhkyu++eabdV7D9dnHhnh/oFsTSlxzzTVZuXJlvbskqqurs8suu2Tu3LnZddddc8wxx6Rjx46prKysndeioQknN9TixYtTFEX+9re/rfMklaNHj87tt9+e//qv/8pVV12Vn/zkJ0neC7kuvPDC7Lvvvk0+ToCm4PENgI/Al7/85bRu3TonnHBC5s+fn+OPP36t/Ws+hPz1r39tcHnNLP4f/LBS8w/omrsj1vQP6w/b75IlS1IUxRr/V/PLCUmy3Xbb5frrr8/ChQvz6KOP5qKLLsrq1atzyimn5Morr1yn/dZ8szx9+vR6wcM+++yTxx9/PIsWLcqMGTPSvn372okaN2aNeQ1btGiRIUOG5Jlnnslf//rXzJgxI7vvvns23XTT9O3bN926dcv06dPzyCOPZOnSpfW+ka/Z1pNPPrnW1+/950O3bt1y5ZVXZsGCBXn66adzySWXpFOnTjnvvPNy3nnnNWlN1kXNB9F1/ba5ZiLSmrtA3u8vf/lL3nrrrXTr1m2Nj0o15Xgae902tddff73B9ppxvX//7du3z6JFixoMN1auXJk33nijwTsi1kf79u3ToUOHtZ6TRVGkZ8+eG7SfxurWrVv22WefPPLII5k9e3aS995DW7RoUe+RnSuuuCJz587NhAkT8vDDD+eyyy7L+eefn4kTJ2bMmDHrvM+aR2vW9KsfHwyual6zAQMGfGgd3++AAw7IPffck8WLF2fGjBk57bTT8swzz+TAAw+s9xgXwMZCKAHwEaiqqsphhx2WefPmpU2bNjniiCPW2r/mLoo1/dzivffemyT1PpyPHj067dq1yy9+8YusXr06U6ZMSWVlZYPPwjdk0KBBSd77Gbn1VVlZmYEDB+bf/u3faue1uPXWW9dp3X322ScVFRWZMWNG7rnnnmy99da1dwYMGzYsq1evzs9+9rPMmTMne+211zo9WlDTp6m+TV1fNa/hAw880OAHjzW9hjVhzC9/+cs8/fTTde4K2WeffTJz5szcfffddfrW2JDXr6KiIv3798+//Mu/1G7//a/fx1XPbbfdNlVVVZk1a1aDP0H7QTXBzB133FFv2W9+85s6fRqjpqY121qbtV23K1eurH1dPupQ7ZVXXmnwp35rxvX+u7QGDBiQ1atX57777qvX/7777suqVas2eLyDBg3K4sWL88wzz2zQdtZmQ8/PmsfbpkyZkieeeCKzZs3KF7/4xTrzlySp/bWkQw89tN42auaCWRcdOnRIkno/81uzjw/OgbLZZpulf//+eeaZZ+rNfbEu2rRpk3322Sff+9738q1vfSvLly9fp3MaoAxCCYCPyPnnn59bbrkld95554dOIDh48OD069cvDzzwQG666aY6y2666abcf//96du3b4YMGVJnWevWrfOlL30p8+fPz/e///08+eST2X///dOlS5d1GuNJJ52UFi1a5LTTTmvwm+fly5fX+cD72GOPNTiBYM03tev67XSXLl3Sv3///O53v8t9991X58P2HnvskU022SQXXnhhknX/gNmhQ4dUVFQ0OKngx6Fbt27Zd99989JLL9V7Bvzhhx/Otddemw4dOuSQQw6ps6zm+C666KIURVEvlFiyZEkuu+yyBucBOPbYY1NVVZVzzz23wUk6V69eXecD8zPPPNPgt+oNvX41t9V/1PVs3rx5TjzxxLzzzjs54YQT6t0KXzP/QI1jjz02rVq1yqWXXlrng/jixYtzwQUXJElOOOGERo9n5MiR6dWrV2677bZ6k8gmded0OPjgg9OxY8f88pe/rJ2LosYPfvCDzJ07N8OHD683n0RTW7VqVf7t3/4tq1evrm2bO3duLrnkklRWVuboo4+ubR8/fnyS5Kyzzsrbb79d2/7222/nzDPPTJJ85Stf2aDxnHbaaUnee4ytoaDprbfeqlev9bWh1/v7A92rr746ScPz8NSEpR8Mnh5//PHa96h1se2226Zdu3aZOnVqncdj3nnnnTU+9vaNb3wjy5cvz/jx4xt8BGjx4sW185ok74VKDQWi6/v+DPBxM6cEwEekR48e6/xhpKKiIlOmTMm+++6bMWPG5KCDDsq2226b559/Prfeemvatm2bn/3sZ3Vm168xduzYXHHFFTnrrLNq/15X2267ba666qqMHz8+/fv3zz/90z+lb9++WbFiRV555ZXcf//92XzzzWtvcf75z3+en/zkJxkyZEi22WabdOjQIX/+858zbdq0tGrVKqeeeuo673vYsGF5+umna/+7RqtWrTJ48OD1nk9is802y2677Zb7778/Rx11VPr27ZvmzZtn1KhR6zS/RlO4/PLLM3jw4Jxxxhm56667svPOO+fVV1/NjTfemGbNmmXy5Mn1AqoBAwakQ4cOWbBgQdq2bZtdd921dlnNsS9YsCA777xzvWfuO3XqlJtuuimHHHJIBg0alGHDhqV///6pqKjIq6++mt///vdZuHBhli1bliS5++67c8YZZ9T+AkGXLl0yb968TJ06Nc2aNcsZZ5xRu+1+/fplq622ynXXXZcWLVqkZ8+eqaioyJe//OUmv+2+5tb4adOmpW/fvjnwwAPTtm3bvPrqq7nrrrty8cUX135g/OxnP5uLL744J598cnbeeeeMGTMmLVu2zE033ZR58+blX//1X7P77rs3eiwtW7bMjTfemBEjRuTII4/MT37ykwwaNCjLli3Lc889lxkzZtR+8Ntss81y1VVX5fDDD8/QoUNz+OGHp0ePHnnsscdy1113ZYsttqh9rv+jtMMOO+Thhx/OwIEDM2LEiFRXV+eGG25IdXV1vvvd79b59YgjjzwyU6dOzQ033JD+/fvn4IMPTkVFRW699dbMnTs3Y8aMqfcLQOtr2LBhueiii3LWWWelT58+2X///fPZz342b775Zl5++eX89re/zZAhQxq822Vdbej13rp16xx++OG58sorc9lll6VTp0454IAD6vU75phjcvHFF+fUU0/Nvffemz59+mTOnDm5/fbbM3r06Fx//fXrNN4WLVrklFNOybe//e0MGDAghxxySFauXJm77747W265ZYOTvI4fPz6PPfZYLrvssmyzzTbZb7/90qNHjyxatChz587Nfffdl2OPPTaXX355kuTkk0/O/PnzM3jw4PTq1SstW7bMY489lnvuuSc9e/bMP//zP6/TWAE+dh/5j44C/ANIUmy11Vbr1Pfss88ukhSTJ0+ut2z27NnF0UcfXWyxxRZFZWVlscUWWxRHHXVUMXv27LVus3fv3kWSomPHjsW7777bYJ+xY8cWSYq5c+fWWzZr1qxi7NixRY8ePYqWLVsWHTp0KPr3718cf/zxxYwZM2r7PfTQQ8UJJ5xQ7LDDDkWHDh2KTTbZpNhmm22KcePGFU899dQ6HX+N2267rUhSVFRUFK+//nqdZRdccEGRpOjatet6HcucOXOKAw88sOjYsWNRUVFRp8733ntvkaSYMGFCg9vs2bNn0bNnz3Ua+9y5c4skxdixY+stmzdvXnHCCScUPXr0KFq0aFF06tSpOOigg4pHHnlkjdsbPXp0kaTYf//96y3r27dvkaT45je/udbxfP3rXy969+5dtGrVqmjbtm3Rr1+/4uijjy5uueWW2n7PPvtscdpppxUDBw4sOnfuXLRs2bLo2bNnceihhxa/+93v6m33kUceKfbZZ5+iXbt2tfW89957116cNfiw+q9YsaL4f//v/xW77LJL0aZNm2LTTTctevfuXRx33HHFnDlz6vW/7bbbii984QvFZpttVmy66abFzjvvXFx99dWNGltDXn755eJrX/ta0atXr6JFixZFx44di1133bX4zne+U6/vI488Uhx88MFF586dixYtWhTdu3cvTjjhhGL+/Pn1+q7tOpwwYcIaazx58uQG3zeSFEOHDi3mz59fHHXUUcXmm29etGrVqhgwYEBxzTXXNHhsq1atKn70ox8VAwcOLFq3bl20bt26+PznP19ceumlxapVq+r1r9lHQ9Z2PPfff39x+OGHF5/5zGeKFi1aFJ07dy523HHH4rTTTiv+8Ic/1Om7tutvTXVZ2/W+Lu6///4iSZGkOOmkk9bY75lnnilGjhxZbL755sWmm25afP7zny9++tOfrvF9YE01Wb16dXHhhRcWW2+9de15csYZZxRvvfXWWo9/2rRpxQEHHFBsvvnmRYsWLYquXbsWu+yyS3H22WcXzz33XG2/66+/vvjnf/7nonfv3kWbNm2Ktm3bFv3798mFA2YAACAASURBVC++9a1vFQsWLFjnugB83CqK4gMz5AAA8IlQUVGRoUOHrnE+GgDY2JlTAgAAACiFUAIAAAAohVACAAAAKIVf3wAA+IQyNRgAn3TulAAAAABKIZQAAAAASvGpenzjtddeK3sIH6pz58554403yh7Gp4Z6Nh21bFrq2bTUs+moZdNSz6alnk1HLZuWejYt9Wxan4R6brnllmtc5k4JAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBRCCQAAAKAUQgkAAACgFEIJAAAAoBSVZQ/gH90ll1yy1uUnn3zyxzSSTwf1bDofVstEPdeHc7NpqWfTca03LfVsWq71pqWeTce13rScm03rk1ZPd0oAAAAApRBKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClEEoAAAAApRBKAAAAAKUQSgAAAAClqNyQle+8887cdtttqa6uTrdu3TJu3Lhst912a+z/7LPPZsqUKZk3b146dOiQUaNGZcSIEbXLb7nlljzyyCN57bXXUllZmT59+uTII49Mjx49NmSYAAAAwEao0XdKPPjgg7n66qtzyCGHZNKkSenXr18uuOCCvPHGGw32X7BgQS688ML069cvkyZNysEHH5zJkyfnoYcequ3z7LPPZsSIEfn2t7+dCRMmpHnz5vn2t7+dN998s7HDBAAAADZSjQ4lbr/99gwdOjTDhw9Pt27dMn78+HTo0CF33XVXg/3vuuuudOjQIePHj0+3bt0yfPjwDB06NNOmTavtc/bZZ2fvvfdOjx490qNHj/zLv/xL/v73v2f27NmNHSYAAACwkWpUKLFy5cq8+OKL2XHHHeu077DDDnn++ecbXGfOnDnZYYcd6rTtuOOOefHFF7Ny5coG13nnnXdSFEU222yzxgwTAAAA2Ig1ak6Jv//971m9enXat29fp72qqipPPfVUg+tUV1fnc5/7XJ229u3bZ9WqVVm6dGk6dOhQb53JkyenV69e6du3b4PbnD59eqZPn54kueiii9K5c+fGHM7HqrKycr3G+Uk4pjKpZ9NZ31om6rk2zs2mpZ5Nx7XetNSzabnWm45zs2mpZ9NyrTetT3o9N2iiy4/SlClT8vzzz+e8885Ls2YN39AxfPjwDB8+vPbvNc1nsTHp3Lnzeo3zk3BMZVLPprO+tUzUc22cm01LPZuOa71pqWfTcq03Hedm01LPpuVab1qfhHpuueWWa1zWqMc32rVrl2bNmmXJkiV12qurq1NVVdXgOlVVVamurq7TtmTJkjRv3jxt27at03711Vfnd7/7Xf7jP/4jXbt2bcwQAQAAgI1co0KJysrKbL311pk1a1ad9qeeeir9+vVrcJ0+ffrUe7Rj1qxZ2XrrrVNZ+b83bEyePLk2kNhqq60aMzwAAADgE6DRv75x4IEHZubMmZkxY0bmzZuXyZMnZ9GiRdl3332TJJdeemkuvfTS2v4jRozIokWLcvXVV2fevHmZMWNGZs6cmZEjR9b2ueKKKzJz5syccsop2WyzzVJdXZ3q6uosW7ZsAw4RAAAA2Bg1ek6JPfbYI0uXLs3NN9+cxYsXp3v37jnrrLOy+eabJ6n/nEqXLl1y1llnZcqUKbU/D3rsscdm0KBBtX1qfk70vPPOq7PuYYcdli996UuNHSoAAACwEdqgiS7322+/7Lfffg0umzhxYr227bffPpMmTVrj9m644YYNGQ4AAADwCdLoxzcAAAAANoRQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAohVACAAAAKIVQAgAAACiFUAIAAAAoRWXZA/i0m3Z99QdaPvg366p+LRP1bDznZtNSz6bjWm9azs2mpZ5NSz2bjlo2LfVsWurZdD6N/05ypwQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUAqhBAAAAFAKoQQAAABQCqEEAAAAUIrKDVn5zjvvzG233Zbq6up069Yt48aNy3bbbbfG/s8++2ymTJmSefPmpUOHDhk1alRGjBixQdsEAAAAPpkafafEgw8+mKuvvjqHHHJIJk2alH79+uWCCy7IG2+80WD/BQsW5MILL0y/fv0yadKkHHzwwZk8eXIeeuihRm8TAAAA+ORqdChx++23Z+jQoRk+fHi6deuW8ePHp0OHDrnrrrsa7H/XXXelQ4cOGT9+fLp165bhw4dn6NChmTZtWqO3CQAAAHxyNSqUWLlyZV588cXsuOOOddp32GGHPP/88w2uM2fOnOywww512nbccce8+OKLWblyZaO2CQAAAHxyNWpOib///e9ZvXp12rdvX6e9qqoqTz31VIPrVFdX53Of+1ydtvbt22fVqlVZunRpiqJY721Onz4906dPT5JcdNFF6dy5c2MOZ41eP2SPtS4fvdd3P3QbvztlSJ2/Kysrs3Llyve1nLfW9Zs9dNyH7mP1oJ9+aJ+NwYbW84O1TJq+np+WWibrX8/1rWXyj1NP1/r6+SRc68kno56u9aa1MVzriXq+n/fO93w813ri30n/y3vnunOtN61Pwr+TPu5abtBEl2UbPnx4hg8fXvv3xjj3xAfH1Llz5/UaZ5dG7OPTqqHjbOp6/qPUMtnwczNRz/dzrTedj+NaX9N+Po1c601LPZuW986m49xsWurZtFzrTeeT+ployy23XOOyRoUS7dq1S7NmzbJkyZI67dXV1amqqmpwnaqqqlRXV9dpW7JkSZo3b562bdsmyXpvEwAAAPjkatScEpWVldl6660za9asOu1PPfVU+vXr1+A6ffr0qfcYxqxZs7L11lunsrKyUdsEAAAAPrka/esbBx54YGbOnJkZM2Zk3rx5mTx5chYtWpR99903SXLppZfm0ksvre0/YsSILFq0KFdffXXmzZuXGTNmZObMmRk5cuQ6bxMAAAD49Gj0nBJ77LFHli5dmptvvjmLFy9O9+7dc9ZZZ2XzzTdPUv85lC5duuSss87KlClTan8e9Nhjj82gQYPWeZsAAADAp8cGTXS53377Zb/99mtw2cSJE+u1bb/99pk0aVKjtwkAAAB8ejT68Q0AAACADSGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASlFZ9gA+yaYetW3ZQ/hUUc+mpZ5NRy2blno2LfVsOmrZtNSzaaln01LPpqOWTesfsZ7ulAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEohlAAAAABKIZQAAAAASiGUAAAAAEpR2ZiViqLIjTfemBkzZuTNN99Mnz598pWvfCXdu3df63oPPfRQrr/++rz++uvp2rVrjjjiiOy6665JkpUrV+a6667LE088kddffz2tW7dO//79c9RRR6Vz586NGSYAAACwEWvUnRJTp07N7bffnmOPPTYXXnhh2rVrl/PPPz/vvPPOGtd54YUX8oMf/CB77rlnvvvd72bPPffM9773vcyZMydJsnz58sydOzejR4/OpEmT8s1vfjMLFy7Md77znaxatapxRwcAAABstNY7lCiKIr/+9a9z8MEHZ9CgQenRo0dOOumkvPPOO3nggQfWuN6vfvWr9O/fP6NHj063bt0yevTo9O/fP7/61a+SJJtuumnOOeec7LHHHtlyyy3Tu3fvHH/88Zk/f37mz5/f+CMEAAAANkrrHUosWLAg1dXV2WGHHWrbWrZsme222y7PP//8Gtd74YUXsuOOO9Zp23HHHfPCCy+scZ233347SdKmTZv1HSYAAACwkVvvOSWqq6uTJFVVVXXa27dvn8WLF691vfbt29dbp2Z7H7Ry5cr8/Oc/z8CBA9OpU6cG+0yfPj3Tp09Pklx00UVNPvfE6x+yvDH7q6ysXL/1/vThXT4pc258Eur5aallsv7Hst61TP5h6rkxnJuNHUcZ1LPpuNab1kZxbibquRau9TVzra879WxarvWm9Umo58ddyw8NJe6///7893//d+3fZ5111kc6oCRZtWpVLrnkkrz11lv55je/ucZ+w4cPz/Dhw2v/fuONNz7ysb1fY/bXuXPn9Vqvy0c0jo3RxlDPT0stk/U/lvWtZfKPU8+N4dxs7Dg2RurZtFzrTefjODcT9Vwb1/qaudablno2Hdd609oY6vlR1HLLLbdc47IPDSV23nnn9OnTp/bvFStWJHnvzof3JyhLliypdyfE+1VVVWXJkiV12pYsWVLvjotVq1blhz/8YV555ZVMnDgxbdu2/bAhAgAAAJ9AHzqnROvWrbPFFlvU/q9bt26pqqrKrFmzavssX748s2fPTr9+/da4nb59+9ZZJ0lmzZqVvn371v69cuXKfP/738/LL7+cCRMm1AssAAAAgE+P9Z7osqKiIvvvv3+mTp2ahx9+OK+88kouu+yybLLJJhkyZEhtv/POOy/XXntt7d/7779/nn766dx6662ZP39+brnlljzzzDM54IADkrx3h0TNT4SecsopqaioSHV1daqrq7N8+fImOFQAAABgY7LeE10myUEHHZTly5fnyiuvzFtvvZXevXvn7LPPTuvWrWv7vP7663UmqOzXr19OPfXUXHfddbn++uuzxRZb5NRTT619NGThwoV59NFHkyRnnnlmnf2deOKJ2WuvvRozVAAAAGAj1ahQoqKiIl/60pfypS99aY19fvSjH9VrGzRoUAYNGtRg/y5duuSGG25ozHAAAACAT6D1fnwDAAAAoCkIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCUAAACAUgglAAAAgFIIJQAAAIBSCCWA/8/enYdlUe//H3+xyCYCIqICrom5pYkLuJWeSIvSzAzNJTOtNLE81fFIy9Hjt0KrY5ZpLqnlLlhqKm2uRyItPZqiEmqauYEgmKKALL8/vJgfyI43DOrzcV1ded/3zNyf+82sr/nMDAAAAACYglACAAAAAACYglACAAAAAACYwrY8I+Xk5CgiIkKbN2/W5cuX5evrq5EjR6p+/frFjrdz506tWrVK8fHxqlOnjp566il16tSp0GHnzZunTZs2aejQoerbt295mgkAAAAAAKqwcvWUWLdunTZs2KARI0YoLCxMLi4uevvtt3X16tUix4mLi9OMGTPUvXt3vffee+revbumT5+uI0eOFBh2586dOnr0qGrWrFme5gEAAAAAgFtAmUOJnJwcRUZGql+/fgoICFCDBg0UEhKiq1evKioqqsjxNm7cqFatWql///7y8fFR//791apVK23cuDHfcOfPn9eiRYv00ksvyda2XB05AAAAAADALaDMoURCQoJSUlLUpk0b4z07Ozu1aNFCv/32W5HjxcXFqW3btvnea9u2reLi4ozXWVlZ+uijj/TEE0/Ix8enrE0DAAAAAAC3kDJ3RUhJSZEkubm55Xvf1dVVycnJxY7n6upaYJzc6UlSeHi4atSooV69epWqLZs2bdKmTZskSVOnTpWHh0epxiut+BI+L8/32dralm28oyUPYunfXVFuhXreLrWUyv5bylxL6Y6pZ1WYN8vbDjNQT8thWbesKjFvStSzGCzrRWNZLz3qaVks65Z1K9SzsmtZYiixY8cOzZs3z3gdGhpaIQ05ePCgtm3bpvfff7/U4wQGBiowMNB4nZiYWBFNK1J5vs/Dw6NM43lWUDuqoqpQz9ulllLZf0tZayndOfWsCvNmedtRFVFPy2JZt5zKmDcl6lkclvWisaxbFvW0HJZ1y6oK9ayIWnp5eRX5WYmhRIcOHeTr62u8vnbtmqTrPR/yJigXL14s0BMiLzc3N128eDHfexcvXjR6XBw8eFApKSl6/vnnjc+zs7O1bNkyRUZGas6cOSU1FQAAAAAA3EJKDCUcHR3l6OhovM7JyZGbm5v279+vpk2bSpIyMjIUGxuroUOHFjmdZs2aaf/+/fke77l//341a9ZMktS7d28FBATkG+edd95R165d8/WGAAAAAAAAt4cy3+jSyspKQUFBWrdunXbt2qWTJ09q9uzZcnBwULdu3YzhpkyZouXLlxuvg4KCFBMTo7Vr1+r06dNas2aNDh48qEceeUTS9ftLNGjQIN9/tra2cnNzK7arBwAAAAAAuDWV65mbjz32mDIyMrRgwQKlpqaqadOmeuONN/L1qIiPj1etWrWM13fffbfGjx+vlStXatWqVapbt67Gjx+f79IQAAAAAABw5yhXKGFlZaXg4GAFBwcXOcysWbMKvBcQEFDgEo3iFDYNAAAAAABweyjz5RsAAAAAAACWQCgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMQSgBAAAAAABMYWt2A1C8hKZhZjfhtkI9LYt6Wg61tCzqaVnU07Kop+VQS8uinpZFPS2HWlpWVasnPSUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApbM1uQGXJyspSWlqaJMnKyqpU42QPeqHYz62vXClzO+Lj45Wenl7m8SwtJydHkuTg4CAbGxuTWwMAAAAAuBPdEaFEVlaWrl69qurVq5c6kJCknLuaFfu5lZNTmdtia2tbZUKAnJwcpaamytHRscq0CQAAAABw57gjLt9IS0srcyBxJ7CyslL16tWNHiQAAAAAAFSmOyKUkEp/ycadhroAAAAAAMxSrss3cnJyFBERoc2bN+vy5cvy9fXVyJEjVb9+/WLH27lzp1atWqX4+HjVqVNHTz31lDp16pRvmDNnzmj58uWKiYlRZmamvL29NW7cOPn4+JSnqZI48C4J9QEAAAAAmKFcPSXWTa/vggAAIABJREFUrVunDRs2aMSIEQoLC5OLi4vefvttXb16tchx4uLiNGPGDHXv3l3vvfeeunfvrunTp+vIkSPGMAkJCXrrrbfk6empf/3rX/rPf/6jgQMHysHBoTzNBAAAAAAAVViZQ4mcnBxFRkaqX79+CggIUIMGDRQSEqKrV68qKiqqyPE2btyoVq1aqX///vLx8VH//v3VqlUrbdy40RhmxYoVatu2rZ5++mk1adJEderUkZ+fnzw8PMr3625x2dnZmjBhglq1aiVvb29FR0eb3SQAAAAAACymzJdvJCQkKCUlRW3atDHes7OzU4sWLfTbb7/pwQcfLHS8uLg4Pfzww/nea9u2rb799ltJ1w/A9+zZo379+umdd97R77//Lk9PT/Xp00ddunQpazNLJeu5vhUy3aLYzP+6TMNv3rxZ4eHhioiIUMOGDeXm5lbs8CkpKXrrrbf0ww8/SJIefPBBvf3223J1dS13mwEAAAAAqChlDiVSUlIkqcABsqurq5KTk4sd78aDY1dXV2N6f/31l9LS0rRmzRoNHDhQQ4YMUUxMjD7++GM5ODjIz8+vwDQ3bdqkTZs2SZKmTp1aZI+K+Ph42doW/KlZxfzOipDbhsLaUpiTJ0+qTp066ty5c6mGHzdunE6fPq0VK1ZIkl599VW9/PLLWrp0abHj2dvbV0pvlPgSPi9PG2xtbe/InjQl1VIqez3v1FpKzJuWRj0th2Xdspg3LYt6Wg7LumVRT8tiWbcs6llQiUfHO3bs0Lx584zXoaGhFdKQ7OxsSVKHDh306KOPSpIaNWqkY8eO6dtvvy00lAgMDFRgYKDxOjExsdBpp6eny8bGpgJaXTaZmZmytbVVZmZmicOOHz9eERERkqQ6derIx8dHO3fu1Ny5c7VkyRKdOXNG7u7uGjBggEJDQ3XkyBFt2bJFa9euVbt27SRdD2oef/xxxcbGqmnTpkV+V3p6epG1q0zlaYOHh0eVaHtVVNa6UMuiMW9aFvW0LJZ1y2HetCzqaVks65ZFPS2HZd2ybtd6enl5FflZiaFEhw4d5Ovra7y+du2apOs9H/KmMRcvXiz2MgE3NzddvHgx33sXL140ely4uLjIxsamwFM27tR7KUyZMkU+Pj5auXKlIiMjZWNjo6lTp2rx4sWaNGmS/P39lZSUpJiYGEnSnj17VL16dXXo0MGYRseOHeXk5KQ9e/YUG0oAAAAAAGCGEkMJR0dHOTo6Gq9zcnLk5uam/fv3Gwe6GRkZio2N1dChQ4ucTrNmzbR//3717fv/7+Owf/9+NWvW7HpDbG1111136cyZM/nGO3v2rGrXrl22X3UbcHFxkbOzs2xsbOTp6anU1FTNnz9fkydP1qBBgyRJjRs3NkKIhIQE1apVK9/jPa2srOTh4aGEhARTfgMAAAAAAMUp89M3rKysFBQUpHXr1mnXrl06efKkZs+eLQcHB3Xr1s0YbsqUKVq+fLnxOigoSDExMVq7dq1Onz6tNWvW6ODBg3rkkUeMYfr27avo6Ght2rRJ586d06ZNmxQdHa3evXvf5M+89cXFxSk9PT1fjQEAAAAAuJWV+UaXkvTYY48pIyNDCxYsUGpqqpo2bao33ngjX4+K+Ph41apVy3h99913a/z48Vq5cqVWrVqlunXravz48fkuDenUqZNeeOEFrVmzRosWLVK9evU0duzYQu8ngfw8PT2VlJSknJwco7dETk6OEhMT5enpaXLrAAAAAAAoqFyhhJWVlYKDgxUcHFzkMLNmzSrwXkBAgAICAoqddo8ePdSjR4/yNOu25uvrK3t7e0VFRalJkyYFPm/fvr1SU1O1e/dudezYUZK0e/duXblyRe3bt6/s5gIAAAAAUKJyhRKofM7Ozho5cqSmTp0qe3t7+fv7Kzk5Wfv379fw4cPl6+urnj17auLEiZo2bZokaeLEiQoMDOQmlwAAAACAKolQ4hYSGhoqV1dXzZgxQ2fPnpWHh4cGDBhgfP7JJ5/orbfe0pAhQyRJvXr10ttvv21WcwEAAAAAKNYdHUrYzP+62M9zThwp9nOrRr7Ffn6zRo8erdGjRxuvra2tFRISopCQkEKHd3Nz08yZMyu0TQAAAAAAWEqZn74BAAAAAABgCYQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSVVh2drYmTJigVq1aydvbW9HR0WY3CQAAAAAAi7E1uwFmemxZ7M1N4Meyjb9uSPMyDb9582aFh4crIiJCDRs2lJubW7HDf/TRR9qyZYsOHjyoq1ev6vTp02X6PgAAAAAAKhM9JaqwEydOyNPTUx07dpSnp6fs7OyKHT4jI0MPP/ywRo0aVUktBAAAAACg/O7onhJV2fjx4xURESFJ8vb2lo+Pj3bu3Km5c+dqyZIlOnPmjNzd3TVgwACFhoZKkv7xj39IkjZs2GBauwEAAAAAKC1CiSpqypQp8vHx0cqVKxUZGSkbGxtNnTpVixcv1qRJk+Tv76+kpCTFxMSY3VQAAAAAAMqFUKKKcnFxkbOzs2xsbOTp6anU1FTNnz9fkydP1qBBgyRJjRs3VocOHUxuKQAAAAAA5cM9JW4RcXFxSk9PV7du3cxuCgAAAAAAFkEoAQAAAAAATEEocYvw9fWVvb29oqKizG4KAAAAAAAWwT0lbhHOzs4aOXKkpk6dKnt7e/n7+ys5OVn79+/X8OHDJUmnT59WcnKyTp06JUnGTTAbN26s6tWrm9Z2AAAAAAAKQyhxCwkNDZWrq6tmzJihs2fPysPDQwMGDDA+f//9943HiEpS7969JUkRERHq0qVLpbcXAAAAAIDi3NGhxLohzYv9POfEkWI/t2rka8HWFDR69GiNHj3aeG1tba2QkBCFhIQUOvyMGTM0Y8aMCm0TAAAAAACWwj0lAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglqrDs7GxNmDBBrVq1kre3t6Kjo81uEgAAAAAAFmNrdgPMtH5VSglD1C7+410ljZ9fn4FuZRp+8+bNCg8PV0REhBo2bCg3t6LH//PPPzVjxgxFR0crISFBnp6e6tu3r8aPHy9HR8cyfS8AAAAAAJXhjg4lqroTJ07I09NTHTt2LHHYo0ePKisrS2FhYWrcuLGOHDmif/7zn0pOTtZ7771XCa0FAAAAAKBsuHyjiho/frwmT56s06dPy9vbW/7+/srJydGcOXPUtWtXNW7cWO3bt1dYWJgkqWfPnpoxY4Z69Oihhg0bKjAwUOPGjdPGjRtN/iUAAAAAABSOnhJV1JQpU+Tj46OVK1cqMjJSNjY2mjp1qhYvXqxJkybJ399fSUlJiomJKXIaly9fLvaSDwAAAAAAzEQoUUW5uLjI2dlZNjY28vT0VGpqqubPn6/Jkydr0KBBkqTGjRurQ4cOhY5/6tQpzZkzR+PGjavMZgMAAAAAUGpcvnGLiIuLU3p6urp161bisOfPn9eQIUN033336fnnn6+E1gEAAAAAUHaEEreZhIQEPfnkk7r77rv18ccfy8rKyuwmAQAAAABQKEKJW4Svr6/s7e0VFRVV5DDx8fEaMGCAfH19NXv2bNnacnUOAAAAAKDq4qj1FuHs7KyRI0dq6tSpsre3l7+/v5KTk7V//34NHz5c586d04ABA1S3bl1NnjxZFy5cMMatVauWbGxsTGw9AAAAAAAFEUrcQkJDQ+Xq6qoZM2bo7Nmz8vDw0IABAyRJ27dv1/Hjx3X8+HF16tQp33g7d+5U/fr1zWgyAAAAAABFuqNDiT4Di39cZs6JI8V+btXI14KtKWj06NEaPXq08dra2lohISEKCQkpMOzAgQM1cODACm0PAAAAAACWxD0lAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglqrDs7GxNmDBBrVq1kre3t6Kjo81uEgAAAAAAFmNrdgPM9PHHH9/kFL4p09AvvfRSmYbfvHmzwsPDFRERoYYNG8rNza3IYbOzs/Xss8/q4MGDSkpKkqurq7p166bXX39d9erVK9P3AgAAAABQGegpUYWdOHFCnp6e6tixozw9PWVnZ1fs8F27dtWcOXP03//+V/PmzdMff/yhUaNGVVJrAQAAAAAomzu6p0RVNn78eEVEREiSvL295ePjo507d2ru3LlasmSJzpw5I3d3dw0YMEChoaGytrbWc889Z4zv4+OjkJAQjRgxQmlpaXJwcDDrpwAAAAAAUChCiSpqypQp8vHx0cqVKxUZGSkbGxtNnTpVixcv1qRJk+Tv76+kpCTFxMQUOn5ycrK++uortWvXjkACAAAAAFAlEUpUUS4uLnJ2dpaNjY08PT2Vmpqq+fPna/LkyRo0aJAkqXHjxurQoUO+8d555x0tWrRIV69elZ+fnxYvXmxG8wEAAAAAKBH3lLhFxMXFKT09Xd26dSt2uDFjxui7777TihUrZGNjo3HjxiknJ6eSWgkAAAAAQOnRU+I24+7uLnd3d911111q2rSpOnbsqJ9//ln+/v5mNw0AAAAAgHzoKXGL8PX1lb29vaKioko9Tm4PifT09IpqFgAAAAAA5UZPiVuEs7OzRo4cqalTp8re3l7+/v5KTk7W/v37NXz4cO3evVsxMTHq2LGjXF1ddeLECb3//vuqX7++OnXqZHbzAQAAAAAogFDiFhIaGipXV1fNmDFDZ8+elYeHhwYMGCBJcnBw0IYNG/T+++/r6tWr8vT0VI8ePfTpp5/y9A0AAAAAQJV0R4cSL730UrGf55w4UuznVo18LdiagkaPHq3Ro0cbr62trRUSEqKQkJACw7Zu3VqrV6+u0PYAAAAAAGBJ3FMCAAAAAACYglACAAAAAACYglACAAAAAACYglACAAAAAACYglACAAAAAACYglACAAAAAACYwionJyfH7EZYypkzZwp9/8qVK3Jycqrk1hTO1tZWmZmZZjcjn6pUn7Ly8PBQYmKi2c24LVBLy6KelkU9LYdaWhb1tCzqaTnU0rKop2VRT8u6Ferp5eVV5Gf0lAAAAAAAAKYglAAAAAAAAKYglKjCsrOzNWHCBLVq1Ure3t6Kjo42u0kAAAAAAFiMrdkNMJPn0dBK/b6EpmFlGn7z5s0KDw9XRESEGjZsKDc3t1KNl5aWpkcffVSHDx9WZGSk2rZtW57mAgAAAABQoegpUYWdOHFCnp6e6tixozw9PWVnZ1eq8f7v//5P9erVq+DWAQAAAABwcwglqqjx48dr8uTJOn36tLy9veXv76+cnBzNmTNHXbt2VePGjdW+fXuFheXvffHdd98pOjpa//rXv0xqOQAAAAAApXNHX75RlU2ZMkU+Pj5auXKlIiMjZWNjo6lTp2rx4sWaNGmS/P39lZSUpJiYGGOcM2fOKDQ0VEuWLJGDg4OJrQcAAAAAoGSEElWUi4uLnJ2dZWNjI09PT6Wmpmr+/PmaPHmyBg0aJElq3LixOnToIEnKysrSuHHj9Pzzz6tVq1b6888/zWw+AAAAAAAl4vKNW0RcXJzS09PVrVu3Qj//+OOPVa1aNb3wwguV3DIAAAAAAMqHnhK3iR9//FG7du1Sw4YN873fp08f9e3bV5988olJLQMAAAAAoHCEErcIX19f2dvbKyoqSk2aNCnw+fTp03XlyhXjdXx8vAYPHqyZM2eqY8eOldlUAAAAAABKhVDiFuHs7KyRI0dq6tSpsre3l7+/v5KTk7V//34NHz5cDRo0yDd89erVJUmNGjWSl5eXGU0GAAAAAKBYhBK3kNDQULm6umrGjBk6e/asPDw8NGDAALObBQAAAABAuVjl5OTkmN0ISzlz5kyh71+5ckVOTk6V3JrC2draKjMz0+xm5FOV6lNWHh4eSkxMNLsZtwVqaVnU07Kop+VQS8uinpZFPS2HWloW9bQs6mlZt0I9i+u9z9M3AAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAACAKe6IUOI2updnhaA+AAAAAAAz3BGhhMSBd1GoCwAAAADALHdEKOHg4KDU1FQOwG+Qk5Oj1NRUOTg4mN0UAAAAAMAdyNbsBlQGGxsbOTo66sqVK5IkKysr09pib2+v9PR0074/V25A4+joKBsbG5NbAwAAAAC4E90RoYR0PZioXr262c2Qh4eHEhMTzW4GAAAAAACmuyMu3wAAAAAAAFUPoQQAAAAAADAFoQQAAAAAADAFoQQAAAAAADAFoQQAAAAAADCFVU7usyEBAAAAAAAqET0lKtnEiRPNbsJthXpaDrW0LOppWdTTcqilZVFPy6KelkMtLYt6Whb1tKxbvZ6EEgAAAAAAwBSEEgAAAAAAwBQ2kydPnmx2I+40TZo0MbsJtxXqaTnU0rKop2VRT8uhlpZFPS2LeloOtbQs6mlZ1NOybuV6cqNLAAAAAABgCi7fAAAAAAAApiCUAAAAAAAApiCUwG1n7Nix+vrrr81uhmnCw8P13HPPKTg4WNu2bStyuMmTJ2vBggWV1zCgAh07dkzBwcFKSEgo03jbtm3TsGHDKqhVFSshIUHBwcE6duxYhX9XeHi4Xn311RKHY70CMwUHB2vnzp1mN8PiDh48qODgYP31118WnW5pl+tbQWnWh5W5zryV7dy5U8HBwWY3AybbtGmTxowZo4EDByo8PLzCv8+2wr/hFvXXX38pPDxce/fuVXJysqpXr6769eurX79+atOmjdnNMwQHB+uVV15RQEBApX/3rFmztH37dgUHB2vAgAHG+wcPHtS///1vffbZZ3JxcSlxOpMnT1b9+vU1cuRIi7QrLCxM9vb2FplWVfX7778rNDRUzZo10//93/8Z7588eVKrV6/Wa6+9pmbNmsnJyanIabz22muysbGpjOZWWbnzsCRZW1urZs2a8vPz01NPPSVnZ2eTW2euouaxW9nYsWPVu3dv9e3b1+ymFGvWrFm6dOlSgWeOHzt2TKGhofrkk0/k6ekpDw8PzZs3TzVq1Cj3d+Xk5Gjr1q3aunWrTp48qezsbHl4eKhVq1Z6+OGH5e3tLUnq27evHn744Zv6XVXVjfUuqv7BwcFG7W8XJR143H///Ro7dmyZppmZmanIyEhFRUXpzJkzqlatmry8vNSjRw/16NFD1apVK3EaCQkJCgkJUVhYmO66664yfX9VlnebI0k1atSQr6+vhg0bZixrhbn77rtvelkvzK2yXJdmPn3yySdLnI4l1pnS9f3WQ4cOFdj/laQPP/xQP/30k3r37m2x/VpLuBW26ZY+Hqhqblz+c/n6+uqdd96xyHdY6rjw8uXLWrBggZ5++mkFBATI0dHRIu0rDqFEEf7zn/8oPT1do0ePVt26dXXx4kUdOnRIly5dMrtpVUq1atW0fv169erVq1QBREXKzMyUra2t6e2oDFu2bFHv3r21fft2nTp1Sj4+PpKkc+fOSZI6duwoKyurQsfNrdOdftCd65577tG4ceOUlZWlU6dO6dNPP1VqaqrGjx9vdtNMVdQ8VpTc+QqVx9raWm5ubuUePycnRzNnztSuXbv0+OOPa9iwYXJ3d1dycrL27Nmj8PBw/f3vf5ckOTg4yMHBochp8fe/Nc2bN8/49549ezR37tx879nZ2ZVpepmZmXrnnXd0/PhxBQcHq0WLFqpevbqOHj2qjRs3ysvLS61atbJY+29FudscSbpw4YKWLl2qDz74QB9++GGhw+cuWzezrBelpOW6qijNfHr58uUSp3Oz68y8atWqpe3bt+uJJ54w9rcuXbqk3bt3q1atWhb5Dksq6zb9VldVt0l5l/9cVbGdiYmJysrKUvv27VWzZs1K+c6qV4UqIDU1VYcPH9abb76pe+65R5JUu3ZtNW3a1BimsDNuNyZ8Y8eO1d/+9jclJSXpxx9/lKOjo4KCgvKN88MPP2jDhg1KTEyUg4ODmjRpookTJxpnsLdu3aqvv/5aCQkJ8vDw0IMPPqigoCBZW1sbZy+mT59utHHWrFkVW5wbtG7dWklJSVq9erWeffbZQoc5dOiQli5dqj/++ENOTk7q2rWrhg4dKltbW82aNUuHDh3SoUOH9N1330mScSaquPGk6/X29vaWvb29tm/fLk9PT4WFhRX42yQmJmrRokU6cOCAJKlNmzYaMWKEsdEIDw/Xrl279J///Mdo87Zt27RgwQItWbLEmMbChQt1+PBhXbt2TR4eHnryySfVtWvXiilsMTIyMhQVFaUpU6YoPT1dW7Zs0dNPP63w8HCtXr1akjRw4EDjt+We9WvevLm+/fZbZWZm6rPPPiswv2ZmZio8PFxRUVFKSUmRu7u7goKCFBQUpOzsbM2dO1cxMTFKSUlRrVq19MADD6hPnz6ytr5+FVju97Rp00br1q1TRkaGOnbsqJEjR1bpnivVqlUzdlJq1aqlLl26GJe9ZGdn66uvvtLmzZt18eJF1atXT4MGDVLHjh0l/f+zeS+//LK+//57HT16VN7e3ho7dqysrKw0b948/fHHH2rUqJHGjRtnnGE9d+6cFi9erCNHjigtLU1eXl4KDg5W+/btjXaVZv1RUYqax3Ll/u6XXnpJmzdvVlxcnIYNG6ZevXqVql43nv3Mm+znDvPKK6/ohx9+0G+//abatWtrxIgR+Xqp7du3T59//rnOnz+vu+66S7169Sr2N02ePFnnz5/X0qVLtXTpUknK1x3xwIED+vzzz5WQkKCmTZtqzJgx+c6I7969WxERETp16pTc3NzUrVs3Pfnkk6buTNxYz8zMTC1evFi7du3SpUuX5Orqqm7dumnIkCGFjv/TTz8pKipKEyZMUIcOHYz3PTw85Ovrq7wP57pxPVnUeuVWFx4ebpzJyj07O2nSpAIH0mWt9aVLl7RgwQLFxsbq0qVLqlOnjvr06aOePXtW7A8qQd4DtOrVqxd474cfftDXX3+txMREeXh46LHHHlNgYGCR09u4caMOHTqkd999N98y7unpqYCAAKWlpUm6vvx+9dVX+vPPPyVJTZs21fDhw40DpZCQEElSaGioJKlly5bKfYL9tm3btH79ep09e1bVq1dX27ZtjeGl62f4pk+frr1798rV1VXBwcG67777imzz0aNHtXLlSh0/flyZmZlq0KCBhg0bpmbNmpVcwHLIu81xc3PTI488omnTpikjI0MpKSmFrlvr16+frwdq7j7KhAkTil1vrVmzRpGRkUpLS5O/v7/q1Kmjbdu2GfuKRS3XxW3H09LS9Nlnn2nXrl1ycHBQUFCQfvvtN9WoUaPMvWpKq6T5VJIRSpw/f17Lly8vdNtx4zozt2fvW2+9pRUrVujkyZPy8fHR888/X+KjFdu1a6dffvlFBw8eVOvWrSVJO3bsUNOmTQucFCppfi/tdq+8Stqmb9++XatWrdJff/2l1q1b69577zU+O3PmjMaPH68PPvhADRo0MN7ftGmTVqxYoblz58rW1lanTp3SkiVLdPjwYdnZ2al169Z65plnjL9TSfNWUccD58+fL9D7uqi/48SJExUREaETJ07otddek5+fn77++mtt2rRJFy5cUN26dfXYY4/lWx+sXr1aW7ZsUUpKSqHrE0vLu/wXZsOGDdq2bZvi4+Pl5OSkdu3aadiwYcZ8f+XKFS1YsEC//vqrrl69qpo1a+rhhx/WI488UqbjwuKOjbZt26bZs2dL+v/r4sroJUgoUYjc5Hj37t1q3rx5mc8U5LVx40YFBwerb9++2rt3rxYtWqTmzZurWbNmOnbsmBYsWKCxY8eqefPmSk1NVUxMjDHupk2bFB4ermeffVZNmjTRyZMnjYX/oYceUlhYmEaNGqUXXnhB7du3Nw4OK5OVlZUGDx6s999/X0FBQapbt26+zy9cuKCwsDB1795dL774ouLj4zVnzhxZW1vr6aef1ogRI3T27Fl5eXlp8ODBkiQXF5cSx8u1Y8cOBQYGasqUKSrs6bbZ2dl67733ZGdnp0mTJkmSFi5cqPfff19hYWFF9ia40WeffaZr165p0qRJcnJy0pkzZ8pbspu2c+dO1a5dWw0aNNB9992nDz/8UIMHD1bfvn1Vq1atAmcQpOvBkJOTk15//fUip/vJJ58oNjZWzzzzjBo3bqzz588rKSlJ0vU6uru76+9//7tcXFx09OhRowvk3/72N2Mahw8flpubm9566y0lJSXpww8/VL169fT4449XTDEsLD4+Xvv27TNCwcjISK1fv17PPfecmjRpoh07duiDDz7QtGnT1KhRI2O88PBwDR8+XHXq1NFnn32mjz76SK6urho0aJBcXV01a9YsLVy40OgOnpaWpnvvvVeDBg2SnZ2doqOj9cEHH+iDDz7I14W3uPVHRSpqHrvxAHzFihUaNmyYxowZIxsbm1LXqzRWrlypoUOHatSoUfryyy81Y8YMzZ49Ww4ODkpMTNT777+vBx54QL1799Yff/yhxYsXFzu91157Tf/4xz/Us2fPAgFGZmam1q5dqzFjxqhatWqaNWuW5s+frzfeeEPS9R3KmTNn6plnnlGLFi2UmJio+fPn69q1a/nWR2b75ptv9Msvv+jll1+Wp6enkpKSil1XRUVFycvLK18gkVdJ68fSrFduNX379tXp06d1+fJl42xWYb3Kylrra9euqUmTJurXr58cHR114MABzZs3Tx4eHsbJj6rm559/1sKFCzV8+HC1adNGv/76qxYsWCA3N7ci55moqCjdc889hV5yYW1tbVxSmJaWpqCgIDVs2FAZGRn68ssvNW3aNH344YeytbXVu+++q9dff12vv/66GjVqZKwQniVoAAAgAElEQVR7fvjhB33++ed66qmn5Ofnp7S0tHz7TdL1g4zBgwdr8ODB2rJliz799FO1bNlSHh4ehbY5LS1N9913n5555hlZWVnp22+/VVhYmD7++GOLXy5xo6tXryo6OloNGjTIt69547o1txdkXiWtt3788UfjhFGLFi20a9curV27tsRekiVtxxcvXqxDhw7pH//4h2rWrKkvv/xShw8fVqdOnSxYmfIrbttRlOXLl2vIkCGqWbOmPv/8c82cOVPTp08vdh1oY2Oj++67T1u3bjVCia1bt+rRRx/V1q1b8w1b0vx+M20vjeK26UeOHNHs2bM1cOBABQQE6ODBg1qxYoUxrpeXl+666y7t2LEjX+i6Y8cOde7cWba2tkpOTtakSZPUs2dPDRs2TFlZWVqxYoXee+89vf3228bxSXHzVlHHA+fPny/171y2bJmefvpp1a1bV46Ojlq5cqV27typkSNHysvLS3FxcZo7d66cnZ3l5+ennTt3av369Xr55ZfVoEEDXbx4UUeOHLmpWt8sKysrPfPMM/L09DROiC5cuNDYHq1cuVInT57UxIkT5erqqoSEBONeM6U9Lizp2KhLly5yc3PTu+++q3fffVceHh6V0gudUKIQNjY2evHFFzV37lxt3rxZjRo10t13363OnTvL19e3TNNq06aNHnroIUnSww8/rG+++UYHDhxQs2bNlJiYKHt7e3Xo0EGOjo6qXbt2vh33L7/8UkOHDjWuC/L09FR8fLy+++47PfTQQ8YMUr169Qrp1ldafn5+at68uVauXFmgy/t3332nmjVratSoUbK2tpaPj4+GDBmiefPmaeDAgXJycpKtra3s7e3z/YaSxstN7D09PYs9KIiJidEff/yhmTNnGgnfSy+9pJdeekkHDhwodQKdmJgof39/4+9j5jXFW7ZsUffu3SVdP3tkb2+v3bt3KyAgoMgzCNWqVTN2XApz9uxZRUdH6/XXXzcS8jp16hif29raGr0vpOu///jx4/rxxx/zhRJOTk56/vnnjb9ZQECAYmJiqnQosW/fPg0bNkzZ2dm6du2aJBnz1Pr169WnTx9169ZN0vUeKIcPH9bXX3+tl156yZjGo48+Kj8/P+Pf06ZN08CBA40dlYceeijfzf8aNWqUb1nv37+/9uzZo507d+qJJ54w3i9u/VGRipvH8nrooYfyvVfaepXGI488Yhz4DB48WP/973914sQJNW/eXN9//708PDw0YsQIWVlZydvbW2fPntWqVauKnJ6zs7Osra3l4OBQYPnIysoydlokqU+fPvr000+Vk5MjKysrrVmzJt9Z7bp162rIkCGaOXOmhg0bVupwsyxy58u8Cgte8zp//rzq1aunFi1ayMrKSh4eHrr77ruLHD53BzCvpUuXGmepJBm9xQpT0nrlVuTg4CA7O7tCz2bl7VlT1lq7u7vn6+VUp04dxcTE6Mcff6yyocT69evVvXt3Yx3k5eWl33//XevWrSsylDh79qxatmxZ4rRvXJe8+OKLGj58uI4eParmzZsb+zc1atTI93f48ssvFRQUpEcffdR478Yz2vfdd59xJnTgwIGKjIzUoUOHiuwtkbuezvXss89q165d2rt3b7E9LMor77Kdnp6uWrVqGT1Cct24bi0slChpvRUZGan7779fDzzwgCTp8ccf18GDB3X27Nli21fcdjwtLU1bt25VSEiIsf80evRojRkzpvwFsbDith1Fybu9fuKJJ/Svf/1LFy5cKPEyjJ49eyo0NFRXrlzR2bNnlZCQoICAgAKhREnz+820vTSK26ZHRkaqdevW6t+/v6Try/mxY8e0ZcsWY/zu3btrw4YNGjx4sKysrJSYmKjY2FgjPPj+++/VsGFDDR061BgnJCREzz77rH7//Xejp3lx81ZRxwNl8eSTT6pt27aSrgdBGzZs0JtvvqkWLVpIur7vevToUX333Xfy8/NTYmKi3Nzc1KZNG9na2srDw6PC72FT2La9d+/eRu0eeeQR431PT08NHTpU7733nsaOHStra2udP39ejRs3Nmpau3ZtY/jSHheW5tgoN5B1cXGptGNMQokiBAQEyM/PT7GxsYqLi9O+ffu0YcMGDRo0yFhwS6Nhw4b5XtesWVMXL16UdP2Ao3bt2goJCVHbtm3Vpk0b+fv7y9HRUX/99ZeSkpI0b948zZ8/3xg/Ozu7xB1TMwwZMkRvvPFGga7lp0+flq+vb760rnnz5srMzNS5c+cK1Kes45XUve7UqVNyd3fPFyLUqVNHNWvW1KlTp0odSgQFBWn+/Pnat2+f7rnnHnXq1KnE764I586dU2xsrHGAZ2VlpW7dumnLli3F3tSmQYMGxR44HD9+XFZWVsVe6/v9999ry5YtOn/+vDIyMpSVlZVvZShJPj4++f5m7u7uOnr0aGl/nilatGihF154QRkZGdq0aZPi4+MVFBSkK1euKDk5ucCBRvPmzbV379587+Wdj11dXSUpXzdHV1dXpaenKz09Xfb29kpLS9Pq1au1Z88epaSkKDMzU9euXcs3zo3TlfKvPypKWeaxvBvvstSrNPL+9tzrGXN/e+76IW8YcDNBTe6N+PJ+X2ZmplJTU+Xs7Kzff/9dR48e1bp164xhcnJyjO7WFXG9Ze58mdfJkyf1wQcfFDlOjx499Pbbb+vll19WmzZt5Ofnp3vvvbdMvej69u2rwMBA7du3TwsXLix22JLWK7ezstY6Oztba9euVXR0tC5cuKBr164pMzOzSt9f4dSpUwUuL2nevLl2795d5Dil3T85d+6cVq1apaNHj+qvv/4y9m0SExOLHOfixYu6cOFCiSFO3vWojY2NXFxcin1qxcWLF7Vq1SodPHhQKSkpys7OVkZGRrFtuRl5l+3Lly/r+++/1zvvvJPvRnelOTAqab115swZI5DI1bRp0xJDieK24+fOnVNWVla+y5kdHBxUv379EttbWYrbdpRmHHd3d2OckkIJHx8fNWzYUD/++KNOnDihrl27Fnq5amnn9/K0vSQlbdNPnz6d79JR6fr2NG8o0bVrV+PSjJYtWyoqKkqenp7G9v7333/X4cOHC32S1blz54z5paL3EfMuN6dOndK1a9f07rvv5hsm775rbiiTexx27733qkOHDhW6XSts2573pvQxMTFas2aNTp8+rStXrig7O1uZmZnGZdW9evXS9OnTdfz4cd1zzz3q0KFDqYLgvCx1bGRphBLFsLOzU5s2bdSmTRsNGDBAc+bMUUREhPr27VvombGsrKwC7934dAMrKytjo+3o6Khp06bp8OHD2r9/v9auXasVK1YoLCzMWGife+65Ys++VBVNmzaVv7+/li5dmu9Mb3HKe3Yx73g3c6+C3OlYW1sX2JHKzMzM9/pvf/ub2rZtq71792r//v1688031a9fv0p/ZNLmzZuVnZ2tF1980Xgvt+3F7UDd7D0doqOj9cUXXxjX2To5Oenbb7/VL7/8km+4wp7mURVDtLzs7e2Ny46effZZ/fvf/9bq1avznYkrSd7fnTtfFfZebi2WLFlipOX16tWTvb29PvnkkwLzXXHrj4pS0jyWtwt0Weer3PVa3t9w42/OVVz9LO3GA8nc78vOzjb+P2DAAHXu3LnAuBXVpTHvfJkrNTW12HGaNGmiWbNm6ddff9WBAwc0a9YsNWzYUG+++WahB8v16tXT6dOn873n4uJS6jMjVfleMRWtrLX++uuvtX79eo0YMUINGjSQg4ODli9fbvFHPFaG4rbdXl5eBeapwkybNk3u7u567rnn5O7uLhsbG73yyitFrg/K4sbLzKysrIxluTCzZs3SxYsXNXz4cNWuXVvVqlXTlClTLNKWwty4bDdp0kTDhw/Xpk2bjJ6HpVm2SlpvldetuB3Pqzzbjpv5zT179tT333+v+Ph449KZG5V2fq+I7V559xvzcnV1VZs2bRQVFWWEErk9InOn165du0J7LueeqJHKV+fSHm9J+Zeb3On+85//LHDpVm47PDw8NGPGDMXExGj//v1avHixVq9erXfeeafCbgBb2LY91/nz5xUWFqYHHnhAAwcOlLOzs44fP66PPvrImFfatWunWbNmad++fTpw4IDCwsLUuXPnfH/fm1ERPT9Li1CiDHx8fIwE3cXFRcnJycZnGRkZOn36dJmvm7axsVHr1q3VunVrBQcHa9SoUfrf//6nwMBA1axZU/Hx8br//vuLHf9mN0CWMnjwYP3973/Xvn37jPe8vb31008/KTs729iAxsbGytbW1rg8wNbWtsBvKM14peHj46MLFy4oISHBSATj4+OVnJxs3GDIxcVFFy9eNLo8StKJEycKTKtWrVoKDAxUYGCg1q5dq2+++aZSQ4msrCxt375dgwcPNi4VyPXJJ59o27Zt5b6bcqNGjZSTk6ODBw/mu8FRrtjYWDVt2tToxitdr+PtaMCAAXr33XeNZfC3337Ld2YuNjb2pu9aHRsbq/vvv9/oeZCRkaH4+HjVq1fvpqZ7s0ozj934+LNcTk5OJdYr9wA+JSXF+LywZa0k3t7e2rVrV75ltjTXgRa2rimNJk2a6PTp00XuSFQljo6OCggIUEBAgHr06KE33nhD586dK3CZhnT97NdHH32kXbt2yd/f34TWVk2lnU/KUuvY2Fi1b9/euBwgJyfHuFFjVeXj46PY2Nh8l+iVtP7r2rWrVqxYoWPHjhU425+dna20tDRlZWXp9OnTGjlypNFl/vfff893oJEbLOT9O7i6usrd3b1Ml16WRmxsrEaMGGGs81JSUvLt31UGa2trZWRkWHSaud3w8/79jh07dlPTrFu3rmxsbHTs2DFjXyw9PV1//vlnmfbNbiddunTR559/Lk9Pz0Iv8b506VKJ83tFKc023dvbu8D2My4ursC0unfvrgULFigwMFAnT57UK6+8YnzWuHFj/fTTT/Lw8Lipmz8Xtu7N3W9ITk42/l2a/QYfHx9Vq1ZN58+fL3CJVl52dnby8/OTn5+f+vXrp+eff16//fabcRlIZTp27JgyMzP1zDPPGMc+//vf/woM5+LiYlym1q5dO3300Ud67rnnVK1atVIdF5bm2MgMhBKFuHTpkqZPn66ePXuqYcOGcnR01LFjx7Ru3Tq1bt1aTk5Oat26tbZu3aoOHTrIxcVFX331VZlXMHv27FF8fLxatGghZ2dnHTx4UFevXjVudBccHKyFCxfKyclJfn5+yszM1PHjx3XhwgXjGn1PT08dOHBALVu2NP0xj3Xr1lVgYKAiIyON93r37q3IyEh99tlnCgoKUkJCgpYtW6aHHnrISDRr166to0ePKiEhQQ4ODnJ2di7VeKVxzz33qGHDhsZN6qTrN3Np3LixsZJq2bKlLl++rDVr1qhLly46dOiQdu3alW86ixYtUrt27VSvXj1dvXpVv/76a6UvuP/73/906dIlPfDAAwVuvtWlSxf98MMP+a7nKwsvLy917txZc+bM0TPPPKMmTZooKSlJ58+f13333ad69epp27Zt2rt3r+rWrasff/xRhw4dui0fK9qqVSv5+Pjoq6++Ut++fRUeHq66desaN248fPiwpk2bdlPfUa9ePf3888/q0KGDbG1tFRERYfEd0vIozTxWXE+okuplZ2cnX19frVu3TnXq1NGVK1e0fPnyMrezV69e2rBhgz7//HP17t1bJ0+e1A8//FDieLVr11ZsbKwuXLhQpscHP/HEE5o2bZpq166tzp07y8bGRn/++aeOHj1a7mWuImzYsEFubm7GjQGjoqLk6OhYZBfkLl266JdfftHHH3+sxx57TPfee6/c3NyUlJSk//73v6aeMTFT7dq1tW/fPp05c0bOzs7Gtc55lbXWXl5eio6OVmxsrGrUqKFvvvlGCQkJaty4cWX8pHLp06ePPvzwQzVp0kRt27bVvn37FBUVpVdffbXIcR555BHt3btXb7/9tp588km1bNlSTk5OOn78uNavX6+nnnpKLVq0UI0aNbR582Z5eHjowoULWrJkSb6zqK6urrKzs9Ovv/6q2rVry87OTk5OTurfv7+++OILubq6ys/PTxkZGTpw4ID69OlT7t9Zr1497dixQ76+vkpLS9OyZcsq9Kk6165dM4LZy5cv69tvv1VaWlqBLvQ3KygoSLNnz9Zdd92lFi1a6Oeff9aRI0duKghzcHBQz549tWzZMtWoUcO40WV2dvYdu75wdHTU3Llzi7x0q3r16iXO7xWlNNv0l19+WW+99ZbWrFlj3Ojyxl6w0vVHzc+bN0+ffvqp7rrrrnzha+/evbV582bNmDFDjz32mFxcXBQfH6+ffvpJTz/9tBwdHUvV3sKOB+rWratatWopIiJCgwcP1vnz5/XVV1+VOC1HR0f16dNHS5YsUU5Ojlq2bKm0tDTFxcXJ2tpagYGB2rZtm7KysuTr6ysHBwdFR0fLxsamQk8Q5V3+c1lbW8vFxUX16tVTTk6ONm7cKH9/f8XFxWnjxo35hl21apUaN26s+vXrKysrS7t27ZKnp6dxyUlpjgtLc2xkBkKJQjg4OMjX11fffPONzp07p2vXrsnd3V3dunUzdsj79eunhIQEvffee3JwcFD//v3LnKxXr15dv/zyi1avXq309HTVrVtXo0ePNm7I8sADD8je3l7r16/XihUrZGdnJx8fn3xnq4cNG6bFixdrzJgxcnd3r/RHgt5owIAB2r59u3HDQHd3d4WGhmrp0qWaMGGCqlevrq5du+qpp54yxunTp49mzZqlV155RRkZGcZjZ0oarzSsrKw0YcIELVy4UP/+978lXV8Yn332WWMD6uPjo1GjRmnNmjVas2aN2rdvr8cffzzf3YdzcnK0cOFCJSUlycHBQffcc0+l33V/y5YtatWqVaF3A+/cubOWL19+U938QkJCtGrVKi1atEiXLl1SrVq1jBvuPPjggzpx4oQ+/vhj5eTkyN/fX3369ClwM6fbRZ8+fTR79mx99NFHunr1qpYtW6aUlBR5eXnp1VdfLXOPqBsNHz5cc+bM0aRJk1S9enUFBQUZy4yZSjOP7d+/v8gN9sMPP1xivcaMGaO5c+cqNDRUderU0ahRo4y7P5eWh4eHXnvtNX3xxRfatGmTmjRposGDB2vmzJnFjhccHKz58+dr3LhxunbtWr4bFxbn3nvv1cSJE/Xll19q/fr1xk5Ljx49ytTuiubg4GA8KtHKykqNGjXS66+/XmSQa2VlpZdfflmbN2/W1q1btWHDBmN717p165sO325VgYGBOnTokCZOnKi0tLRCHwla1lr3799fCQkJevfdd2VnZ6cePXqoe/fuOnXqVGX8pHLp1KmTRowYofXr1+uLL76Qh4eHRo4cWeRNLqXr9zl48803tXHjRm3dulXLli2TnZ2dvLy81KNHD919992ytrbW3//+dy1atEivvvqq6tatq2HDhuV7LLeNjY1GjBih1atXKyIiQi1atNDkyZPVq1cv2draav369Vq2bJmcnZ3Vrl27m/qdY8aM0bx58/TPf/5T7u7uevLJJyv0spoDBw7o+eef/3/t3XlYVdXi8PEvM5ogEDKDqIA445CamFylxOEtu4mzlOZwVdSbaSXXvIITiSVWTpiFJRKoaY7liKIYDiUOiAxXkUEZFEFQGQ6c9w+es38cDygoeA62Ps/j83j22XuftRdrr7X22msAKh+cbGxsmDNnDh06dCAnJ6fefsfd3Z3s7GzCw8MpKSmhV69evPXWW0+cE6Q23n//fb777jup/jt06FAKCgr+tvPLgPKcAI+rTXqvLV9fX9q3b1/rpVdrU6Y/evSIadOmScvKd+jQgREjRqjMKWRgYEDPnj2Jjo6WHmQVzMzMWLJkCeHh4SxfvpzS0lLMzc3p0qVLndJFTc8DH330EZs2beKTTz7B0dGRMWPG8MUXXzz1fKNGjaJ58+bs3buXTZs20aRJExwdHRk2bBhQ+XfbvXs3W7Zsoby8HDs7O+bNm9egk9lXvf8VzMzM2LBhAy1btmTChAns3r2biIgI2rZti4+PD6tXr5b21dPTIyIigpycHPT09HBxceGzzz6Tvq/Nc2Ftno3UQUvemAaKCYIgCIIgCIJQZytXrqS8vFxanro+lJWVMWPGDN55553n6rEiPFlJSQkffvgh06dPV5rPQRBeFqKnhCAIgiAIgiC8REpKSjh06BBubm7o6OgQGxvL+fPnnzj8pjZu3LhBZmYmTk5OPHr0iN27d1NcXEyfPn3qKeRCdeLj43F2dhYNEsJLS/SUEARBEARBEISXSGlpKStWrODGjRuUlpZibW3NsGHDnvuh9saNG4SEhHDr1i10dHRwdHTEx8dHLcukC4Lw8hCNEoIgCIIgCIIgCIIgqEX1U8UKgiAIgiAIgiAIgiA0MNEo0QBGjhxJbGzsc51j7ty5tZ4ZvrEoKipiypQpZGVlqTsoKlatWsXevXvVHYyXXn3cG5pk7dq1SjNAP/65Ol988YXaV8lpbI4fP46Pj88T99mzZ89TZyR/2dJfTXx9fdmzZ88T9/Hx8eH48eMvJkAvudjYWEaOHFkv58rMzGTBggWMGzeu1jPsQ+NJ2zk5OYwcOZL//e9/z7VPdbZt28aUKVMYOXKkSNvPqTGkp2dNJ7V1/PhxaWUCTVNQUMCkSZO4e/fuC//ta9euMW/ePMaMGYO/v3+D/U5D5hUvi5etHBcTXdbR2rVrOXHihPTZyMgIZ2dnfHx8sLW1VWPINN+uXbvo2rUrVlZWAISGhpKYmEh6ejomJibVPqidPn2aXbt2cfv2bYyNjRk0aBDvvPNOtee/du0a/v7+2Nraqiy19PDhQyIiIjhz5oy03OWYMWOkiZm8vb1ZtGgRnp6eT1zaqT7k5eWxfft2Lly4QEFBAcbGxnTt2pURI0bUuMa9pvP19SU3N7fG79u3b9+ghVddrV27lsLCQpUZyP/3v//h5+cnLUNVVxMnTnyuZVkbi8OHD/PTTz8RGhqKrm5lMSKTyZgwYQKWlpZK919WVhazZ89m4cKFdOrUSV1BBmDjxo288sorag3Ds3i83FEICgqqdnnawMDAGpemfFk01jT4NBERERgYGBAcHIyhoaG6g1MnT2uY8fDwYMSIEU89j7m5ORs3bqx2GcOapKWlsWPHDubNm4eLi0uDl+PPyt/fH3t7eyZNmqS0/fjx43z//fds2bLlmc+dk5PDzJkzpc+6urq0aNECT0/PGutNmqq+0tKzkslkREREMHv2bGlbeno627Zt48aNG+Tk5ODt7a0SzkePHhEZGcnZs2cpKCigVatWTJgwAScnJ2mfiIgIYmNjuXv3Lrq6urRq1YpRo0bRtm1blXDI5XICAwOJi4vj448/pnfv3gA0b94cDw8Ptm3bxvTp0xsoFqq3efNmWrZsyfz582vMo6qWWdra2piamtKtWzfGjBlDs2bN6i0sz5JXaKqaynlnZ2eWLVumhhC9eKJR4hl06tSJWbNmAZUPmGFhYXz55ZcEBwerOWSaq6SkhGPHjimtpSuXy/Hw8CAtLY1Lly6pHHPhwgW++eYbJk6ciJubG5mZmYSEhKCvr8+gQYOU9i0qKmLNmjV06tSJvLw8pe9kMhlLly6lWbNmzJkzBzMzM/Ly8qSKLICDgwOWlpZER0ernLs+5eTk8Pnnn2NhYYGvry/W1tZkZWURERGBn58fS5cubdD1kRtKYGAgFRUVAKSmprJ8+XKWL1+Oubk5gFJcv8w0tSJc3zp06EBJSQkpKSm4uroCkJycTNOmTbl9+zb379/H2NgYgCtXrqCnp1dthas2ZDJZvYXbxMSk3s71olUtdxQer4jJZDJ0dXWluH+Zvcg0+CIo/nZZWVm89tprjbIc2Lhxo/T/P//8k5CQEKVt+vr6FBUVPfU82tradb5XFT0wX3vtNbS0tOp07MvmP//5D46OjpSVlXHlyhU2btyIubl5o1odo77S0rOKjY1FX1+f9u3bS9tKSkpo0aIFvXr1IiIiotrjNmzYQFpaGr6+vrz66qtER0ezZMkSgoODMTMzA8DGxoZJkyZhYWFBaWkp+/fvZ/ny5Xz99dcq6X7v3r01pud//OMf+Pn54ePjU68P+k+TlZWFl5eXVL+riaLMKi8vJyMjg/Xr1/PgwQM++uijegvLs+QVmqy6cv7vUn8G0SjxTPT09KSbwMTEhKFDh7JixQpKS0vR19dX2vfzzz/HxcWF999/X9r28OFDpkyZwuzZs+nVqxcFBQWEhIRw8eJFmjdvjre39wu9nhfhwoULAEqVwg8//BCo7HZdXaNEdHQ03bt3x8vLCwBLS0veffdddu/ejZeXl1JGvWHDBjw8PJDL5Zw5c0bpPMePH+f+/fssXrxYurmrq/D16NGDmJiYBm2U+P7779HS0mLhwoXSm0xzc3MWLlzI7Nmz+f777/Hz8wOqf6Py+Bt+uVzOnj17OHLkCHl5eVhZWTFs2DD69esnHZOXl8dPP/3ExYsXAXBxcWHChAlYW1sDlV1ez5w5w3vvvUdERAQFBQV07NiRadOm1frhpup+igclY2PjaguLoqIiVq1axYULF2jevDkjR46sU3hfpKtXrxIWFsbNmzdp2rQp7u7ujB8/vsZC4vG/T0lJCZs2bSI2NhZDQ0MGDx6sckx0dDS//fYbmZmZUiVowoQJmJmZIZfLmT17Nm+99ZbSm67bt2/z73//my+++EItM57b2NhgamrKlStXpAfC+Ph4OnbsSG5uLvHx8bz++uvSdhcXF/T19SkrK2Pr1q3ExMTw8OFDadb2qucICAhg/vz5bN++ndTUVObNm1dtGHbv3s2+ffsoLi6mV69etXqIGzlypPS2SfFW8eOPP+bw4cMkJibSokULJk6cSOfOnesppupP1XJHQdEzzMDAgBMnTmBhYUFgYCC+vr54eXlJaSYrK4sNGzaQnJyMubm5UnmksHXrVs6ePTlWV3wAACAASURBVMudO3cwMTHh9ddfZ+TIkejr65OTk8OsWbNYvnw5bdq0kY45cuQIP//8MyEhIS+84vQsaVBLS4vNmzc/Nf0tXLiQn3/+mbS0NOzs7Jg6darSfXbixAkiIyO5f/8+HTt2xM3NTSV858+fZ/v27WRkZGBiYkLfvn0ZMWKEFE++vr54eHhw584dzp49S+fOnaXu8jdv3mTHjh14e3vzj3/8g5kzZxIYGKgU91XTsqaomj4VPZIeT7OKB8nc3FzCw8Orve8U92bVa87IyGDLli0kJCSgr69Px44dmTBhAiYmJmzbto0dO3YAMGrUKIBGP/xVUZY4Ozvz+++/U1xczOuvv87kyZNV6pmPMzIykuK9f//+HDx4kOvXr0uNEikpKURERHDjxg1kMhkODg74+Pjg4uKidJ78/HwCAwOJj4/H2NiY0aNHS2V1QEAAdnZ2SvWThw8fMnXqVGbNmkWvXr2e6/rrKy3Bk9NOTU6dOkW3bt2Utjk5OUk9Hnbt2qVyTGlpKWfOnGHu3Ll06NABqLxP//zzTw4dOsTo0aMBlOo7AO+//z7Hjh0jNTVVKS9JSUnht99+44svvmDKlCkqv+fg4ICpqSlnzpzB09OzxmupiyeV0VV74qxfv57169czY8YM/vGPf1R7rqpl1quvvkqfPn2UhhpUVFSwc+dOjh49SkFBAdbW1owePZrXXntN6Ty3b99m8+bNXL9+Xfr7dunSBVDNK2qbh2uq6sp5hdqU4y+ivt+QxJwSz+nRo0ecPn0aBweHaguKN954g5iYGOktMsCZM2fQ19eXMrx169aRlZXFwoUL+eSTT4iOjiYnJ+eFXcOLkJCQQOvWrev0BqOsrAw9PT2lbfr6+ty9e1dpqMDBgwcpKChg+PDh1Z7n3LlztG3blh9++IEpU6YwZ84ctm3bpvIG1snJiZSUFEpLS+twZbVXVFREXFwcXl5eKl2rDQwM8PLyIi4urk6t/xERERw7doxJkyYRHBzMP//5T7777jv++usvoPKhOCAgAD09Pfz9/Vm6dCmmpqYsWbKEkpIS6Tw5OTmcPn2aefPm8fnnn5Oamlrjm4DntWPHDnr06MHKlSvp06cP69ev586dO3UK74uQl5dHYGAgjo6OrFixgmnTphETE0N4eHitz7FlyxYuXbrE3LlzWbhwIampqSQkJCjtI5PJGDFiBCtXrmT+/PkUFhby9ddfA6ClpcWAAQNUxgxGRUXh6Oio1kK2Q4cOxMfHS5/j4+Pp0KED7du3V9p+9epVqYIWFhbG6dOnmT59OitWrMDe3p5ly5Zx7949pXNv3bqV0aNHs3r1apydnVV++/Tp00RERDBy5EhWrFiBjY0N+/fvf6briIiIYPDgwaxcuZI2bdqwevVqiouLn+lc6nDy5EkAFi9eXO0cBBUVFaxcuRK5XM7SpUuZPn0627dvV8n/DAwMmD59OsHBwUyaNImYmBh27twJVDbidu7cmaioKKVjoqKieOONN9T2JqeuabC26S88PJyxY8eyYsUKjIyM+Pbbb6VhWcnJyaxbt44333yToKAgunfvrvIAHBcXx7fffsugQYP46quvmD59OrGxsSp5x/79+7G1teWLL75gzJgxbNy4ERsbG/7f//t/bNy4sdF1ua+Lutx39+7dY9GiRdjb27N8+XIWLlxIcXExQUFBVFRU8M477/Cvf/0LqHzDXvWNemN29epVbt68ycKFC5k7dy4XL14kLCys1sfL5XKuXbtGZmamUj5aXFxMv379CAgIYPny5Tg6OhIYGEhhYaHS8du2baNHjx4EBQXx5ptvsnbtWmnsvqenJ6dOnaKsrEzaPyYmBkNDQ7p37/6cV143T0pLT0s7Nbl27ZpSI2BtlJeXU1FRUW299dq1a9UeI5PJOHLkCE2aNFEaivfo0SO++eYbpk6dSvPmzWv8TScnJ65evVqncD7Jk/JIxVAJAwMDJkyYwMaNG2vd+yY7O5u4uDh0dHSkbQcOHGDv3r2MGzeOL7/8kp49e/Lll1+SmpqqEqbBgwcTFBRE586dCQoKUukR/bgn5eGNUW3KcU2s79eVaJR4BnFxcfj4+ODj48MHH3zA1atXlcadVdWnTx/u37+vVEE6deoUvXv3Rk9Pj1u3bnHhwgWmTp2Kq6srrVq1wtfXt8EejNUlNzcXU1PTOh3j5ubG+fPnuXjxIhUVFdy6dYt9+/YBlS348H/jSGfNmoW2dvXJOTs7m9jYWGQyGX5+fowaNYrDhw+rVBBNTU0pLy9/amb3rG7fvo1cLsfOzq7a7+3s7JDL5bWeCLS4uJh9+/Yxbdo03NzcsLCwoG/fvnh6enLw4EGgspIgl8uZMWMGLVu2xNbWlqlTp1JcXMyff/4pnauiogJfX19atmyJi4sLb775JpcvX37+i65Gv3796NevH1ZWVowaNQodHR2pUK1teOtD1ftY8W/RokXS9wcPHsTU1JTJkydjZ2dH9+7dGTduHL///nutGkiKi4s5duwY48ePx83NDQcHB2bMmKHSMDdgwAC6deuGpaUlTk5OTJ48mYSEBGkCq/79+3P79m2SkpKAyr/ViRMnGDBgQD3GRt117NiRpKQkysrKKC0tJSkpSeWBMDMzk3v37tGxY0eKi4s5dOgQ48aNo1u3btLbCxMTEym9KowYMYIuXbpgaWlZbev9gQMH8PDw4K233sLGxob33ntPacxuXQwdOpQePXpgbW3N2LFjKSoqUqkUaYLH0+vy5cuBygaD999/H1tb22rzlsuXL5ORkcGsWbNo1aoVrq6uTJgwgfLycqX9vL29cXV1xcLCgm7duvHPf/6TmJgY6XtPT09iYmKksikjI4Pk5GS1psO6pMH27dvXOv2NGjWKjh07Ymtry/Dhw8nMzJTKhQMHDtCxY0fee+89bGxseOutt+jZs6fS8bt27eLtt9+mf//+WFlZ0bFjR8aNG8fhw4eVKsbt2rVj2LBhWFlZYW1tjYmJCTo6OhgaGmJiYtLo5pSoi7rcd4cOHaJly5aMHz8eOzs7WrZsycyZM0lJSeH69esYGhoqvU1/Wbpza2trM2PGDBwcHHBzc2PcuHEcOXLkqY2mixYtwsfHh7Fjx/Lf//4XT09PpZ4LHTt2pF+/ftjZ2WFra8uHH36Inp6e1KNVoWfPnkp5bMeOHaXG3169eqGtrc3Zs2el/aOioujXr98Lb6R8Ulp6WtqpzoMHD3j48GGd66xNmjTBxcWFnTt3kpeXR0VFBdHR0SQlJak0fP7555/4+Pgwbtw49u/fz8KFC5XS7XfffYebmxtdu3Z94m+ampo+cT6vunhaGV11qETTpk0xMTF5Yq8dRZk1btw4Zs2aRUZGBsOGDZO+37t3L2+//TZ9+/bFxsaGUaNG0a5dO5VJmgcOHEifPn2wtbVlwoQJmJubc+jQoSdey5PycE1WXb00LCysVuW4Jtb360oM33gG7dq1k1rli4qKOHToEMuWLWPZsmUqY6yMjIxwc3Pj5MmT0nwHV65ckR5+MjMz0dLSUqpQt2jRQhp79rIoKyt7apfDx3l6epKVlUVQUBDl5eU0adKEIUOGsH37drS0tCgrKyM4OBgfH58ndt2Wy+UYGxszbdo0tLW1ad26NUVFRfz444/4+PhID4mK8Km7Qai2BXpGRgZlZWXSw4lCeXk5LVq0AOD69evk5OSodPMqLS0lOztb+mxubq40H4KpqSn3799/1kt4IgcHB+n/Ojo6GBsbS79V2/DWh6r3sUJaWhpffvklgPR2qWpjl6urKzKZjKysLFq2bPnE82dlZSGTyZS6xBoaGipdP1Re844dO0hNTaWoqEh6aLlz5w6vvvoqJiYmdOvWjaioKFxcXKTeNH379n2u639eHTt2pKysjKSkJOkes7KywsTEhKysLPLz84mPj8fAwAAnJycyMzMpLy9XGsKlra2Ns7MzGRkZSud+2huqzMxMlYdhZ2fnZ1rZp+rfUVEJLSgoqPN5Gtrj6VVfX59vvvnmqb1lMjMzMTMzUyqbnJycVBrHYmNj2b9/P1lZWRQXF1NRUaH0JrFHjx58//33nD17lr59+xIVFYWTk5NKen6R6pIGmzZtWuv0VzVNKMrigoICXn31VTIzM1XeBLu4uHDs2DHp8/Xr10lJSWH37t3SNrlcTmlpKfn5+VI6q+ub2JdJXe6769evk5CQUO0KPFlZWc/cIKnpWrZsqdQw5eLigkwmIzs7+4nlz+zZs3FwcEAmk5Gens4PP/yAoaGhNHygoKCAyMhI4uPjyc/Pp6KigtLSUqnHYtXfq8rZ2VlquNDT0+ONN94gKioKd3d30tPTSUlJYcaMGfV1+bX2pLT0LGlHUQesa50VYObMmaxfv16qb7Zq1Qp3d3du3LihtF+HDh1YuXIl9+/f5+jRowQHB0tvtqOjo7l58yaBgYFP/T19ff16q7NmZ2fXOo+sDUWZVVpaypEjR8jOzmbIkCFA5VCfe/fuqczz4+rqqtI4VjUdamtr4+Tk9NTwPCkP12TV1UubNm3KqVOnnlqOa2J9v65Eo8QzMDAwkFaQAGjdujUffPABR44ckTL9qt544w1CQkKYPHkyp0+fxtzcnHbt2int87JPzGRkZFTnSYm0tLQYP348Y8eOJT8/H2NjY6k1z9LSknv37pGZmcm6detYt24dUFnxk8vljB49Gj8/P7p06YKJiQm6urpKD5e2traUlJRQWFgovYlVhK+hxlVZW1ujpaVFRkaGyps1qGxk0NHRkRpYtLS0VLqbVW0VVXz32WefqTSGKbrIyeVyHB0dq51YqOrESI83hGhpaT2xa+PzeNJv1Ta89eHx+xgq35DURn3dr8XFxSxbtoxOnToxc+ZMmjdvTmFhIf/973+VuuV5enry9ddfM2HCBI4dO0bPnj1f6MRW1bGwsKBFixbSG2lFnmZoaEjr1q2Jj48nPj4eV1fXOr85e5ErR1TtTqr4u2piN8/q0qti+/NKSkpi9erVeHt788EHH/DKK69w/vx5pZUAdHV16devH1FRUbz++utER0dL4/fVpS5psC73bNU0oVCXNFFRUYG3t7c0p0VVVcuX2vztFOVW1d+vz8lf1aUu951cLqdr167VjqF+Utd2TdSkSRMePnyosv3Bgwf1NlHyq6++KuUVdnZ2ZGdnExkZyXvvvYe+vj5r166loKCADz74gBYtWqCnp8fixYvrnK48PT2ZN28ed+7ckRrNa+oJ2pCelJaeJe0YGRmhpaX1TBNpWllZERAQQHFxMY8ePcLU1JTg4GCVF2eGhoZYWVlhZWWFi4sLs2fP5ujRo3h7e0tvxR8Pc3BwMC4uLixZskTaVlRUpBFzAVSnapn14YcfEhAQwI4dO+pt+eQned48XF1qKudrQxPr+3UlGiXqiba2do2tlT169CAkJIS//vqLkydP4u7uLmWctra2yOVyUlJSpBbDO3fuNIpuRnXh6OhY7VI3taGtrS21dMbExODi4oKxsTFNmzaV3morHDp0iEuXLjFv3jypEGjbtq00r4eignf79m0MDAyUZq9PT0/HzMyswbp+NmvWDDc3Nw4ePMjQoUOVKqQlJSUcPHiQ1157TaqYGBsbS8NUFG7evCn1grCzs0NPT4/c3Fw6duxY7W+2atWKmJgYjIyMGsVSiJoUXltbW/744w+ldHPt2jV0dXWxtLR86vFWVlbo6OiQnJws7V9cXEx6err0+datWxQWFjJ27FgpvT4+UStUDmVq2rQphw8f5s8//5QmQ1W3qmP6q07e1aFDB65cucLVq1cZOnQoUNmQqKurS2JiolToVlRUkJycjLu7e51+19bWVmXoQHJy8vNezkvJ1taWvLw87ty5IzVepqSkKFXQEhMTMTMzU5pkubouwZ6ensyZM4eDBw9SXFysEbP51zYN1lf6U6S9qhRDqxRat25NZmbmM1cuq1I8cFQtCzRxeFFDatWqFX/88Qfm5uaNfiZ6GxsbLly4gFwuV2oou3HjBjY2Nkr7pqWlUVxcLPWWSE5OrnX5U5W2tjbl5eXIZDJpfoOJEydK85rl5+erDC9Q/N7jeaytra302d7eHmdnZ44cOcLJkyerfSmnbs+SdnR1dbGzsyMjI0NlssvaMjQ0xNDQkKKiIi5evMj48eOfuL9cLpcahcaMGcPbb7+t9P28efPw8fFRmQQyPT1d5SXns6rPMro63t7eLF++nDfffBMzMzNMTU1JTExUWqb52rVrKg1bSUlJUh1X8bykSRP8vgi1Kcc1qf78rMScEs+grKyM/Px88vPzycjI4IcffqC4uLjGyX309fXp1asXv/zyCzdu3FCqONnY2ODm5sbGjRtJSkoiNTWVtWvXqnQbW7NmDWvWrGnQ62pIbm5uZGRkKE2klJWVRWpqKvfu3UMmk5GamkpqaqqUMd+/f59Dhw6RkZFBamoqoaGh/PHHH0yYMAGoLDgcHByU/hkbG6Onp4eDg4NUkA8cOJCioiI2b97MrVu3iIuLY9u2bQwcOFCpUpCQkCDN6NtQJk2aREVFBUuWLOHKlSvcuXOH+Ph4li5dio6OjrQiCVR2Tb5w4QLnz5/n1q1b/Pjjj0rdK5s0acLbb7/Nli1bOHbsmBSfhw4d4siRI0BlL53mzZsTFBTE1atXycnJ4erVq/z000/cvn27Qa/1WWhSeL28vLh37x6bNm0iIyODv/76i61btzJo0KBaveE0NDRkwIABbN26lUuXLpGens769euVWqTNzc3R09Pj999/Jzs7m7/++ovIyEiVc2lra9O/f3/Cw8MxMzNTKsTVqUOHDiQnJ5OcnCxNZgnQvn17Tp8+Lc3sDJXxMXDgQLZu3cpff/1FRkYG3333Hfn5+dIKO7U1ZMgQTpw4wZEjR7h9+za7du0iJSWlXq/tZdGpUydsbW1Zu3YtqampJCUl8eOPPyq9SbK2tiYvL4+TJ0+SnZ3NoUOHlOaTULCxscHV1ZWwsDB69eqlEUvg1jYN1lf6Gzx4MJcvX2bXrl3cvn2bI0eOcO7cOaV9hg8fTkxMDJGRkaSlpZGZmUlsbGydJilU0NfXx9nZmd27d5Oenk5iYqJSD5a/Ay8vLx4+fMjq1atJTk4mOzubS5cuERISwqNHj9QdvDoZOHAg2dnZ/PDDD6SmpkpzZcXExKhMbFpeXs769etJT0/n0qVLhIeH4+np+dS5RgoLC8nPz+fu3btcuHCBAwcO0KFDB+l+tba25uTJk2RkZJCSksLXX39d7QP72bNnlfLYK1euSN3vFTw9PdmzZ4/GNFI+7lnTTpcuXVQmp6xaT1UMxUpNTVUaNhgXF8eFCxfIycnh0qVLBAQEYGtrK61Q8fDhQyIiIkhOTubOnTtcv36ddevWcffuXalnlZmZmUrdFirrC1UbpEpKSrh+/Xq1q/88i/oso6vToUMH7OzspAmU33nnHfbu3cupU6e4desWkZGRJCQkqDTIHD58mNjYWG7dusXmzZu5c+cOAwcOfO7waKKqz5eKf/fv369VOa5J9edn1bibnNXk8uXLTJ06Fah8MLSxsWHOnDlKFaLH9evXj+PHj9OqVSuVVsAZM2YQEhJCQEAAxsbGeHt7q4zveXysX2Pj4OCAk5OT0pKbGzZsUJo1+NNPPwUqG2AUb41PnDghVcBcXFzw9/ev8/hRc3NzFixYwE8//cQnn3yCiYkJ/fv3V1qto7S0lLNnz7JgwYLnus6nsbCw4IsvvmD79u18++235OfnI5fLcXV1JSgoSKmLVf/+/bl58ybr168HKgvXnj17KjXsjBo1iubNm7N37142bdokzeCsmEzIwMCAgIAAwsPDWbVqlTR5U4cOHerUkqpYZmnRokVPTOfPq77CWx/MzMzw8/MjLCyMTz/9lFdeeQV3d3fGjBlT63P4+PhQUlLCypUrMTAwYNCgQUqTZBobG+Pr68vPP//MwYMHcXBw4P3331eZJwQq08OOHTvo37+/xgz36tChAzKZTKm7MFSOCy0tLaVJkyZKcx6MGzcOQFqvvFWrVixYsKDOE4r16dOH7OxsIiIiKCkpoUePHgwdOvSZe2O9zLS1tZk3bx4hISH85z//kZYSU6zwApW9+d555x02b95MaWkpXbp0YdSoUWzatEnlfAMGDCAhIUHtE60q1CUN1kf6c3FxYdq0adIylB06dGDEiBH88MMP0j5ubm7Mnz+fX375hb1796Kjo4O1tXWNS+c9zfTp0wkJCcHPzw9LS0smT56sNCnvy87MzIwlS5YQHh7O8uXLKS0txdzcnC5duqisdKDpLC0tCQgIIDIykmXLllFaWoqtrS1z5sxRmdSwffv22NvbExAQQElJCb169XrqG3dAKj+0tbUxNTWla9euSuXW9OnT2bhxI5999hlmZmaMGDGi2jHlI0aM4MyZM4SGhmJsbMz06dNV6l99+vQhNDSU3r1706RJk2eJkgb1rGnH09OTTz/9lKKiIqlelpeXJ9VToXIOhiNHjtC+fXv8/f2BykaHn3/+mbt379KsWTN69erFmDFjpEYfHR0d0tPTiYqKorCwECMjI9q0aUNAQMBT56l63Llz56odDv486quMrsnbb7/NunXrGDZsGIMHD+bRo0ds3bqV/Px8bGxsmDt3rtIqJABjx45l37593LhxA3Nzc+bNm6fxc0M8q6rPlwpmZmZs2LDhqeW4JtWfn5WWvDEMshFeCnFxcYSGhhIcHFzjShnq8vvvv3P+/Hk+//xztfz2Tz/9xMcff0yPHj1e+O/XRlRUFOHh4axevbrRZG4vm+TkZBYuXMiaNWtU5hARhBfl119/JSoqSqkyJAhC/Vq7di2FhYXMnz9f3UF5ory8PGbMmIG/vz+urq7qDk69Wr16NXZ2dkrD2jSJn58fQ4cOVfuk14JQXzTryVB4qbm5ueHl5SUtdahJdHV1lYZOvEiDBg1i5syZpKenq33lj5pcuHCBcePGiQYJNSgrK+Pu3btERkbSs2dP0SAhqIViPpTffvuNwYMHqzs4giCokUwmIz8/n59//llaovBlM378eI0YoladgoICevfuXS9zPQiCphA9JQRBEDTY8ePHWb9+PY6OjnzyySeiUUJQi7Vr1xITE0OPHj3497//Xe3s5oIg1A9N7ymhGNJpbW3NnDlzVLrcC4Ig1JVolBAEQRAEQRAEQRAEQS3E8A1BEARBEARBEARBENRCNEo8QVFREVOmTFFa7kdT+Pn5ERsbq+5gCGqiyWlz1apV7N27V93BqBMRn4Km0uS0+Xcth3x9fdmzZ4/0OT8/n6VLl+Lj48PIkSPVGLIXT5PTZ2PLO0VcCn8H//vf/xg5ciQ5OTn1el5/f3++//77ej1nQxL3uyqxJOgT7Nq1i65du0rLjIWGhpKYmEh6ejomJiasXbtW5ZjTp09L65cbGxszaNAgpbWnFePwHhccHIytra30OTY2lsjISLKzs7G0tGTMmDH07NlT+n748OH89NNP9OzZU+NWshAanrrSZnp6Otu2bePGjRvk5OTg7e2tUgn39vZm0aJFeHp6auwkUY9TV3z+8ccf7N69m6ysLMrLy7GysmLo0KFKSwdqanxWVFSwbds2Tp48SX5+PiYmJrzxxhuMGDFCmm9ALpezfft2jh49SlFREc7OzkyaNAl7e/saz6tYahEql7Rr0qQJtra2dO/encGDB2NoaPhCrk9TNETarOratWv4+/tja2vLV199pfTd37kcetKY/sDAQAwMDKTPe/bs4d69ewQFBWnksogNSV155/Hjx1m3bp3KPmFhYejr6wOam3fWpCHi8urVq4SHh3Pr1i1KSkpo0aIFAwYMUMkPDhw4wKFDh8jNzcXIyIgePXowfvx4Kb9tbHEp1KymvO1///sffn5+rFmzBgsLCzWFTtXatWufutz3tm3bXlBo6o/IO1WJRokalJSUcOzYMT777DNpm1wux8PDg7S0NC5duqRyzIULF/jmm2+YOHEibm5uZGZmEhISgr6+PoMGDVLad9WqVdLaxwDGxsbS/5OSkli9ejUjR46kZ8+enD17llWrVrFkyRKcnZ0B6NatGyEhIcTFxdGtW7f6vnxBg6kzbSoqNb169SIiIqLa8Dk4OGBpaUl0dLTKuTWROuPTyMiI9957D1tbW3R0dPjrr7/YsGEDxsbG0n2tqfH566+/cvDgQXx9fXFwcCAtLY21a9eiq6srLaG2e/du9u3bx4wZM7CxsWHHjh0sXbqU1atXP/HhzcbGBn9/f+RyOUVFRVy7dk1ainLx4sWYmJi8qMtUq4ZOm0VFRaxZs4ZOnTqRl5en9J0oh2pW9R4GyMrKolWrVlhbW6spROqhzrwTwMDAgG+//VZpm6JSDZqbd1anoeLS0NCQwYMH4+DggIGBAdeuXeO7777DwMAALy8vAE6dOkVYWBjTpk3D1dWVnJwc1q9fT1lZGdOnTwcaV1wKL5eJEycybtw46fOsWbMYM2YMffr0UWOono/IO6snGiVqcOHCBQDatm0rbVMsGblnz55qE0x0dDTdu3eXMnpLS0veffdddu/ejZeXF1paWtK+xsbGKolEYf/+/XTo0IH33nsPADs7O+Lj49m/fz8fffQRUPkGsWvXrpw6depvVRkU1Js2nZyccHJyAipbeWvSo0cPYmJiGkXlRZ3x2bFjR6XPQ4YM4cSJE1y7dk3pvtbE+ExKSqJ79+706NEDAAsLC7p3705KSgpQWcAeOHCAd999l969ewMwc+ZMJk+ezKlTp3jrrbdqPLeOjo7U8GBqaoq9vT09evRg7ty5hIWFMXPmTADi4uLYuXMn6enpQGX6/OCDD7CzswMgICAAOzs7Jk2aJJ374cOHTJ06lVmzZtGrVy/OnDnD9u3buX37Nvr6+jg4ODBnzhyNaPho6LS5YcMGPDw8kMvlnDlzRuk8ohyqma+vL15eXrzzzjv4+vqSm5sLVMa9h4cHvr6+PHz4kC1bHeumAQAAF5tJREFUtnDu3DlKS0tp1aoV77//Pm3atFFz6OuPOvNOhafdp5qYd1anoeKydevWtG7dWjrGwsKCs2fPkpCQIB2XmJiIs7Mz/fr1k/bx8PBQyRMaS1wK9efq1auEhYVx8+ZNmjZtiru7O+PHj0dXt/LxsaysjK1btxITE8PDhw9xdHTEx8dHaYnYuLg4Nm/eTG5uLm3atGHgwIF1CkPTpk1V3tY3bdq02ntfLpcTHh7O0aNH0dLSol+/fowfP17qySeTyYiIiODUqVMUFRVhb2/PqFGjcHNzq2vUPBeRd1bv5epvWY8SEhJo3bq10h/5acrKytDT01Papq+vz927d6VKi4Kfnx9Tp05l8eLFXLlyRem7pKQkunTporStS5cuJCUlKW1zcnIiISGh1uETXg7qTJu15eTkREpKCqWlpc90/IukKfEpl8u5fPkyt27dol27dkrfaWJ8urq6Eh8fT2ZmJgAZGRnEx8fTtWtXAHJycsjPz6dz587SMfr6+rRr147ExMQ6/56pqSl9+/bl3LlzVFRUAFBcXMyQIUNYvnw5/v7+NGnShBUrViCTyQDw9PTk1KlTlJWVSeeJiYnB0NCQ7t27k5+fz+rVq/Hw8CA4OJiAgACpYq4JGjJtHjx4kIKCAoYPH17teUQ5VDuBgYF06tSJ119/nY0bNzJx4kTkcjmBgYHk5eUxf/58goKCaNeuHYsXL+bevXvqDnK9UXfeWVpayowZM5g2bRpffPEFN27cUNlHE/PO6jR0XCrcuHGDxMRE2rdvL21zdXUlNTVVurfv3LnD+fPnpbxcobHEpVA/8vLyCAwMxNHRkRUrVjBt2jRiYmIIDw+X9gkLC+P06dNMnz6dFStWYG9vz7Jly6R87s6dO6xcuZLOnTsTFBTEoEGDCAsLa7Awnzx5Eh0dHZYsWcKHH37IgQMHOH36tPT9unXrSEhIYPbs2Xz11Vd4eHiwYsUKUlNTGyxM1RF5Z/VET4ka5ObmYmpqWqdj3Nzc2Lx5MxcvXqRTp05kZWWxb98+oHIiLAsLC0xNTZk8eTJOTk7IZDKio6NZsmQJ/v7+0oNIfn4+zZs3Vzp38+bNyc/PV9pmZmZGXl4e5eXlYs34vxF1ps3aMjU1pby8nLy8PGm8nKZSd3w+fPiQf/3rX8hkMrS1tZk0aZJKZVAT43PYsGE8evSIjz/+GG1tbcrLy3nvvfekVnxFfvV4a3zz5s2f+cHMzs6OR48eUVhYSPPmzaUeGAozZszggw8+ICUlBVdXV3r16kVoaChnz57F3d0dgKioKPr164eurq6Uf/bu3ZsWLVoAld0WNUVDpc20tDR27NjBsmXLapwLQpRDtWNsbIyenh76+vpSWr9y5Qqpqal8//33UpfY0aNH8+effxIdHc2wYcPUGeR6o86808bGhunTp+Po6MijR484cOAACxcuZOXKlUrDaDQx76xOQ8WlwrRp07h//z7l5eWMGDFC6W21u7s7hYWFLFq0CIDy8nL69eun1GUeGk9cCk8XFxeHj4+P0ja5XK70+eDBg9K9qK2tjZ2dHePGjWPjxo2MGjUKuVzOoUOHmDZtmtRTburUqcTHx3Pw4EFGjx7NoUOHMDc3Z+LEiWhpaWFra8vt27eJjIxskOuys7Nj1KhRQGUecfToUa5cuULfvn3JysoiJiaGtWvXYm5uDsCgQYO4dOkSR44cYfLkyQ0SpuqIvLN6olGiBmVlZUrja2rD09OTrKwsgoKCKC8vp0mTJgwZMoTt27dLrWE2NjbY2NhIx7i4uJCbm8uePXvq/OCnr6+PXC6nrKzsb1kZ/LtqLGkTaBRvVNQdn4aGhqxcuZLi4mIuX77Mjz/+SIsWLejUqZO0jybG5+nTp4mOjmb27NnY29uTmppKaGgoFhYWDBgwoEF/WxHHWVlZREZGkpKSwv3796moqEAul3Pnzh0A9PT0eOONN4iKisLd3Z309HRSUlKYMWMGAI6OjnTq1Im5c+fSuXNnOnfuTO/evZ/a7fFFaYi0WVZWRnBwMD4+PvUymZkoh1Rdv36d0tJSpWFDUPn3zM7OVlOo6p86804XFxdcXFykfdq2bcsnn3zCb7/9JnWDBs3MO6vTUHGpsHjxYoqLi0lKSmLr1q1YWFhIvcKuXr3KL7/8wuTJk3F2diYrK4vQ0FC2bdsmPeBB44lL4enatWvHv/71L6VtaWlpfPnll9LnzMxMnJ2dlRquXV1dkclk0ooR5eXlSkMQtLW1cXZ2JiMjQ+kcVdNj1fu2vrVs2VLps6mpKQUFBUBlLyG5XM6cOXOU9pHJZCpDaRuayDurJxolamBkZERRUVGdjtHS0mL8+PGMHTuW/Px8jI2NuXz5MlA59qcmTk5OSt2LTExMpJtIoaCgQOWNY1FREXp6en+72ej/7tSZNmtLET5Nebh7EnXHp7a2ttQK7ejoSGZmJrt27VJqlNDE+AwLC+Ptt9+WeiA4ODiQm5vLrl27GDBggJRf5efnS28loDIve/wNfG1lZGTQpEkTaQKnFStWYGZmxpQpUzAzM0NHR4ePP/5YGr4BlQX5vHnzuHPnDlFRUbi4uEhzTmhra/P555+TnJzMxYsXOXbsGOHh4fj7++Po6PhMYaxPDZE27927R2ZmJuvWrZNm4JbL5cjlckaPHo2fnx9dunQR5dBzqKiooHnz5ixevFjlu5dpdQ51551VaWtr06ZNG5Xl9TQx76xOQ8elogHSwcGBgoICtm/fLjVKRERE4O7ujqenp7RPcXExISEheHt7S42NjSUuhaczMDBQefv94MGDWh+vpaWl0rNCEzzeMF41nHK5HC0tLQIDA6U5MRTq2kDwvETeWUNYXtgvNTKKh4Nnoa2tjZmZGbq6usTExODi4vLEP2pqaqpSRc/FxUVlkpNLly6ptC6mpaUpTWAk/D2oM23WVnp6OmZmZhoxWeDTaFp8VlRUKM2BAJoZnyUlJSpd/7W1taUKgIWFBSYmJkp5WWlpKdeuXVN6s1Jb9+7d49SpU/Tq1QttbW0KCwvJzMzkn//8J507d5aGdpSXlysdZ29vj7OzM0eOHOHkyZP0799f6XstLS1cXFwYMWIEgYGBmJqaPlNDXENoiLRpZmbGl19+SVBQkPTvrbfewsrKiqCgIOlvI8qhZ9e6dWsKCgrQ0tLCyspK6d+zNshpIk3KO+VyOTdv3lTZRxPzzuq8yLhU9GxSeFpertBY4lKoH7a2tiQnJ0tzOEHlEtK6urpYWlpiaWmJrq6u0hxRFRUVJCcnSw3/inNUTUvJyckv7iKqcHR0RC6Xk5+fr5Ivm5mZvfCwiLxTlegpUQM3Nze2bt1KYWEhRkZGQGVX4eLiYu7du4dMJpMmRrGzs0NXV5f79+8TGxtL+/btkclkREVF8ccffyitGbt//35atGiBvb09MpmMkydPcu7cOebOnSvtM2TIEBYtWsSvv/7Ka6+9xtmzZ4mPj1d563Lt2jWViciEl58606ZMJpO65ZWWlpKfn09qaiqGhoZKre4JCQmNJm2qMz537tyJk5MTlpaWlJWVceHCBU6ePMnEiROVwqiJ8dm9e3d+/fVXLCwssLOzIzU1lX379uHh4QFUPuwPGTKEXbt2YWtri7W1NTt37sTQ0JC+ffs+8dzl5eXk5+dLS4ImJiaya9cumjVrxtixYwF45ZVXMDIy4ujRo5ibm5OXl8eWLVuqHULg6enJd999h46OjtIyYklJSVy+fFnqGXDjxg3u3r0rVajUrSHSpq6ursq8GYp5EapuF+UQPHr0SGUCtNqs2d6pUyfatm1LUFAQ48ePx9bWlvz8fOLi4ujUqVOdh8NpKnXmndu3b8fZ2Rlra2tpXHRaWhpTpkxRCqMm5p3Vaai4/O2337CwsJC6dCckJLB3716lOSW6d+/O/v37adOmjTR8IzIykm7duinlp40lLoX64eXlxYEDB9i0aRNDhgwhJyeHrVu3MmjQIAwMDAAYOHAgW7duxcjICAsLC/bv309+fr40t9TAgQPZt28fmzdvxsvLi7S0NA4fPqyW67GxsaFv376sW7eO999/n1atWlFUVER8fDyWlpb06tXrhYVF5J3VE40SNXBwcMDJyUlpOZQNGzZw9epVaZ9PP/0UgDVr1khd406cOMGWLVuAyjdN/v7+0hKKUPlQFxYWxt27d9HX18fe3p758+crLafWtm1bPvroIyIiIoiMjMTKyoqPPvpIWhseKmfFTUxMZNasWQ0XCYJGUmfazMvLk84NkJ2dzZEjR2jfvj3+/v5AZWPF2bNnWbBgQcNEQD1TZ3wWFxezadMmaR9bW1t8fX2VHto1NT4//PBDIiMj2bRpEwUFBZiamuLp6Ym3t7e0z7BhwygtLeX777/nwYMHODk5sWDBgqd2Yb916xZTp05FS0uLpk2bYmNjg6enJ4MHD5aO1dbWZs6cOYSGhjJ37lysrKzw8fHhq6++Ujlfnz59CA0NpXfv3kq/3bRpUxITE/n999958OABr776KsOHD9eYFTgaKm3WhiiHKitlVfM7oFYVVy0tLfz8/IiIiCAkJEQa9tK2bVuNSVv1QZ1554MHD9i4cSP5+fk0bdqUVq1aERAQoHQeTc07q9NQcVlRUcHWrVvJzc2VhgqOHTtWaUnm4cOHo6WlRWRkJHfv3sXY2Jju3bszevRoaZ/GFJdC/TAzM8PPz4+wsDA+/fRTXnnlFdzd3RkzZoy0j2Iy1PXr1/PgwQNatWrFggULpEkczc3NmTdvHj/++CNHjhyhdevWjB07lm+//Vbpt0aOHIm3tzcjR45s0GuaMWMGO3fulPKXZs2a4eTk9MLnlBB5Z/W05Jo4KEhDxMXFERoaSnBwcI0zlKvLli1bpFn7hb8fTU6bv//+O+fPn+fzzz9Xd1BqTcTnyy0vL48ZM2bg7++vtH56Y6DJaVOUQ4Imp8/GlneKuBT+jnJycpg1axYBAQGNrnx+HuJ+V6Xjr3i9KaiwsrJCLpdjamrKK6+8ou7gKLl58yZDhw4Vk4v9TWly2kxNTWXgwIFSl7TGQMTny0kmk3H//n3Cw8PR0dFRevPXWGhy2hTlkKDJ6bOx5Z0iLoW/o5MnT2JsbMyQIUPUHZQXStzvqkRPCUEQBOGlFB8fT0BAANbW1syZM0cjVtQQBEEQBEEQlIlGCUEQBEEQBEEQBEEQ1EKzBrEIgiAIgiAIgiAIgvC3IRolBEEQBEEQBEEQBEFQC9EoIQiCINSroqIipkyZQlZWlrqDomLVqlXs3btX3cEQ1ESkTUEQhLoTeafQ0HTVHQBBEATh5bJr1y66du2KlZUVAKGhoSQmJpKeno6JiQlr165VOeb06dPs2rWL27dvY2xszKBBg3jnnXek7xWTVj4uODgYW1tb6fPDhw+JiIjgzJkzFBYW8uqrrzJmzBj69OkDgLe3N4sWLcLT05OmTZvW96ULGq4h0mZV165dw9/fH1tbW7766iul70TaFAShsWqIvPPq1auEh4dz69YtSkpKaNGiBQMGDFDJXw8cOMChQ4fIzc3FyMiIHj16MH78eGnlJ5F3vhxEo4QgCIJQb0pKSjh27BifffaZtE0ul+Ph4UFaWhqXLl1SOebChQt88803TJw4ETc3NzIzMwkJCUFfX59BgwYp7btq1SqaNWsmfTY2Npb+L5PJWLp0Kc2aNWPOnDmYmZmRl5eHru7/FXUODg5YWloSHR2tcm7h5dbQabOoqIg1a9bQqVMn8vLylL4TaVMQhMaqofJOQ0NDBg8ejIODAwYGBly7do3vvvsOAwMDvLy8ADh16hRhYWFMmzYNV1dXcnJyWL9+PWVlZUyfPh0QeefLQgzfEARBEOrNhQsXAGjbtq207cMPP2Tw4MFYW1tXe0x0dDTdu3fHy8sLS0tLunXrxrvvvsvu3bt5fIEoY2NjTExMpH/a2v9XjB0/fpz79+/z6aef4urqioWFBa6urjg5OSmdo0ePHsTExNTXJQuNREOnzQ0bNuDh4YGzs7PKeUTaFAShsWqovLN169a4u7tjb2+PhYUF/fr1o0uXLiQkJEjnSUxMxNnZmX79+mFhYUHHjh3x8PAgJSVF6fdE3tn4iUYJQRAEod4kJCTQunVrtLS0an1MWVkZenp6Stv09fW5e/cuubm5Stv9/PyYOnUqixcv5sqVK0rfnTt3jrZt2/LDDz8wZcoU5syZw7Zt25DJZEr7OTk5kZKSQmlpaR2vTmjMGjJtHjx4kIKCAoYPH17teUTaFAShsWrocl3hxo0bJCYm0r59e2mbq6srqampJCUlAXDnzh3Onz9P165dlY4VeWfjJxolBEEQhHqTm5uLqalpnY5xc3Pj/PnzXLx4kYqKCm7dusW+ffsAyM/PB8DU1JTJkyczd+5c5s2bh42NDUuWLFF6o5KdnU1sbCwymQw/Pz9GjRrF4cOHCQ8PV/o9U1NTysvLVbrYCy+3hkqbaWlp7Nixg1mzZin13KlKpE1BEBqrhso7FaZNm8bYsWOZP38+Xl5eDBw4UPrO3d2dMWPGsGjRIsaMGcOMGTNwcHBg3LhxSucQeWfjJ+aUEARBEOpNWVkZ+vr6dTrG09OTrKwsgoKCKC8vp0mTJgwZMoTt27dLb2ZsbGywsbGRjnFxcSE3N5c9e/bQrl07oHKMq7GxMdOmTUNbW5vWrVtTVFTEjz/+iI+Pj3QuRfjEG5W/l4ZIm2VlZQQHB+Pj44OFhUWN5xFpUxCExqqhynWFxYsXU1xcTFJSElu3bpWGckDlZJi//PILkydPxtnZmaysLEJDQ9m2bRujRo2SziHyzsZPNEoIgiAI9cbIyIiioqI6HaOlpcX48eMZO3Ys+fn5GBsbc/nyZQAsLS1rPM7JyYnTp09Ln01MTNDV1VV6W21ra0tJSQmFhYXSpJiK8FWdJFN4+TVE2rx37x6ZmZmsW7eOdevWAZUNEHK5nNGjR+Pn50eXLl1E2hQEodFq6HJd0aDr4OBAQUEB27dvlxolIiIicHd3x9PTU9qnuLiYkJAQvL290dHRAUTe+TIQjRKCIAhCvXF0dOTEiRPPdKy2tjZmZmYAxMTE4OLi8sQKRmpqKiYmJtLntm3bEhMTQ0VFhfTwd/v2bQwMDDAyMpL2S09Px8zMTOlY4eXXEGmzadOmfPnll0r7Hjp0iEuXLjFv3jypsi3SpiAIjdWLLNflcjllZWXS55KSEpVhcdra2ioTDYu8s/ETjRKCIAhCvXFzc2Pr1q0UFhZKD1tZWVkUFxdz7949ZDIZqampANjZ2aGrq8v9+/eJjY2lffv2yGQyoqKi+OOPPwgICJDOu3//flq0aIG9vT0ymYyTJ09y7tw55s6dK+0zcOBADh48yObNmxk0aBA5OTls27aNgQMHKnUXTUhIoEuXLi8mQgSN0RBpU1dXFwcHB6XfMTY2Rk9PT2m7SJuCIDRWDVWu//bbb1hYWEhDMxMSEti7d6/SnBLdu3dn//79tGnTRhq+ERkZSbdu3aReEopjRd7ZuGnJH29qEgRBEITnsGDBAt544w1pvXB/f3+uXr2qst+aNWuwsLDg/v37rFixgrS0NKByvojRo0crLa24e/dujh49yt27d9HX18fe3p53332Xbt26KZ0zKSmJn376iRs3bmBiYkK/fv0YPnw4urqVbfClpaVMmTKFBQsW4OLi0lBRIGiohkibj9u2bRtnzpzhq6++Utou0qYgCI1VQ+Sd+/fv5+jRo+Tm5qKtrY2VlRUDBgzgrbfeknpHlJeXs3PnTk6ePMndu3cxNjame/fujB49mmbNmgEi73xZiEYJQRAEoV7FxcURGhpKcHBwjasRqMvvv//O+fPn+fzzz9UdFEENRNoUBEGoO5F3Cg1Nx9/f31/dgRAEQRBeHlZWVsjlckxNTXnllVfUHRwlqampDBw4UGkcv/D3IdKmIAhC3Ym8U2hooqeEIAiCIAiCIAiCIAhqoVn9bwRBEARBEARBEARB+NsQjRKCIAiCIAiCIAiCIKiFaJQQBEEQBEEQBEEQBEEtRKOEIAiCIAiCIAiCIAhqIRolBEEQBEEQBEEQBEFQC9EoIQiCIAiCIAiCIAiCWvx/UODBlE7aE4YAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 676 + }, + "id": "Z8sdZC7-wwdh", + "outputId": "2288028a-5149-4e5b-9380-9c12e72378ad" + }, + "source": [ + "plot_components(components_df, 'fc1', ascending=False)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAALlCAYAAADzMFwcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVyVdf7//+eRI6Agi+JxAcU1t3JJEVMLV3AL0/ik6TSYptlt0JxuMxY1JqkljjOl5TT5s8Q117Fc0lxwKTMrLUVzTy0RNwTcRYXr90dfTh45oByOXi6P++3WrXhf7+u6Xud9ruvEeXJd78tiGIYhAAAAAACAO6yE2QUAAAAAAIAHE6EEAAAAAAAwBaEEAAAAAAAwBaEEAAAAAAAwBaEEAAAAAAAwBaEEAAAAAAAwBaEEANyn+vXrJ4vFosOHD5tdSrG58lrWr18vi8WihISEYu//8OHDslgs6tev3y2vM23aNFksFk2bNu2O7/teM2fOHDVp0kRlypSRxWLRsGHDzC4JcKsH4TwGAFcRSgCACywWiywWi0qUKKFffvmlwH5t27a19y3ul1M44pd8c7gaduWFNOvXr3do//bbb9W3b1+dO3dOL730kkaOHKlOnToVua7vv/9e8fHx6ty5sypWrCiLxaKQkJAC++cdP+4IrXD3uJ/CWAB4UFjNLgAA7lVWq1XXrl3TJ598onfeeSff8v3792v9+vX2fnfa2LFj9dprryk4OPiO79vd7sXX0qNHD7Vo0UKVKlUyu5S72hdffCHDMDRjxgy1bNnS5e18+umnmjhxokqWLKn69evrxIkTbqwSAADcLlwpAQAuqlChgpo1a6akpCSnocPHH38sSXryySfvdGmSpEqVKqlu3boqWbKkKft3p3vxtfj7+6tu3bry9/c3u5S7WlpamiSpcuXKxdpOv3799OOPP+r8+fPatm2bO0oDAAB3AKEEABTDwIEDdfz4cS1btsyh/erVq5o2bZpatmyp+vXrF7j+/v379ec//1nBwcHy9PRU5cqV9ec//1n79+936Dd48GBZLBYtXrzY6Xa+++47WSwWxcTE2NsKu4z5u+++U0xMjCpWrChPT09VqVJFL774ov0L4vUOHjyoQYMGqVatWipVqpTKli2rRx55RIMHD9bp06cLGx5Jv3/ZdHaFQ2hoqCwWi0aPHu3QvmLFClksFr355psFvpaEhARVr15dkjR9+nT7LTIF3Sazbds2de3aVQEBASpdurQiIiK0adOmm9buzOHDh9W7d28FBQXJ29tbzZo1y/f+S4XPKbFy5Uq1atVKPj4+Klu2rJ566int2bPnppee3+q+88yZM0dt27ZVQECAvL29Va9ePY0ZM0bZ2dn5+n799dd68sknFRISIi8vL1WsWFEtWrTQW2+9Ze9jsVg0ffp0SVL16tXtY16tWrXCB82JvPFJSkrKt73rX39qaqqGDh2q2rVr24+/5s2b5ztuGjdurCZNmsjT07PItRTFvHnz1L59e5UtW1be3t6qVq2ann32WW3ZssWhX3Z2thITE/XII4+odOnS8vPz0+OPP6758+fn2+b1tyL98ssviomJUbly5VSmTBlFRkZq586dkqRTp05p0KBBqlSpkry9vRUWFqZ169bl215CQoL9Vpnp06erSZMmKlWqlGw2m/r376/jx487fW23+nl04z4WLlyo5s2bq3Tp0ipbtqx69+6to0ePOt1HRkaG4uPjVa9ePZUqVUr+/v5q3769Vq1ala/v9efQunXr1KZNG5UpU0Z+fn7q2rWrdu/e7dDf1eMzMTFRFotFEydOdLo8LS1NVqtVzZo1c2gbNWqUWrVqZf8crVy5svr06aNdu3YVur/rtWnTRhaLxemywj5DUlNTFRcXpxo1asjLy0vlypVTdHS0fvjhh3x9z507p9GjR+vhhx+Wn5+fypQpo5o1a6pXr17aunXrLdcKALcLt28AQDE8++yzeuWVV/Txxx/rqaeesrcvWbJEJ0+e1Lhx43TgwAGn6/7www/q0KGDzp07p+joaNWvX1979uzRrFmztHjxYq1Zs0ZhYWGSpNjYWE2ePFkzZsxQ9+7d820r7xfxW5lfYerUqRo0aJC8vLwUHR2tKlWqaP/+/fr444+1dOlSbd68WVWrVpUkHTt2TGFhYTp79qy6dOmip59+WpcvX9ahQ4c0c+ZMxcXFqVy5coXur127dpo9e7b27NmjunXrSpIOHDig3377TZKUnJysESNG2PsnJydLktq3b1/gNtu0aaOsrCxNnDhRjRo1chj7xo0bO/TdsmWL/vnPf+qxxx7TCy+8oN9++03/+9//1L59e23btk116tS56Zjl+fXXX9W8eXPVqFFDzz33nDIyMjRv3jx1795da9asUdu2bW+6jblz56pPnz7y9vbWM888o0qVKmnTpk167LHH1KhRI7ftu3///kpKSlJISIiefvppBQQEaPPmzRoxYoSSk5O1evVqWa2//xrw5ZdfqmvXrvLz81N0dLSCg4OVkZGh3bt368MPP9TIkSMlSSNHjtTnn3+u7du36+WXX1ZAQIAk2f9dFI0bN77p9rZs2aKoqChlZGToiSeeUM+ePXXx4kXt2rVLCQkJDsfN7WYYhp5//nlNnz5dQUFB6tmzp8qXL6/U1FStW7dOderUsX9pvXLliqKiorRhwwbVrVtXf/nLX3Tx4kUtXLhQvXr10rZt25ze8nX48GGFh4erXr166tevnw4fPqzPPvtMbdq00bfffqtOnTrJz89PvXr1UkZGhubOnavOnTtr37599nP2eu+9955WrVqlXr16qVOnTtq4caOSkpK0fv16fffddypfvry9b1E+j6734YcfasmSJYqOjlZERIS+++47zZs3T9u3b9e2bdvk5eVl7/vrr7+qTZs2Onz4sB5//HF16tRJFy5c0LJly9SpUydNnjxZAwcOzLePZcuWafHixercubMGDx6sXbt2afny5frhhx+0a9cuBQUFSXL9+Hzuuef0xhtvaMaMGXr55ZfzLZ81a5ZycnIcPl+/+uorJSYmqm3btnr66afl6+ur/fv3a+HChVqyZIm++eabQs/n4vjxxx8VGRmpjIwMRUVFqWfPnkpPT9fnn3+u1q1b67PPPlOXLl0k/X7cdurUyf4Z88ILL8hqtdqP28cff1xNmza9LXUCwC0zAABFJskIDg42DMMwBgwYYHh4eBhHjhyxL4+KijL8/PyMCxcuGG+88YYhyUhKSrIvz83NNerWrWtIMmbNmuWw7blz5xqSjDp16hg5OTn29oceesjw9PQ0Tp8+7dD/8uXLRmBgoGGz2YyrV6/a22NjYw1JxqFDh+xte/fuNUqWLGnUrFnTSE1NddjOmjVrjBIlShhPPfWUve399983JBkTJkzINwbnz583Ll68eNOx+uSTTwxJxqRJk+xtH330kSHJ6Nixo+Hp6WlcuHDBvqxx48ZGqVKljOzs7EJfy6FDhwxJRmxsrNP9rlu3zpCUb+yv3/9LL7100/qv35ckIyEhwWHZl19+aUgyOnfu7NCelJSUb99nz541AgICDE9PT2Pbtm0O/V999VX7Ppy9Tlf23aNHj3zv0ciRI/O9pz179jQk5avJMAzj1KlTDj87ey+Ko6DtZWdnG9WqVTMkGbNnz8633vXnmzPXn6PuMHnyZEOSERYWZmRlZTksu3btmpGWlmb/+Z133rG/L9efkydOnDBCQ0MNScY333xjb7/+PR4zZozDtkeNGmVIMgIDA40XX3zR4TNhxowZhiRj2LBhDuvkvcclS5Y0fvzxR4dlw4YNMyQZ/fv3t7e58nmUt48yZcoYKSkpDus8++yzhiRj3rx5Du0RERGGxWIx5syZ49CemZlpNGrUyPD29jaOHz9ub887jj08PIw1a9Y4rPPaa68Zkoxx48Y5tLt6fEZGRhqSjB07duRbVr9+fcPT09NIT0+3t504ccI4e/Zsvr7btm0zfHx8jE6dOjm0F/R5FRERYRT067izz5CrV68aNWvWNLy8vIz169c79D969KhRuXJlo2LFisbly5cNwzCMlJQUQ5LD53qenJwcIyMjw+m+AeBO4vYNACimgQMHKicnR1OnTpX0+18DV69erb59+6p06dJO19m0aZP27Nmjxx57TH379nVY1qtXL7Vu3Vp79+7Vxo0b7e2xsbG6cuWK5syZ49B/6dKlyszMVN++fe1/+S7If//7X129elUTJ07Md0tF+/btFR0draVLl+rcuXMOy0qVKpVvWz4+Pk7bb5R3xUPeFRB5/12hQgUNHTpUV65csb/O06dPa/v27WrdurXbLsNv1apVvitI+vfvL6vVqu+//75I2woNDdU//vEPh7aoqChVrVr1lra1ePFiZWVlqW/fvvn+ivqPf/yj0L/oFmXfEydOlNVq1dSpU/O9RyNGjFC5cuU0e/bsfPtw9n7m/RX6Tlu6dKkOHz6s6Oho9enTJ9/ywp6scTt88MEHkqTJkyfnmyfEw8PDYULTqVOnymKx6N1333U4J202m/3qjrw5Z65XrVo1vfbaaw5tsbGxkn6/HWT8+PEqUeKPX9369Okjq9Va4Bwazz33nJo0aeLQlpCQIH9/f3366af223hc+TzKM3ToUD3yyCMObXlXO1x/XG7fvl0bNmzQ008/rd69ezv0DwgI0FtvvaXLly/rf//7X7599O7dO9+VU4MGDcq3j+LIG+e8q87ybNmyRbt27VLXrl0drgqz2WwqU6ZMvu00atRI7dq107p163T16lW31Ha9L774Qr/88ouGDBmiiIgIh2WVK1fW8OHDdfz4cYfPW8n5uV2iRAkFBga6vUYAKCpu3wCAYgoPD9cjjzyiqVOn6h//+Ic+/vhj5ebmOr0MOc+PP/4o6fdbG5xp166dNm7cqJ9++klPPPGEJOnPf/6zRowYoenTp+svf/mLvW9Rbt349ttvJUkbNmxweu/xyZMnlZOTo3379qlp06aKjo7W66+/rr/85S9auXKloqKi1KpVK9WvX7/A+6BvFBoaqho1amj9+vXKzc2134feoUMHRUREyGq1Kjk5WZGRkVq3bp0MwyhwXFxx/X3geUqWLKkKFSooMzOzSNtq3LixPDw88rVXqVLFPraF+emnnyRJrVu3zrfM19dXjRs3zvfIzKLu++LFi9q+fbuCgoI0YcIEp9vy8vJyuB+/b9++WrRokcLDw9WrVy+1bdtWrVq1uuNf/K+3efNmSVLnzp1NqyHPhQsXtHPnTlWoUCHfl/wbnTt3TgcOHFBwcLD9dqXr5R3becfC9Zy9x3kTgD700EP5vgR7eHioQoUKSk1NdVrLjV9apd8nYG3cuLE2bNig3bt3q3Hjxi59HuVxdn5VqVJFkhzOr7xj9MyZM04fw3rq1ClJyjdPRFH2URw9evSQv7+/Zs+ercTERPv7UNjn6xdffKGPPvpIW7ZsUXp6er4Jj9PT093+9J28cfz111+djmPe/B+7d+9Wly5dVL9+fTVu3Fhz5szRr7/+qu7du6t169Zq1qzZbZ9/BQBuFaEEALjBwIEDNXToUK1YsUJJSUlq2rRpoV9ezpw5I0kF/sKa156VlWVvCwkJUfv27bV69Wrt3r1b9erV08mTJ/Xll1+qcePGatiw4U3rzJuYcvz48YX2O3/+vKTfA4Xvv/9eCQkJ+vLLL7Vo0SJJv38h+Nvf/qahQ4fedJ/S71dLTJkyRT/++KNKliypU6dOqX379ipTpozCwsLsf9W7lfkkiqqgqw+sVqtycnLctq3c3Nybrp/3vleoUMHp8oLai7LvzMxMGYahU6dOOUxSWZiePXtq2bJl+ve//62pU6dq8uTJkqSmTZtq7Nix6tix4y1tx53yjv274TGwRanFlXM7j7MnteRdaVHQU1ysVmuBf5Ev6HiqWLGiQ63FqdnZcZlX8/XnV95nz+rVq7V69Wqn+5H++OxxZR/FUapUKT3zzDOaMmWKVq1apc6dO9uvTCtfvny+cGzixIkaNmyYAgMD1bFjR1WtWlWlS5eWxWKxz2vhbELZ4sobxwULFhTaL28cPTw8tHbtWo0aNUoLFy7Uq6++KkkqU6aMYmNjNXbsWPn6+rq9TgAoCm7fAAA3eO6551SqVCkNHjxYR48etV9aXJC8LxgFzYJ/7Ngxh355brzEePbs2bp27Zq9/WbytnfmzBkZhlHgP9f/hbVevXqaN2+eTp8+rS1btigxMVG5ubl6+eWX9cknn9zSfvP+ArtmzZp8wUO7du30008/KSMjQ8nJyfL399ejjz56S9u91/j5+UmSTpw44XR5Qe1FkfceN2nSpND32DAMh/W6du2qtWvXKjMzU8nJyfrrX/+qn3/+Wd26dSvS0wTcJe+LaEFPcbiTilKLq+f27VDQ8ZRXW14Nd6LmvHUnTpxY6DGZ9zQWM9z4+frFF1/o9OnT6tOnj8PjiK9du6aEhARVrFhRP//8s+bNm6fx48frrbfeUkJCQqHh4o3ybsdx9ljpwoKrxYsXFzqOeZPTSlJgYKDee+89HTlyxD6pcd26dTVp0iS99NJLt1wrANwuhBIA4AYBAQGKiYlRamqqfHx89OyzzxbaP+8qioIu1c97zN+NX8579uwpPz8/zZo1S7m5uZo+fbqsVqvTe+6dadGihaTfH/9YVFarVU2bNtWrr75qn9fi888/v6V127VrJ4vFouTkZK1du1Y1atSwP6avffv2ys3N1YwZM7R//361adPG6W0KN8rr466/lN4Jee+7s3vzz58/X+DcAEXh6+urBg0a6Oeff1ZGRkaR1/fx8VG7du307rvv6vXXX9eVK1e0YsUK+/I7Ne55x+r1+zaLj4+PHn74YZ04ccLpbRfXy3vc4tGjR50+SrOgc/t22LBhQ762M2fOaNu2bfZHxEqufx4VRXE+e4qiOMdnq1atVLt2bS1evFhnzpyxhxM3hr7p6enKyspSy5Yt811dcv78efvtMLcib06HI0eO5Ft242NmpeKPY61atTRgwABt2LBBvr6+BT5mGgDuJEIJAHCTMWPG6LPPPtPKlSudToB2vVatWqlOnTrauHGjFi5c6LBs4cKF+vrrr/XQQw/lm3sg7xLjo0eP6r333tP27dvVpUsX2Wy2W6oxLi5OJUuW1F//+lft27cv3/IrV644/LK7detW+6Xd18v7C2xBE3neyGazqUGDBvrmm2/01VdfOdye0bJlS3l7e2vs2LGSCr6v/UaBgYGyWCz2R4veC7p3726/b3379u0Oy8aMGeP0L6OueOWVV3TlyhX179/f6TYzMzMdvjh99dVXTv9S6+x9zpvs73aP+5NPPqlq1appyZIl+SZ3lVTgPAq3S96tSi+++GK+cyI3N9d+NYH0+0SqhmHo73//u8OX4/T0dI0ePdre53abOXNmvhAlISFBZ86c0bPPPmt/XKern0dF0axZMz3++ONatGiRfVLgG+3YsUMnT550eR9S8Y/P2NhYXb58WR9++KGWL1+uhg0b5rsVz2azqXTp0tq6davD7SZXr17Vyy+/rPT09FveX/PmzSVJU6ZMcWhPTk52etx3795dNWvW1H/+8x8tX77c6Ta//fZbXbx4UZJ06NAhHTx4MF+fzMxMZWdn39JkxQBwuzGnBAC4SdWqVVW1atVb6muxWDR9+nR17NhRvXr1Uvfu3VW3bl3t3btXn3/+ucqUKaMZM2Y4zLSfJzY2Vh9//LHi4+PtP9+qunXraurUqerfv78aNGigTp066aGHHtLVq1f122+/6euvv1b58uW1Z88eSb9/qZk8ebJat26tmjVrKjAwUL/88ouWLl0qLy8vDRs27Jb33b59e+3cudP+33m8vLzUqlWrIs8n4evrq/DwcH399dfq27evHnroIXl4eCg6OvqW5tcwg5+fn/7zn//oueeeU8uWLfXMM8+oUqVK2rRpk7Zv366IiAht2LDB6fteFP3799fWrVv14YcfqmbNmvandGRkZOjQoUP66quv9Pzzz+ujjz6S9PsX7qNHj6pVq1aqVq2aPD09tXXrVq1du1ahoaEOT0to3769xo8fr4EDB+rpp59WmTJlFBAQoLi4uGLVfCNPT08tWLBAkZGR6tOnjyZPnqwWLVro8uXL2r17t5KTkx2ClD179igxMdFhG5mZmQ4TFP7rX/9y+WkiL7zwgr7++mvNnDlTtWvXVvfu3VW+fHmlpaVp7dq16t+/v33iwb/97W9asWKFFi9erEaNGqlLly66ePGiFixYoJMnT2r48OHF+oJ/qzp37qxWrVrZj7ONGzdq48aNqlatmsNYFefzqCg+/fRTtWvXTgMGDND777+v8PBwBQQEKDU1VSkpKdq5c6e+/fbbWw5ZnSnu8fncc8/pzTff1MiRI3X16lWnn68lSpTQ0KFDlZiYqEceeUTdu3fXlStXtG7dOmVkZKht27b2q0tu5vnnn9f48eM1duxYbd++XfXr19e+ffu0YsUK9ejRI9/TSEqWLKlFixYpKipKXbt2VcuWLdW4cWOVLl1aR44c0Q8//KCDBw/q2LFjKl26tLZv366ePXsqLCxM9erVU+XKlXXq1CktXrxYV69etc8xAQCmuv1PHQWA+48kIzg4+Jb6vvHGG/meNZ9nz549xp/+9CejYsWKhtVqNSpWrGj07dvX2LNnT6HbrFWrliHJKFu2rJGdne20T2xsrCHJOHToUL5lKSkpRmxsrFG1alXD09PTCAwMNBo0aGAMGjTISE5OtvfbvHmzMXjwYKNhw4ZGYGCg4e3tbdSsWdPo16+fsWPHjlt6/XmWLFliSDIsFotx4sQJh2XvvPOOIcmoUKFCkV7L/v37jW7duhlly5Y1LBaLwzivW7fOkGSMHDnS6TZDQ0ON0NDQW6r90KFDhiQjNjbW6fKIiAjjxv+lJiUlFfi+L1++3HjssceMUqVKGQEBAUZ0dLSxe/duo2vXroYkIzMzs1j7zrN06VKja9euRvny5Y2SJUsaFSpUMMLCwow33njD2L17t73fvHnzjN69exu1atUyfHx8jDJlyhgNGjQwXn/9dePkyZP5tvvvf//bqFu3ruHp6WlIuuVxdKaw49QwDOPXX381XnrpJaNatWpGyZIljbJlyxrNmzc33n77bYd+ee93Yf8UtI+imDVrlvHEE08Yfn5+hpeXl1GtWjWjT58+xtatWx36Xbp0yXj77beNBg0aGN7e3oavr6/RqlUr49NPP823zZu9x5KMiIgIp8ucHccjR440JBnr1q0zkpKSjEaNGhne3t5GUFCQ0a9fPyMtLc3ptoryeXT9Poryes6ePWu8/fbbxqOPPmr4+PgY3t7eRrVq1YwuXboYkydPNs6fP2/vW9g5VNi4FPf4bN++vSHJsFqtxvHjx532uXr1qvHvf//bqFevnuHt7W1UqFDB+NOf/mQcPnzY6TFd2Jjs3LnT6Ny5s+Hr62v4+PgYERERxvr16wt9/SdOnDBeffVVo0GDBkapUqUMHx8fo1atWsbTTz9tzJw507h69aphGIZx5MgRIz4+3mjZsqVRoUIFw9PT0wgODjY6depkLF++vEjjAgC3i8UwbpjpCgAA3HE5OTmqUaOGrly54nArAFBUCQkJeuutt7Ru3Tq1adPG7HIAACgUc0oAAHAHZWVl2e/3zmMYhsaMGaPffvtNPXr0MKkyAACAO485JQAAuIM2b96sXr16KTIyUtWqVdP58+e1efNmbdu2TVWqVLHPSwAAAPAgIJQAAOAOqlOnjrp166ZvvvlGy5cv17Vr1xQSEqKhQ4fq9ddfL9YkfwAAAPca5pQAAAAAAACmYE4JAAAAAABgCkIJAAAAAABgivtqTom0tDSzS7ipoKAgpaenm13GfYPxdB/G0r0YT/diPN2HsXQvxtO9GE/3YSzdi/F0L8bTve6F8axcuXKBy7hSAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmOK+evoGAAAAAAC3S05Oji5fvixJslgsJlfzuxMnTig7O9u0/RuGIQ8PD3l7e7u0PqEEAAAAAAA3kZOTo0uXLsnHx+euCSQkyWq1ysPDw9QaLl++rKtXr6pkyZJFXpfbNwAAAAAAuInLly/fdYHE3cLLy0tXrlxxaV1CCQAAAAAAbgGBhHPFGRdCCQAAAAAAboJAonCujg+hBAAAAAAAMAWhBAAAAAAA97Hc3FwNHz5cDRo0UHBwsDZt2mR2SXY8fQMAAAAAABflDIy+o/vzmLKkyOskJydr/vz5WrBggUJDQxUQEFBo/6ysLI0YMUKrV6+WJHXs2FFjxoyRv7+/SzUXhislAAAAAAC4jx0+fFg2m01hYWGy2Wzy9PQstH9cXJx27typWbNmadasWdq5c6eGDh16W2rjSgkAAAAAAO5Tw4YN04IFCyRJwcHBCgkJ0ebNmzV58mTNnDlTaWlpKlu2rGJiYhQfH6/9+/dr3bp1+vzzz9WsWTNJ0rhx49SjRw8dOHBAtWrVcmt9hBIAAAAAANynRo0apZCQEM2dO1fLly+Xh4eHEhMTNWPGDI0cOVLh4eE6ffq0du7cKUnaunWrfHx87IGEJIWFhal06dLaunUroQQAAAAAALg1fn5+8vX1lYeHh2w2my5cuKApU6YoISFBvXv3liRVr17dHkKcPHlS5cqVc3jEp8ViUVBQkE6ePOn2+phTAgAAAACAB8S+ffuUnZ2t1q1bm12KJEIJAAAAAADw/9hsNp0+fVqGYdjbDMNQenq6bDab2/dHKAEAAAAAwAOidu3a8vLy0saNG50ub9q0qS5cuKAtW7bY27Zs2aKLFy+qadOmbq+HOSUAAAAAAHhA+Pr6asCAAUpMTJSXl5fCw8OVmZmplJQUxcbGqnbt2mrbtq1ee+01jRs3TpL02muvqUOHDm6f5FIilAAAAAAA4IESHx8vf39/TZgwQceOHVNQUJBiYmLsyydNmqQRI0aob9++kqTIyEiNGTPmttRiMa6/UeQel5aWZnYJNxUUFKT09HSzy7hvMJ7uw1i6F+PpXoyn+zCW7sV4uhfj6T6MpXsxnu51r47nxYsXVbp0abPLyMdqteratWtml1Ho+FSuXLnA9ZhTAgAAAAAAmIJQAgJu4lwAACAASURBVAAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJQAgAAAAAAmIJHghZD99l7btpncd+6d6CS+8PNxpOxLBrG03041wEAAIDbgyslAAAAAACAKQglAAAAAACAKQglAAAAAACAKQglAAAAAAC4j+Xm5mr48OFq0KCBgoODtWnTJrNLsmOiSwAAAAAAXHQrk6K7kysTrCcnJ2v+/PlasGCBQkNDFRAQUGj/iRMnau3atfr555916dIlHT161NVyb4orJQAAAAAAuI8dPnxYNptNYWFhstls8vT0LLT/lStX1LlzZ73wwgu3vTaulAAAAAAA4D41bNgwLViwQJIUHByskJAQbd68WZMnT9bMmTOVlpamsmXLKiYmRvHx8ZKkv//975KkZcuW3fb6CCUAAAAAALhPjRo1SiEhIZo7d66WL18uDw8PJSYmasaMGRo5cqTCw8N1+vRp7dy505T6CCUAAAAAALhP+fn5ydfXVx4eHrLZbLpw4YKmTJmihIQE9e7dW5JUvXp1NWvWzJT6mFMCAAAAAIAHxL59+5Sdna3WrVubXYokQgkAAAAAAGASQgkAAAAAAB4QtWvXlpeXlzZu3Gh2KZKYUwIAAAAAgAeGr6+vBgwYoMTERHl5eSk8PFyZmZlKSUlRbGysJOno0aPKzMxUamqqJNknwaxevbp8fHzcWg+hBAAAAAAAD5D4+Hj5+/trwoQJOnbsmIKCghQTE2NfPn78ePtjRCUpKipKkrRgwQK1bNnSrbUQSgAAAAAA4KLFfeuaXcJNDR48WIMHD7b/XKJECcXFxSkuLs5p/wkTJmjChAl3pDbmlAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAAAKYglAAAAAAA4D6Wm5ur4cOHq0GDBgoODtamTZvMLsnOanYBAAAAAADcq5bOy7qj+3uyV0CR10lOTtb8+fO1YMEChYaGKiCg4G0cOXJEEyZM0KZNm3Ty5EnZbDZFR0dr2LBhKlWqVHFKd4pQAgBwR3WfvafQ5Yv71r1DlQAAADwYDh8+LJvNprCwsJv2PXDggHJycjR27FhVr15d+/fv16uvvqrMzEz985//dHtthBIAANzDCHnc52ZjKTGeRcF4uhfnunsxnu7DuX73GzZsmBYsWCBJCg4OVkhIiDZv3qzJkydr5syZSktLU9myZRUTE6P4+Hi1bdtWbdu2ta8fGhqqIUOGaPz48YQSAAAAAADg1o0aNUohISGaO3euli9fLg8PDyUmJmrGjBkaOXKkwsPDdfr0ae3cubPAbZw/f77QWz6Kg1ACAAAAAID7lJ+fn3x9feXh4SGbzaYLFy5oypQpSkhIUO/evSVJ1atXV7NmzZyun5qaqo8++khDhgy5LfXx9A0AAAAAAB4Q+/btU3Z2tlq3bn3TvqdOnVLfvn31xBNPaNCgQbelHkIJAAAAAADg4OTJk/q///s/1alTR++//74sFstt2Q+hBAAAAAAAD4jatWvLy8tLGzduLLDPiRMnFBMTo9q1a+vDDz+U1Xr7Zn5gTgkAAAAAAB4Qvr6+GjBggBITE+Xl5aXw8HBlZmYqJSVFsbGxOn78uGJiYlSxYkUlJCQoIyPDvm65cuXk4eHh1noIJQAAAAAAeIDEx8fL399fEyZM0LFjxxQUFKSYmBhJ0oYNG3To0CEdOnRIzZs3d1hv8+bNqlKliltrIZQAAAAAAMBFT/Zy36MyD5y+VOjyWuVKubTdwYMHa/DgwfafS5Qoobi4OMXFxeXr26tXL/Xq1cul/biCOSUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApCCUAAAAAAIApivX0jZUrV2rJkiXKyspSSEiI+vXrp3r16hXYf9euXZo+fbpSU1MVGBio6OhoRUZGOvTJzMzU7Nmz9dNPP+ny5cuy2WwaOHCg6tevX5xSAQAAAADAXcblUGLTpk2aNm2aBgwYoLp162rVqlV655139N577ykoKChf/5MnT2rs2LFq27athgwZoj179uiTTz6Rn5+fWrRoIUm6cOGCRowYobp16yo+Pl5+fn46ceKE/Pz8XH+FAAAAAADgruRyKLFs2TJFRESoQ4cOkqT+/ftr27ZtWrVqlfr06ZOv/6pVqxQYGKj+/ftLkkJCQnTgwAEtXbrUHkosXrxYgYGBDs9KtdlsrpYIAAAAAADuYi6FEteuXdPBgwf15JNPOrQ3bNhQe/fudbrO/v371bBhQ4e2Ro0aacOGDbp27ZqsVqt++OEHNW7cWO+9955+/vlnBQYGqn379oqKipLFYnGlVAAAAAAAcJdyaaLLs2fPKjc3V/7+/g7tAQEBysrKcrpOVlaWAgICHNr8/f2Vk5Ojc+fOSfr9Fo9Vq1apQoUKeuONN9SlSxfNnj1bK1eudKVMAAAAAAAeeLm5uRo+fLgaNGig4OBgbdq0yeyS7Io10aW75ebmqmbNmvbbP6pXr65jx45p5cqV6tSpU77+a9as0Zo1ayRJiYmJTueyMNuNNVmt1ruyznuBs3FjPF3HselejKf7cK67F8emezGe7sV4ug9j6V6Mp3vdL+N54sQJWa35v0K/++67d7SOV155JV+bs7qut2rVKs2fP1+fffaZQkNDFRAQUOA6ubm5io2N1c8//6z09HT5+/vr8ccf14gRI1SpUqUC9+Hl5eXS++pSKOHn56cSJUrozJkzDu3OrobI4+wqijNnzsjDw0NlypSRJAUGBiokJMShT0hIiFasWOF0mx06dLDPaSFJ6enpRX4tt9uNNQUFBd2Vdd4LnI0b4+k6jk33Yjzdh3PdvTg23YvxdC/G030YS/diPN3rfhnP7OxseXh4mF2Grl275vCz1WrN13ajX375RTabTU2aNClwO3lyc3PVsmVLxcXFqUKFCjp27JhGjx6tfv366YsvvihwH9nZ2QW+r5UrVy5wPZdu37BarapRo4ZSUlIc2nfs2KE6deo4Xad27drasWOHQ1tKSopq1KhhT2jq1KmjtLQ0hz5paWn3ZIoGAAAAAIDZhg0bpoSEBB09elTBwcEKDw+XYRj66KOP1KpVK1WvXl1NmzbV2LFjJUklSpTQwIED1bRpU4WEhCgsLExxcXHatm2bLl++7Pb6XAolJKlbt25av369kpOTlZqaqqSkJGVkZKhjx46SpEmTJmnSpEn2/pGRkcrIyNC0adOUmpqq5ORkrV+/3mGyzK5du2r//v1atGiRjh8/rm+//VYrVqxQVFRUMV4iAAAAAAAPplGjRumvf/2rKlWqpJ9++knLly9XYmKiJk6cqCFDhmjt2rWaPHlygbdmZGZmatGiRWrSpIm8vb3dXp/Lc0q0bNlS586d06JFi5SZmakqVaooPj5e5cuXl5T/Eh2bzab4+HhNnz7d/njQ559/3v44UEmqVauW/v73v2vOnDn63//+p6CgIPXq1YtQAgAAAAAAF/j5+cnX11ceHh6y2Wy6cOGCpkyZooSEBPXu3VvS7/M5NmvWzGG9t99+W0lJSbp06ZIeffRRzZgx47bUV6yJLqOiogoMDBISEvK11a9fX+PGjSt0m48++qgeffTR4pQFAAAAAACc2Ldvn7Kzs9W6detC+7300kvq3bu3jh49qnfffVdDhgzRrFmzZLFY3FrPXfX0DQAAAAAAYL6yZcuqbNmyqlmzpmrVqqWwsDB9//33Cg8Pd+t+XJ5TAgAAAAAA3Ftq164tLy8vbdy48ZbXMQxD0u9P2HA3rpQAAAAAAOAB4evrqwEDBigxMVFeXl4KDw9XZmamUlJSFBsbqy1btmjnzp0KCwuTv7+/Dh8+rPHjx6tKlSpq3ry52+shlAAAAAAA4AESHx8vf39/TZgwQceOHVNQUJBiYmIkSd7e3lq2bJnGjx+vS5cuyWazqU2bNvrvf/97dz19AwAAAACAB93QoUPdtq0Dpy8VurxWuVIubXfw4MEaPHiw/ecSJUooLi5OcXFx+fo+/PDDWrhwoUv7cQVzSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAAFMQSgAAAAAAcB/Lzc3V8OHD1aBBAwUHB2vTpk1ml2RnNbsAAAAAAADuVbYD8e7b1s06ZEona40t8naTk5M1f/58LViwQKGhoQoICLil9S5fvqxu3bpp9+7dWr58uRo1alTkfd8MV0oAAAAAAHAfO3z4sGw2m8LCwmSz2eTp6XlL640ePVqVKlW6rbURSgAAAAAAcJ8aNmyYEhISdPToUQUHBys8PFyGYeijjz5Sq1atVL16dTVt2lRjxzpegbFy5Upt2rRJb7755m2tj9s3AAAAAAC4T40aNUohISGaO3euli9fLg8PDyUmJmrGjBkaOXKkwsPDdfr0ae3cudO+TlpamuLj4zVz5kx5e3vf1voIJQAAAAAAuE/5+fnJ19dXHh4estlsunDhgqZMmaKEhAT17t1bklS9enU1a9ZMkpSTk6MhQ4Zo0KBBatCggY4cOXJb6+P2DQAAAAAAHhD79u1Tdna2Wrdu7XT5+++/r5IlS+rFF1+8I/VwpQQAAAAAAJAkffPNN/ruu+8UGhrq0P7kk08qOjpakyZNcuv+CCUAAAAAAHhA1K5dW15eXtq4caNq1KiRb/m7776rixcv2n8+ceKE+vTpow8++EBhYWFur4dQAgAAAACAB4Svr68GDBigxMREeXl5KTw8XJmZmUpJSVFsbKyqVq3q0N/Hx0eSVK1aNVWuXNnt9RBKAAAAAADwAImPj5e/v78mTJigY8eOKSgoSDExMabUQigBAAAAAICLTtYa67ZtHTh9qdDltcqVcmm7gwcP1uDBg+0/lyhRQnFxcYqLi7vpulWqVNHRo0dd2u+tIJQw2fvvv1/o8qFDh96hSu4PjKf73GwsJcazKDg2AQAAgPx4JCgAAAAAADAFoQQAAAAAADAFoQQAAAAAADAFoQQAAAAAADdhGIbZJdzVXB0fQgkAAAAAAG4BwYRzxRkXQgkAAAAAAG7C29tbFy5cIJhwIjs7W56eni6tyyNBAQAAAAC4CQ8PD5UqVUoXL16UJFksFrfvY++xM4Uur1wqfyDi5eWl7Oxst9dyqwzDkIeHh0qWLOnS+oQSAAAAAADcAg8PD/n4+Ny27f9/238rdHnXhyvlawsKClJ6evrtKum24/YNAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCkIJAAAAAABgCmtxVl65cqWWLFmirKwshYSEqF+/fqpXr16B/Xft2qXp06crNTVVgYGBio6OVmRkpNO+n332mebMmaOoqCgNGDCgOGUCAAAAAIC7kMtXSmzatEnTpk1Tjx49NG7cONWpU0fvvPOO0tPTnfY/efKkxo4dqzp16mjcuHF66qmnlJSUpM2bN+fru2/fPq1Zs0ahoaGulgcAAAAAAO5yLocSy5YtU0REhDp06KCQkBD1799fgYGBWrVqldP+q1atUmBgoPr376+QkBB16NBBERERWrp0qUO/ixcv6oMPPtBLL70kHx8fV8sDAAAAAAB3OZdCiWvXrungwYNq1KiRQ3vDhg21d+9ep+vs379fDRs2dGhr1KiRDh48qGvXrtnbJk+erPDwcD388MOulAYAAAAAAO4RLoUSZ8+eVW5urvz9/R3aAwIClJWV5XSdrKwsBQQEOLT5+/srJydH586dkyStWbNGx48fV+/evV0pCwAAAAAA3EOKNdGlO6WlpWnOnDkaPXq0rNZbK2vNmjVas2aNJCkxMVFBQUG3s0SX3FiT1WotUp1342syi7OxYDxdV9xj09k2HmSc6+7jjnMdf3DHuY4/MJ7uxXi6D2PpXoynezGe7nM//p7kUijh5+enEiVK6MyZMw7tzq6GyOPsKoozZ87Iw8NDZcqU0fbt23Xu3Dm98sor9uW5ubnavXu3Vq9erZkzZ6pkyZIO63fo0EEdOnSw/1zQJJtmurGmoKCgItV5N74mszgbC8bTdcU9Np1t40HGue4+7jjX8Qd3nOv4A+PpXoyn+zCW7sV4uhfj6T736u9JlStXLnCZS6GE1WpVjRo1lJKSoscee8zevmPHDoWHhztdp3bt2vrhhx8c2lJSUlSjRg1ZrVaFhYXpX//6l8Py//73v6pYsaJ69Ohxy1dPAAAAAACAe4PL3/S7deumDz74QLVq1VKdOnW0evVqZWRkqGPHjpKkSZMmSZLi4uIkSZGRkVq5cqWmTZumDh06aO/evVq/fr1efvllSZKPj0++p214eXnJ19dXVatWdbVMAAAAAABwl3I5lGjZsqXOnTunRYsWKTMzU1WqVFF8fLzKly8vKf9lJTabTfHx8Zo+fbr98aDPP/+8WrRoUbxXAAAAAAAA7knFuiciKipKUVFRTpclJCTka6tfv77GjRt3y9t3tg0AAAAAAHB/cOmRoAAAAAAAAMVFKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExBKAEAAAAAAExhLc7KK1eu1JIlS5SVlaWQkBD169dP9erVK7D/rl27NH36dKWmpiowMFDR0dGKjIy0L//ss8/0/fffKy0tTVarVbVr11afPn1UtWrV4pQJAAAAAADuQi5fKbFp0yZNmzZNPXr00Lhx41SnTh298847Sk9Pd9r/5MmTGjt2rOrUqaNx48bpqaeeUlJSkjZv3mzvs2vXLkVGRmr06NEaOXKkPDw8NHr0aJ0/f97VMgEAAAAAwF3K5VBi2bJlioiIUIcOHRQSEqL+/fsrMDBQq1atctp/1apVCgwMVP/+/RUSEqIOHTooIiJCS5cutfd544031LZtW1WtWlVVq1bVkCFDdPbsWe3Zs8fVMgEAAAAAwF3KpVDi2rVrOnjwoBo1auTQ3rBhQ+3du9fpOvv371fDhg0d2ho1aqSDBw/q2rVrTte5dOmSDMOQr6+vK2UCAAAAAIC7mEtzSpw9e1a5ubny9/d3aA8ICNCOHTucrpOVlaVHHnnEoc3f3185OTk6d+6cAgMD862TlJSkatWq6aGHHnK6zTVr1mjNmjWSpMTERAUFBbnycm6rG2uyWq1FqvNufE1mcTYWjKfrintsOtvGg4xz3X3cca7jD+441/EHxtO9GE/3YSzdi/F0L8bTfe7H35OKNdHl7TR9+nTt3btXo0aNUokSzi/o6NChgzp06GD/uaD5LMx0Y01BQUFFqvNufE1mcTYWjKfrintsOtvGg4xz3X3cca7jD+441/EHxtO9GE/3YSzdi/F0L8bTfe7V35MqV65c4DKXbt/w8/NTiRIldObMGYf2rKwsBQQEOF0nICBAWVlZDm1nzpyRh4eHypQp49A+bdo0ffPNN3rzzTdVoUIFV0oEAAAAAAB3OZdCCavVqho1aiglJcWhfceOHapTp47TdWrXrp3v1o6UlBTVqFFDVusfF2wkJSXZA4ng4GBXygMAAAAAAPcAl5++0a1bN61fv17JyclKTU1VUlKSMjIy1LFjR0nSpEmTNGnSJHv/yMhIZWRkaNq0aUpNTVVycrLWr1+vJ5980t7n448/1vr16/Xyyy/L19dXWVlZysrK0uXLl4vxEgEAAAAAwN3I5TklWrZsqXPnzmnRokXKzMxUlSpVFB8fr/Lly0vKf6+LzWZTfHy8pk+fbn886PPPP68WLVrY++Q9TnTUqFEO68bExOiZZ55xtVQAAAAAAHAXKtZEl1FRUYqKinK6LCEhIV9b/fr1NW7cuAK3N3/+/OKUAwAAAAAA7iEu374BAAAAAABQHIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAADAFIQSAAAAAAD8/+zdeVhWdf7/8ReKJKYCioh6u47KqAmT+SvGvaKNcskKs7RxKS11HGua0qux1CzMLq00LDUzdTRNG3NNu9Asl6/W5MLighsxoIKIN+IKCL8/vDjDLesNt35An4/r8qr75pxzf86bcz7nnNf9OQcYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGCEu+kGAACQ35pl9kLedXyvZz/vm9MYAAAA3FCMlAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGuJtuAAAAzpoxY0axPx89evRNagkAAADKg5ESAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABghHt5Zt64caNWr14tu90um82mQYMGqU2bNkVOv3//fi1YsECJiYny8fFRr1699PDDD5drmQAAAAAAoHIq80iJHTt26KuvvtKTTz6pDz74QAEBAXr//feVmppa6PQpKSkKDw9XQECAPvjgA/Xp00fz58/Xzp07y7xMAAAAAABQeZU5lFi7dq26d++ukJAQ2Ww2DRkyRD4+Pvrhhx8Knf6HH36Qj4+PhgwZIpvNppCQEHXv3l1r1qwp8zIBAAAAAEDlVaZQIjs7W8eOHVNQUJDD+4GBgTp06FCh8xw+fFiBgYEO7wUFBenYsWPKzs4u0zIBAAAAAEDlVaZQ4ty5c8rJyZGXl5fD+97e3rLb7YXOY7fb5e3t7fCel5eXrl69qoyMjDItEwAAAAAAVF7letClaZGRkYqMjJQkTZkyRb6+vi5dfvKTnYr9+faVO0pcxvyII9e94xiwTJo0qdj5q+x8qcTPyAmeW+I0FUF561mwlpKr63mr1FIqSz2dq6V0+9STfd05rqjn9dzd3ZWdnZ3vndujnjdjXz+evLDEz5gcerLYn1eGWko3Z1+nnv9zM+pZUi2lylHPm7GvS+WvZ2WopUTf6Wrs6651M66JKtu+XqZQonbt2qpSpYrS09Md3i9suo9XygAAIABJREFUNESewkY8pKenq2rVqqpVq5YkOb3MkJAQhYSEWK9v9gMxXfF5JS3D7ya1oyKoCPW8VWoplX9dSjP/7VLPirBtuqodFUFZ1sPX19ep+ajn7dcGV6go61FR2lFeFWU9Kko7yqsirEdFaIOrVIR1qQhtcIWKsh4VpR3lVRHW40a0oWHDhkX+rEy3b7i7u6tFixaKiopyeD86OloBAQGFztOqVStFR0c7vBcVFaUWLVrI3d29TMsEAAAAAACVV5n/+sYTTzyhLVu2aNOmTUpMTNT8+fOVlpamhx56SJL06aef6tNPP7Wmf/jhh5WWlqavvvpKiYmJ2rRpk7Zs2aKePXuWepkAAAAAAODWUeZnSnTq1EkZGRn697//rbNnz6px48YaN26c6tWrJ6ngkA8/Pz+NGzdOCxYssP486ODBgxUcHFzqZQIAAAAAgFtHuR50+cgjj+iRRx4p9GcTJkwo8F7btm31wQcflHmZAAAAAADg1lHm2zcAAAAAAADKg1ACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMMLddANudT37eTu89vX1VWpqqqHWVG7X11KinuXBtula1BMAAABwHiMlAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADDCvSwz5ebmavny5dq0aZPOnz+vVq1aaejQoWrcuHGx8+3cuVPLli1TcnKy6tevr/79++vee++VJGVnZ2vp0qXau3evkpOT5enpqXbt2un555+Xr69vWZoJAAAAAAAqsDKNlFi1apXWrl2rwYMHKzw8XLVr19bkyZN16dKlIueJi4vTxx9/rK5du2rq1Knq2rWrpk+frsOHD0uSMjMzdfz4cfXt21cffPCB3njjDZ05c0bvvfeerl69Wra1AwAAAAAAFZbToURubq7Wr1+vPn36KDg4WE2aNNGoUaN06dIlbdu2rcj51q1bp3bt2qlv376y2Wzq27ev2rVrp3Xr1kmSatSoofHjx6tTp05q2LChWrZsqWHDhikpKUlJSUllX0MAAAAAAFAhOR1KpKSkyG63KzAw0HrPw8NDbdq00aFDh4qcLy4uTkFBQQ7vBQUFKS4ursh5Ll68KEm68847nW0mAAAAAACo4JwOJex2uyTJ29vb4X0vLy+lp6cXO5+Xl1eBefKWd73s7GwtWrRI99xzj+rWretsMwEAAAAAQAVX4oMut27dqjlz5livx40bd0MbJElXr17VjBkzdOHCBb3xxhtFThcZGanIyEhJ0pQpU1z+QMzkEn5els9zd3d3br4jJU9SWR4EWhnqeavUUnJ+XZyupXTb1LMibJtlbYcJ1NN1XLOvFx7+u1JlqKXkqm2Teuahnq7Dvu5a1NO12Nddq/z1vPVqWWIo0bFjR7Vq1cp6nZWVJenayIf8jU1PTy8wEiI/b2/vAiMp0tPTC4y4uHr1qj755BMlJCRowoQJqlWrVpHLDAkJUUhIiPU6NTW1pNVxqbJ8nq+vr1Pz+d2gdlREFaGet0otJefXxdlaSrdPPSvCtlnWdlRE1NO1KsJ6VIQ2uEJFWY+K0o7yqijrUVHaUV4VYT0qQhtcpSKsS0VogytUlPWoKO0or4qwHjeiDQ0bNizyZyXevuHp6Sl/f3/rn81mk7e3t6KioqxpMjMzdfDgQQUEBBS5nNatWzvMI0lRUVFq3bq19To7O1sfffSRfv/9d73zzjsFAgsAAAAAAHDrcPqZEm5ubgoNDdWqVau0a9cuJSQkaNasWapevbq6dOliTTdp0iQtWbLEeh0aGqqYmBh99913SkpK0sqVKxUbG6vHH39c0rUREnl/IvRvf/ub3NzcZLfbZbfblZmZ6YJVBQAAAAAAFUmJt28Upnfv3srMzNS8efN04cIFtWzZUm+99ZY8PT2taZKTkx0eUBkQEKAxY8Zo6dKlWrZsmfz9/TVmzBjr1pAzZ87oP//5jyRp7NixDp83YsQI9ejRoyxNBQAAAAAAFVSZQgk3NzeFhYUpLCysyGkiIiIKvBccHKzg4OBCp/fz89M333xTluYAAAAAAIBKyOnbNwAAAAAAAFyBUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwwt10AwAAgDmjR48ueaIj4258Q24R1NO1SqwntXQK9XQd9nXXYtt0rcpWT0ZKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABghLvpBlRkVeeuNt2EWwr1dB1q6VrUEwAAADCDkRIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACPcTTcAAABUbCktw0034ZZCPV2HWroW9XQt6uk61NK1Klo9GSkBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADDC3XQDAAC3lqpzV5tuAgAAACoJRkoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjHAvy0y5ublavny5Nm3apPPnz6tVq1YaOnSoGjduXOx8O3fu1LJly5ScnKz69eurf//+uvfeewudds6cOYqMjNSAAQPUq1evsjQTAAAAAABUYGUaKbFq1SqtXbtWgwcPVnh4uGrXrq3Jkyfr0qVLRc4TFxenjz/+WF27dtXUqVPVtWtXTZ8+XYcPHy4w7c6dO3XkyBH5+PiUpXkAAAAAAKAScDqUyM3N1fr169WnTx8FBwerSZMmGjVqlC5duqRt27YVOd+6devUrl079e3bVzabTX379lW7du20bt06h+lOnz6t+fPna/To0XJ3L9NADgAAAAAAUAk4HUqkpKTIbrcrMDDQes/Dw0Nt2rTRoUOHipwvLi5OQUFBDu8FBQUpLi7Oen316lV98skneuqpp2Sz2ZxtGgAAAAAAqEScHopgt9slSd7e3g7ve3l56ezZs8XO5+XlVWCevOVJ0jfffKNatWrp4YcfLlVbIiMjFRkZKUmaMmWKfH19SzWfSe7u7s6180jJk1SG9b5RXF1Paunk+lPPIrGvu9btWs/kUkxT8nrYi/1pZaiDq5RUz9LVgnrmoZ6uczP29dIt49ZA3+la7OuuVf563nr7eomhxNatWzVnzhzr9bhx425IQ2JjY7VlyxZ9+OGHpZ4nJCREISEh1uvU1NQb0TSX8vX1daqdfqWYpjKs943i6npSS+fWn3oWjX3dtahn0cq7HrdKHVzBFbWgnv9DPV2LeroWfafrsG261q1az4YNGxb5sxJDiY4dO6pVq1bW66ysLEnXRj7kT2DS09MLjITIz9vbW+np6Q7vpaenWyMuYmNjZbfbNWzYMOvnOTk5Wrx4sdavX6/PP/+8pKYCAAAAAIBKpMRQwtPTU56entbr3NxceXt7KyoqSi1btpQkZWZm6uDBgxowYECRy2ndurWioqIc/rxnVFSUWrduLUl65JFHFBwc7DDPe++9p86dOzuMhgAAAKXXs5/j7ZZlGRWF/6GerkU9XYdauhb1dC3q6TrX11Kq/PV0+kGXbm5uCg0N1apVq7Rr1y4lJCRo1qxZql69urp06WJNN2nSJC1ZssR6HRoaqpiYGH333XdKSkrSypUrFRsbq8cff1zStedLNGnSxOGfu7u7vL29ix3qAQAAAAAAKqcy/c3N3r17KzMzU/PmzdOFCxfUsmVLvfXWWw4jKpKTk1W3bl3rdUBAgMaMGaOlS5dq2bJl8vf315gxYxxuDQEAAAAAALePMoUSbm5uCgsLU1hYWJHTREREFHgvODi4wC0axSlsGQAAAAAA4Nbg9O0bAAAAAAAArkAoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBHuphsAAAAKV3XuatNNuKVQT9einq5DLV2LeroW9XQt6lkQIyUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACHfTDQAAwNVSWoabbgIAAABKgZESAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABjhXpaZcnNztXz5cm3atEnnz59Xq1atNHToUDVu3LjY+Xbu3Klly5YpOTlZ9evXV//+/XXvvfc6THPixAktWbJEMTExys7OVqNGjfTXv/5VNputLE0FAAAAAAAVVJlGSqxatUpr167V4MGDFR4ertq1a2vy5Mm6dOlSkfPExcXp448/VteuXTV16lR17dpV06dP1+HDh61pUlJSNH78ePn5+entt9/WtGnT1K9fP1WvXr0szQQAAAAAABWY06FEbm6u1q9frz59+ig4OFhNmjTRqFGjdOnSJW3btq3I+datW6d27dqpb9++stls6tu3r9q1a6d169ZZ03z99dcKCgrSCy+8oBYtWqh+/frq0KGDfH19y7Z2AAAAAACgwnI6lEhJSZHdbldgYKD1noeHh9q0aaNDhw4VOV9cXJyCgoIc3gsKClJcXJwkKScnR7/99ptsNpvee+89DR06VOPGjdOOHTucbSIAAAAAAKgEnA4l7Ha7JMnb29vhfS8vL6Wnpxc7n5eXV4F58pZ37tw5Xb58WStXrlRQUJDGjx+vzp07a8aMGdq9e7ezzQQAAAAAABVciQ+63Lp1q+bMmWO9Hjdu3A1pSE5OjiSpY8eOeuKJJyRJzZo109GjR7VhwwZ16NChwDyRkZGKjIyUJE2ZMqVS3Obh7u7uXDuPlDxJZVjvG8XV9aSWTq4/9SwS+7prlWn7RKGopWtRT9einq5DLV2LeroW9XStyl7PEkOJjh07qlWrVtbrrKwsSddGPuRf8fT09AIjIfLz9vYuMJIiPT3dGnFRu3ZtVa1atcBf2WjUqFGRt3CEhIQoJCTEep2amlrS6hjn6+vrVDv9SjFNZVjvG8XV9aSWzq0/9Swa+7prlWX7ROGopWtRT9einq5DLV2LeroW9XStylDPhg0bFvmzEm/f8PT0lL+/v/XPZrPJ29tbUVFR1jSZmZk6ePCgAgICilxO69atHeaRpKioKLVu3VrStXTnD3/4g06cOOEwzcmTJ1WvXr2SmgkAAAAAACoZp58p4ebmptDQUK1atUq7du1SQkKCZs2aperVq6tLly7WdJMmTdKSJUus16GhoYqJidF3332npKQkrVy5UrGxsXr88cetaXr16qUdO3YoMjJSp06dUmRkpHbs2KFHHnmknKsJAAAAAAAqmhJv3yhM7969lZmZqXnz5unChQtq2bKl3nrrLXl6elrTJCcnq27dutbrgIAAjRkzRkuXLtWyZcvk7++vMWPGONwacu+992r48OFauXKl5s+frwYNGmjkyJGFPk8CAAAAAABUbmUKJdzc3BQWFqawsLAip4mIiCjwXnBwsIKDg4tddo8ePdSjR4+yNAsAAAAAAFQiTt++AQAAAAAA4AqEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAI9xNNwDFS2kZbroJtxTq6VrU03WoJQAAAG5HjJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIQgkAAAAAAGAEoQQAAAAAADCCUAIAAAAAABhBKAEAAAAAAIwglAAAAAAAAEYQSgAAAAAAACMIJQAAAAAAgBGEEgAAAAAAwAhCCQAAAAAAYAShBAAAAAAAMIJQAgAAAAAAGEEoAQAAAAAAjCCUAAAAAAAARhBKAAAAAAAAIwglAAAAAACAEYQSAAAAAADACEIJAAAAAABgBKEEAAAAAAAwglACAAAAAAAYQSgBAAAAAACMIJQAAAAAAABGEEoAAAAAAAAjCCUAAAAAAIARhBIAAAAAAMAIt9zc3FzTjQAAAAAAALcfRkrcZGPHjjXdhFsK9XQdaula1NO1qKfrUEvXop6uRT1dh1q6FvV0LerpWpW9noQSAAAAAADACEIJAAAAAABgRNUJEyZMMN2I202LFi1MN+GWQj1dh1q6FvV0LerpOtTStaina1FP16GWrkU9XYt6ulZlricPugQAAAAAAEZw+wYAAAAAADCCUAIAAAAAABhBKIHbRmxsrMLCwnTu3DnTTQFQDhEREZoyZYrpZtxS6B9hwsSJE/XTTz+Zbkahdu/erX/84x/Kycm5aZ85cOBAbdmy5aZ9njPCwsK0c+dO080ok2+++UZ///vfb+pnjhw5UqtXr76pnwkzv+vb3d///nd988035V6OuwvaUqnZ7XZ999132r17t86cOSNPT0/5+/urc+fOuv/++1W9evVC59uyZYvmzZunRYsWlfqzYmNjNXHiRH3xxReqXbu2q1ahUrLb7Vq5cqVV91q1aqlp06Z69NFH1aFDB9PNq/QiIiKUkZHh8r9ZXJbtvqKJiIiwToKrVq2qO++8U40bN9Z9992nkJAQubu7rlssa71SUlI0atQol3TyN1P+2lapUkU+Pj7q0KGD+vfvr5o1a7rscwYPHqzK9DikytDfBQQEaM6cOapVq5bpppTKzdyPTQoLC9Nrr72m4OBgly97//79WrNmjY4dO6azZ89qxIgR6tGjh8M0ly9f1pIlS/TLL78oIyNDvr6+euihh/TEE09I+l9fVZgBAwaoV69eRX7+7t27lZqaqq5du1rvRUZGavv27Tp+/LguXryoTz/9VH5+fg7zHTt2TIsXL9bRo0dVpUoV3XffffrLX/7icL4WHR2tZcuWKSEhQXfccYe6d++u/v37q2rVqpKuXbisWLGi0HbNnTtXXl5e6tChg5YtW6Zt27apW7duRReyEPm3z/xatWql9957z6ll3WgjR47U6dOni/x527ZtVZGfiV/Y+c5vv/2mjz76SE888YSeffZZ9erVS4899tgN+fzKcl5U1uudimLLli2aNWtWsdO88847N6k1RTN5vL+Rx4ub4dY4apdRSkqKxo8frxo1aqhfv35q2rSpPDw89N///lebNm1SrVq11KVLF9PNvOXk1d3T01P9+/dXs2bNlJOTo5iYGM2dO1efffZZgXmys7NvmZNMmNe+fXv99a9/VU5Ojs6dO6eYmBgtX75cW7du1fjx4yv8wbkiy6vt1atXlZiYqM8++0wXLlzQmDFjXPYZNWrUcNmybrSy9HcmuLu7y9vbu8if531bXKVKxRlgWVn344pSy8uXL6tx48bq3r27Pv3000KnWbBggaKjozVq1Cj5+fnpwIEDmj17tmrXrq1u3brJ19dXc+bMcZjnl19+0bx580o8MV6/fr169OjhUIcrV64oMDBQHTt21IIFCwrMk5aWpnfffVd//vOfNXToUF28eFELFixQRESE9e1ofHy8wsPD1adPH40aNUppaWmaO3eucnJy9MILL0iSevXqpYcffthh2R9//LHc3Nzk5eVlvXf//ffr+++/dzqUkP63feZXEc9jwsPDrW0yPj5e77//vt5//335+vpKqphtLs7PP/+szz//XAMGDFBoaKgkqXr16hW2P7gZboXrnU6dOulPf/qT9XrmzJmqWbOmBg8ebL1Xs2ZNxcbGmmiepMpzvK+oKldP42JffPGFqlSpovDwcIfOys/PT/fcc88N/yYuJydHs2fPVkxMjOx2u+rWrasHH3xQPXv2tA7SeQlwYGCgVq1apczMTP2///f/NHToUN1xxx2SpNzcXK1evVqRkZFKS0uTv7+/evfuXaaD6M0wb948SdKUKVMc6m6z2axvTMLCwjRkyBDFxMRo3759euihhzRgwIAS65WQkKCvvvpKR48eVU5Ojvz9/fWXv/xFd911l/U5v//+u77++mslJCTIZrNp2LBhlfpP6Dhr//79+te//qXff/9dNWrUUOfOnTVgwADrxGP//v1avHixEhISVKVKFTVs2FCvvPKKMjIyrJQ6LCxMkvT0009b/1+ZVKtWzboAq1Onjpo1a6bAwEC9+eabWr16tcLCwvTzzz/r+++/V1JSkjw8PNS2bVsNGjRIderUkfS/kU/jx48vdHuKjY0tsl7Z2dlaunSptm3bpvPnz6tx48bq16+fwwE3v4sXL2revHnat2+fLl26JB8fHz322GN6/PHHb0K1nJO/tnXr1lWnTp2s4cil6fOuXr2qRYsWWd8ydu/eXVlZWUpKSrK+rbv+m7EJEybIZrOpRo0a2rRpk9zc3NStWzcNGDDAWq7dbtfs2bMVFRUlLy8vPfPMM1q7dq3uu+++G7oNl6a/S01N1fz58xUdHS1JCgwM1ODBg1W3bl1J177V3bVrl/r27aulS5cqPT1dd911l15++WVr1F1xfV/etvrmm29q6dKlOnHihGw2m4YPH271fdeP5Mv79u/VV1/V4sWLlZSUpKlTpyozM1NLly7V8ePHlZ2drSZNmmjgwIFq3bq1tW4XL17U4sWL9euvv+rChQvy8/PTM888ow4dOmj48OF65ZVXHC5Yo6KiFB4ers8++6zYYOR6pdmPS9rXsrOztXDhQu3atUsZGRny8vJSly5d9Pzzz0u69i1y9+7dderUKf3666+qXr26evbs6TAC4OLFi1q0aJF+/fVXZWZmqnnz5nrhhRf0hz/8QZLKXMuRI0dKkqZPny5JqlevniIiIiRJ//nPf7R8+XIlJibK29tbXbp00TPPPOPUBWSHDh2sb+7ylnu9uLg4devWzTqG+vn5afPmzTp8+LC6deumKlWqFPid7dq1S+3bty8wwiG/c+fOKTo6WgMGDHB4P69PO3r0aKHz7d69W1WqVNGLL75o7dsvvfSSXn/9dZ06dUr+/v7asWOHbDabtV/7+/vr+eef10cffaRnnnlGnp6eBS7V4T7dAAAgAElEQVRSU1NTdeDAgQIhQseOHfXll19ay3ZG/u2zMKdOndLnn3+uw4cPy9fX1wpM8jt8+LC++OILJSYmqlGjRnr22Wc1ZcoUvfPOO2rXrp0kKTExUYsWLdKBAwfk4eGhu+66S4MGDSr1vpR/5G7eSKnatWsXOv/58+c1ffp07dmzR15eXgoLC3M410xLS9PChQu1b98+SVLr1q01aNAgNWjQoFRtKa9169Zp8eLFevnllx3aldeHTps2TVLx/WVJfUJ+xR3nJSkrK0tz5szR9u3b5enpqdDQUKf6Dldx5npn7dq12rJli5KTk1WjRg3dfffdGjhwoO68805J/+vPXnvtNS1YsECpqalWABcVFaUlS5YoPT1dHTt21PDhw+Xh4SGp/NcqHh4e1rKka/uXh4dHkdv59u3bizxeStKPP/6o1atXKyUlxRoBFhoaWq6wuDTHe6n0x/zQ0FCtWLFC586dU1BQUIF1cEZGRobmzZungwcPKiMjQ/Xr11fPnj11//33W9OU5lwqPT1ds2fP1r59++Tl5aWnn366TO0pzG0bSmRkZGjfvn3q379/kempm5vbDW1DTk6O6tSpo1dffVW1a9fWkSNHrOGzDzzwgDXdgQMH5O3trfHjx+vMmTP66KOP1KBBAz355JOSpKVLl2rnzp0aOnSoGjZsqLi4OM2ePVs1a9asMEOD85w/f1579+5Vv379Cq17XqcnSStWrFD//v01cOBAubm5lapen3zyiZo2bar3339fVatWVUJCgkMnJklLlizR888/Lx8fH3311VeaOXOmpk+ffsN/3xVBWlqawsPD1bVrV40YMULJycn6/PPPVaVKFb3wwgu6evWqPvzwQ91///3Wt93Hjx9XlSpVFBAQoEGDBunrr7/WzJkzJemW+uahSZMm+tOf/qRdu3ZZFzPPPPOMGjVqpIyMDC1evFiffPKJJk6c6DBfUdtTcfWaNWuWkpOTNXr0aNWtW1d79uzRBx98oPDwcDVr1qxA25YuXaqEhASNHTtWXl5eSklJqRT3/icnJ2vv3r3WkOnS7MNr1qzRTz/9pOHDh6tJkybauHGjtm3bpubNmxf7WVu3blVoaKjeffddxcfHa8aMGWrRooX17U9ERITsdrveeecdeXh4aOHChcUOWXaF0vR3OTk5mjp1qjw8PKyhp19++aU+/PBDhYeHW/1SSkqKduzYoddff11XrlzRxx9/rKVLl2rYsGGSStf3LVq0yArWVqxYoSlTpmjmzJlWwH29rKwsffvtt3rppZdUu3Zt+fj46OjRo+rWrZsGDRokNzc3bdiwQeHh4ZoxY4Zq1aql3NxchYeH6/z58xoxYoQaNGigEydOKCsrS9WrV1fnzp31448/OoQSmzdvVocOHZwKJIpy/X5c0r72/fff69dff9Xf/vY3+fn56cyZMzpx4oTDMtetW6fevXvr6aefVmxsrL788kvVr19f9913n7W+NWrU0NixY1WzZk1t2bJFkyZN0scffywfH58y1zI8PFwvvviihg8frnvuucc6Kdy7d69mzpypQYMGqU2bNkpNTdXcuXOVlZVV6IVteQQEBOi3337TAw88IF9fXx06dEjx8fFF3paRnJysmJgYvfrqq8Uu9+DBg3J3d1eTJk2cak9WVpaqVq3qcOGQt50fPHhQ/v7+ys7OVrVq1Rzm8/DwUFZWlo4dO2ZdzOe3efNm1axZU/fdd5/D+76+vvLy8tL+/fudDiWKk5OTow8//FA1a9bU5MmTdeXKFX311VfKzs62prl8+bKmTJmiwMBAjRo1SmfPntVXX33lsJyzZ8/qnXfe0f3336+BAwfq6tWr+vrrrzV16lRNnjzZ5aNxVqxYoeeee07PPfecNm/erM8++0xt27aVr6+vrly5ookTJ6p169aaMGGC3N3dtWbNGr377rv66KOPiuxnXGXp0qVau3atXn/99RLPfYvrL0vTJ+Qp6bxo3bp1CgsLU69evbRnzx7Nnz9ff/zjH9W6detS9x3l5ez1jpubmwYNGiQ/Pz+lpqbqyy+/1JdffukQ2GVnZ2vt2rUaPXq0srOzNW3aNE2bNk3VqlXT3//+d2VkZGjatGnauHGjevbsKenmXquUdLyMjIzUN998oyFDhqhFixZKSEjQ7Nmz5e7urkcffbRMn1na6xtnjvlbt27VG2+8oStXrmjOnDn67LPP9Oabb5apfVlZWWrRooX69OkjT09PRUdHa86cOfL19VX79u2t6Uo6l5o1a5ZOnz6t8ePH64477tCCBQuUkpJSpjZd77YNJU6dOqXc3Fw1bNjQ4f2XX35ZFy5ckCR17drV2oBvBHd3d/Xr18967efnp+PHj2v79u0OoUSNGjU0bNgwValSRTabTcHBwYqJidGTTz6py5cva+3atfrnP/+pNm3aWMs5cuSINm7cWOFCiby622y2Eqft1KmTHnzwQYf3SqpXamqqevbsqUaNGklSoScR/fr1s771eeqpp/T2228rLS3NSihvZRs3bpSPj4/1LZPNZtPzzz+vOXPmqF+/fsrKytKFCxfUsWNHq3Z5tZT+N2zeFRcPFZHNZrOS6/z7YP369fXiiy/q1Vdf1ZkzZxy2leK2p8LqderUKW3fvl0RERHW8NhHH31UUVFRioyM1Isvvig/Pz+H50mcPn1azZs3V8uWLSVd+8a0otq7d68GDhyonJwcZWVlSZJ1oVSaPm/9+vXq3bu3ddE6aNAg7d27t8TPtdls1rIbNmyoTZs2KSYmRl26dNGJEye0b98+TZ482foWesSIEdY30TdKafq7mJgY/f7775o5c6b1zfLo0aM1evRoRUdHKzAwUNK1E5mRI0da21RISIh+/PFHazml6fueeuopa4TAiBEj9PLLL2vbtm0F+tk8OTk5Gjp0qMNIsvyjziRpyJAh2rVrl/bs2aNu3bopOjpacXFxmjZtmrXe9evXt6Z/8MEH9dZbbyktLU116tTR+fPn9euvv+q1114rskbOytuPS7OvnT59Wg0aNFCbNm3k5uYmX19fBQQEOCyvZcuW6tu3r6Rr29bRo0etUTaxsbGKj4/XvHnzrIuaZ599Vr/99pt+/vln9e7du8y1zPtG7M4773ToQ1auXOnwDVfeSICZM2daIb6rDBkyRHPmzNGIESOscHHw4MG65557Cp1+06ZNql27tjp27Fjsck+fPi0vLy+nL5rvuusuLVy4UN99952eeOIJXb58WYsXL5Z07QJdkoKCgrRu3Tr9/PPP6ty5s9LT0/Xtt986TJNfTk6OfvzxR3Xt2rVAmCFdG4VTlpPuvL4wv0ceeUQDBgxQdHS0EhMTHbbNQYMG6e2337am3bp1q3JycvTKK6/Iw8NDjRs3Vt++fTVjxgxrmh9++EFNmzZ1GHEyatQoDRkyRMeOHbOOGa7SrVs365vtfv36af369dq/f7+6deum7du3Kzc3VyNGjLC2wWHDhunFF1/Ub7/9pk6dOrm0LflFRUVp9+7dGjt2bKnOe4vrL0vTJ+Rxd3cv9rwoMDDQush97LHH9P333ys6OlqtW7cudd9RXs5e7+Qfgenn56cBAwZo6tSpGjlypMOIxrxwQZI6d+6sdevWae7cuVa/1bFjR8XGxqpnz543/VqlpOPlt99+qwEDBljnGX5+fkpOTtbGjRvLHEqU9vqmtMf8zMxMjRo1yuofhg0bprffflsnT54s08ijOnXqOITJ9evXV0xMjLZv3+4QSpR0LrVnzx5NmjRJf/zjHyVdG9FX1HOFnHXbhhJFmTRpkjXEOO+E+kb64YcftHnzZp0+fVqZmZm6evVqgQsOm83mcOCuU6eOjhw5IunasL2srCy9//77DvMUtpyKwJlbYgq7paKkej3++OOaPXu2fvrpJ7Vv31733Xefw0W1JDVt2tT6/7yh+Onp6bdFKJGUlKRWrVo5bE9//OMflZ2drVOnTqlp06bq0aOH3nvvPd11111q3769goODrU7xVpebm2udTB07dkwrVqxQfHy8zp8/b227qampDtuKs9vT8ePHlZubW+CbxOzs7AIXKXkefvhhTZ8+XcePH1f79u3VsWNHtW3btuwregO1adNGw4cPV2ZmpiIjI5WcnGzd1ysVvw9fvHhRdrvd4UTazc1NLVu21JkzZ4r93Py/B0ny8fFRenq6pGvbvZubm8OQWF9fX+v3daOUpr9LTExUnTp1HIa6169fXz4+PkpMTLROUHx9fR2epeHj4+MwWqY0fV/+WyyqV6+uJk2aKDExsci2Va1atcDInfT0dC1btkyxsbGy2+3KyclRZmamUlNTJV3bvr29vYs8MfvDH/6gJk2aaMuWLerbt6+2bdummjVr6u677y6hUqWXtx+XZl/r0aOHJk+erL/97W8KDAxUhw4d9Kc//cmhj8xft7zXu3btknStn8jMzNTQoUMdpsnKylJycrL1uiy1LMqxY8d05MgRrVq1ymGdMzMzZbfbXfYNq3TtW+NDhw7pjTfeUL169XTgwAEtWrRIfn5+BW43u3r1qrZs2aLu3buXeBtJZmZmoQFASRo3bqyRI0dqwYIF+vrrr1W1alU99thj8vLysvruoKAgDRw4UPPmzdOsWbNUrVo1PfXUUzpw4EChIcjevXt15swZhYSEFPqZHh4eyszMdLqteX1hfnn7cFJSkurUqeNwbG3ZsqVDoJSUlKQmTZo4jHi6PmQ4duyYDhw4UCD8kK5dJLk6lMg/sqVq1aqqXbu21Q8dO3ZMKSkpBUbrZGZmOuwLN0Ljxo118eJFLV++XAEBAQ6jbgtTXH9Zmj6htIo7LpW277hRirreiYmJ0cqVK5WUlKSLFy8qJydH2dnZstvt1jGzWrVqDiGHt7e3vL29HW4t8PLyso4vN/tapbjj5blz53TmzBnNmTNHc+fOtabJyckp1237pZ23tMf8ovqHpKSkMoUSOTk5+u6777Rjxw6lpaUpKytL2dnZBUaOleZcKn+/Uq9ePZedS922oYS/v7/1y80vbyO50cPMJGnHjh1asGCBdQ9pjRo1tGHDBv36668O0+V9O5Ff3saf998333yzwIVjYfOZ1qBBA7m5uSkxMVH33ntvsdNeP/ypNPUKCwtT165dtWfPHu3bt0/Lly/XSy+95PCtd3H1vJ3lnQyNGDFCoaGh2rt3r/7zn//o66+/1j/+8Y8in3dwK0lMTJSfn58uX76s9957T+3bt9eoUaPk5eWljIwMvf322w7DayXnt6e8C6bw8PACJ+7XD7fPc/fddysiIkJ79+5VdHS0wsPD9ec//1kjRowow1reWHfccYf1rdOQIUM0ceJErVixQmFhYaXu88ri+t+Dm5ub8f3amf6uMPkvUK7fVvJuactTmr7PWe7u7gVOxCMiIpSenq6//OUvqlevnqpVq6ZJkyYV2C+K88ADD+j7779X37599eOPP6p79+4uHWaetx+XZl9r0aKFIiIitG/fPkVHRysiIkJNmzbVP//5z1K1KScnR15eXpo0aVKBn3l6elr/78pa5uTk6Omnn9af//znAj9z5V/2yszM1JIlS/Taa69ZIx+aNm2q+Ph4rVmzpsAx4bfffpPdbi/VNlerVi3rW1pndenSRV26dJHdbrfOE9auXeswIueJJ57Q448/rrNnz6pmzZpKSUnRkiVLCn3ORWRkpAICAooM0s6fP1+muubvC2+U3Nxc3X333YXetpP/gZ2uUlw/lJubq2bNmhX6UGNX/vWlwvj4+OjNN9/UxIkT9e677+qf//xnsZ9ZXH9Z3j4hv+KOS6XtO8rLmeud06dPKzw8XA8++KD69eunmjVr6vjx4/rkk08c+qXC6lDYtmHqWqW47TTvvy+99FKRI2DKorzH+7x23iirV6/WmjVrNHjwYDVp0kTVq1fXkiVLCtwKXJpzqRvVzorzGO2brFatWgoMDNSGDRt0+fJlI204ePCgWrZsqUcffVQtWrSQv7+/0+mozWZTtWrVdPr0afn7+zv8q4gjJWrWrKmgoCBt3Lix0LoXd5JS2no1aNBAoaGhGjdunB544AFt3rzZpetQmTVq1EiHDx92uJjJu7c3/wlds2bN1KdPH02YMEHt2rWzHjro7u5+U/9m+82UkJCgffv2KTg4WCdOnFBGRoaee+45tW3bVo0aNbKSYmcUVq9mzZopNzdXdru9wD5bXNqc97T7kSNH6pVXXtFPP/10U0ZzldfTTz+tVatWKS0trcR9uEaNGvL29rZGgknXTmaKeuhdaTVq1Ei5ubk6duyY9d6ZM2eUlpZWruWWpDT9nc1mU1pamsPw8OTkZJ09e7ZUt7nlV1Lfd/jwYev/L1++rP/+978FRlOU5ODBg9afNmvcuLGqV6/uMCS+efPmstvtxY7A6Nq1q86cOaMNGzbo+PHjDg/aKq/8+3Fp9zVPT08FBwfrpZde0tixYxUTE6NTp05ZP89fN+nawx/zfjctWrRQenq63NzcCnxGSReFJdVSunaCeH0f0qJFCyUlJRX4PH9/f5ee4GdnZ+vq1asFLkCqVKlS6HFg06ZNatu2bYFh4oVp3ry5zp07V65n43h7e6t69erasWOHPDw8rG8Y87i5ualOnTry8PDQ9u3bVbdu3QIjMNPS0rR79+4ib2HKzMzUqVOnXP4w7EaNGiktLc1hVMyRI0ccTv4bNWqkhIQEh1Ea+ftG6VodExMT5evrW2BbcOWFbWk0b95cp06dUq1atQq05UaHEtK1b5YnTJigK1eu6N1331VGRkax0xfXX5bUJ+RX1vOi8vQdznDmeufo0aPKzs7WoEGD1Lp1azVs2LDQW56cVZGuVby9veXj46Pk5ORC+9CyKu31TWmP+UX1D84es/McPHhQ99xzj7p166ZmzZqpfv36OnnypFPLyDuXyt8Ppaamuuxc6rYNJSTpxRdfVG5urt58801t27ZNiYmJOnHihLZt26bff//d4UD86aefFvkns6RrG8+YMWP0yy+/lPrzGzRooOPHj2vPnj06efKkVqxYof379zu1Dp6enurZs6cWLVqkzZs369SpU4qPj9cPP/ygyMhIp5Z1swwdOlS5ubkaO3as/u///k8nTpxQUlKSfvjhB73++utFzldSvTIzM/XFF18oNjZWKSkpOnz4sA4ePOj0if2t4tKlS4qPj3f4d/fdd+vs2bPW07x3796txYsX69FHH9Udd9yhlJQULV68WIcOHdLp06ete9/yalivXj1lZWUpKipK586d05UrVwyvZdlkZWXJbrcrLS1N8fHxWrt2rSZOnKgWLVqoZ8+e8vX1VbVq1bRhwwYlJydr9+7dWrZsmdOfU1i9GjZsqC5dumjWrFnauXOnkpOTdfToUa1evdoaEn69ZcuW6ZdfftHJkyeVmJioXbt2yc/Pr0zDn2+2du3ayWaz6d///nep+rzQ0FCtXr1av/zyi06cOKGFCxfq7Nmz5UrmGzZsqKCgIM2dO1dxcXGKj4/XrFmzdMcdd9zwB9yW1N+1b99eTZs21cyZM3X06FEdPXpUM2bMUPPmzYu8ned6pe37vv32W0VFRem///2vPvvsM7m7uzv9Z+AaNGigrVu3KjExUUeOHNEnn3zi8K3UXXfdpZYtW2ratGnau3evUlJSFBUV5XBsvPPOOxUcHKyFCxeqTZs2ZX4yf0n7cWn2tbVr11rH/1OnTmnbtm3y9PR0uP3q8OHDWrlypU6ePKnIyEj9/PPP1n3X7du3V0BAgKZOnao9e/YoJSVFcXFx+uabb3TgwIFy1VK69m1mdHS07Ha7zp8/L+nas0G2b9+uZcuWKSEhQUlJSdq5c6f+9a9/OVW/y5cvW8eG3NxcpaamKj4+3joRrlGjhtq2baslS5ZY29aWLVv0008/FfgmMDU1VXv37i3y4v56zZs3l5eXlw4ePOjwvt1uV3x8vHWynJiYaN1Cl2fD/2/v3uN6vP/Hjz86vVU6RwdatdaJHBIzPrIkp/E1G6k5Hz7MYj528BG2fTA24+trbGsTFsohmbWUz+IjYcLwRQ5FChNJKaXQ4V3v3x9+XV9v75riXe9387rfbt3oel/X9X5dz17X67qu1/U6JCZy5coVcnJySExMJCIiglGjRik12d+1axfXr18nOzubn376iV9++YVJkyapVLAkJyfTokWLWludwKO/vYGBwTO9Ua3Jn4//1FTCdOzYkbZt2xIWFsa1a9fIyMhg06ZNSpVKvr6+6OrqsmbNGm7cuMHZs2eJjY0F/u9N5cCBA3nw4AGrVq3i8uXL3L59m7NnzxIeHs7Dhw8bnObn0bt3b8zNzVm+fDlpaWnk5eWRlpZGZGRkgx9+npWlpSULFixALpfz+eef11rp9bTysj5lwuOe9b7oecqOhqrv8469vT0KhYLdu3eTl5fH4cOH2b1793N/v7Y9qwQFBREXF0dCQgI5OTlcv36dgwcPSufXs6rP8019r/kymUypfFi3bh0+Pj5PvV7m5eWp3Ps/ePCANm3acP78eS5evMjNmzf58ccfGzxWTps2bfD29mbt2rXSvVRYWFidrXwb6oXtvgGP+vAsX76c2NhYtm/fTkFBAXp6ejg4ODBgwAClwU6e1sdTLpeTk5PDgwcP6lynpga85qLTv39/aWRThULBa6+9xtChQ5UGY6mP4OBgzM3NiY+PZ/369RgZGeHs7Ky2QXLUzdbWlmXLlhEbG8uWLVsoLCzE1NQUJycnlf6Xj3tavHR1dbl//z7ff/89d+/exdTUFB8fn1r7Wr4I0tPTmTNnjtKy1157jXnz5rF582bmzJlDy5Yt6dWrF6NGjQIeFYK3bt1i5cqV0lRYvXv3lvKSh4cH/fv3Z/Xq1ZSUlDTbKUHPnTsnDR7bsmVLXnrpJUaOHEm/fv3Q19fH0NCQGTNmsG3bNvbs2YOjoyPjx49X6Q/5NHXFa/r06fz8889s3ryZgoICTExMcHV1rfMh1MDAgOjoaPLy8jAwMMDd3f2ZR2DWhKFDh/L999+zevXqp5Z5Q4cOpaioiO+//x4dHR369OlD9+7dn6mlyuNmzJjBmjVrWLRoEWZmZgQHB0vxbExPK+90dHSYM2cOERER0swuHTt2ZPLkyfWuMKlv2TdmzBgiIyPJycnhpZdeIjQ0tMEz6ISEhLB27VpCQ0OxsrJi5MiRSjf+urq6zJ8/n6ioKL799lvKysqkKUEf17dvXw4dOvRc3Uuedh4DTz3XDA0NiY+P59atW+jo6ODs7Mz8+fOVmjQPGTKEP/74g59//hlDQ0OCgoKkAdJ0dHSYN28e0dHRhIeHU1xcjIWFBR4eHk+d6u5psQQYN24ckZGRhISEYGVlRVhYGN7e3sydO5edO3cSHx+Pnp4e9vb29OnTp0Hxy8rKUppNKCYmhpiYGPz8/KRBYD/44AO2bt3KN998Q2lpKa1btyY4OFhlMLj9+/djbGysMntFXXR1dfH39+fw4cNKFRx79+7lp59+kn7/6quvgEd/x5rjy8zMJCYmhrKyMtq2bcu7776rEuvTp0/z888/U1lZibOzM3PmzFEZt0ShULB//3569+5dZ5fdlJQUfH19n6lLb03+fJyVlZU049Xs2bMJDw9n/vz50pSgq1evltY1MjIiNDSU9evXM2fOHBwcHBg5ciQrV66Uyi0rKysWL17M1q1b+fLLL6moqKBVq1Z07txZWqdmut/HpxFtDC1atGDRokVs3bqVlStX8uDBAywtLfHy8nrqGA/qZGFhwYIFC1i8eDGLFi1SGjwUnl5e1qdMeNyz3hc9T9nRUPV93nFycmLixInExcURHR2Nh4cH48aNY9WqVc+dhvo8q9RM+13zb2MJCAigRYsWxMfHs23bNmQyGQ4ODs88yGWN+jzf1Peab2NjQ69evVi2bJnSlKBPU1vldGhoKMOHDycvL48vv/wSmUxGnz596N2795+2aqzN9OnTCQ8Pl+6lAgMD1TYbnI5C051uXyCHDx9mzZo1DX6bIQiC8CKbM2cOnp6eTJ48WW37vHfvHtOmTWPWrFlK01P+FdU8lKxfv16tYw48jyNHjrB27VrCw8ObZAynZzVjxgwGDhxY5xSYwrMrLi7mo48+YunSpbWO9aBpxcXFfPjhh3z11Vdak74TJ06wYsUKpVkOniY5OZmtW7eyatWqJq0cEISGmj59Ov379+ftt9/WdFI0KiYmht9//53/+Z//0XRSmtQL3VKiqdT0Sfz111+Vpl0RBEEQlOXn55Oamkr79u2Ry+UkJSXxxx9//Gkrqvo4f/48Dx8+xNHRkeLiYqKjozEzM3shBnDVJuXl5RQVFREbGyu9rRJeTObm5oSEhHDnzh2teeh/XH5+vjRFs6YcOHAAW1tbrK2tyc7OZuPGjXTt2rVBlYunT59mzJgxokJC0GrZ2dkYGBgwdOhQTSdF0BBRKdEEjhw5woYNG/Dw8FCZ+kcQBEH4Pzo6Ohw8eJCoqChpzu/58+crTef5LORyudQFRiaT4ebmxqJFixrcfUF4PnFxccTGxuLp6cmIESM0nRxBw2pm9dBGrq6uap9Ss6GKi4vZsWMHd+/excLCAh8fH8aMGdOgfXz00UeNlDpBUJ+XXnpJqfuS8OIR3TcEQRAEQRAEQRAEQdCIF3r2DUEQBEEQBEEQBEEQNEdUSvyJ0tJSpk6dWufcxJoUFRVFRESEppPRINocz5UrVxIfH6/pZDRIXl4eQUFBZGVl1XubAwcOvLCzkWijmJgYPv74Y00no1kLCgri2LFjdf7+V/Us578gCNpLm++RmuM9pzrMmDGDXbt2/ek6L0pZLPKn+ohY1k6MKfEnYmNj6dKlC3Z2dgBs2LCBS5cukZ2djYWFBWFhYSrbHDlyRJrP3MzMjEGDBqmM2p2YmMiePXvIy8ujVatWDB8+HD8/P+nzhQsXkpaWprJvBwcHVq5cCcCwYcOYOXMmQ4YMwdbWVp2H3Wg0FU+ABw8eEB0dze+//05JSUsDXlgAAB+2SURBVAnW1taMGjWKv/3tbwAEBgayYMECAgICMDY2bqQI1F9YWBglJSXMnTtXaXlWVhbz5s3ju+++o1WrVqxduxZTU1MNpfKvLywsjIMHDwKPpvK1trame/fuBAUFibEIntPjsQUwNTXFzc2NcePG0bZtWw2mTPOeNp2cn5+fyhSbL7KavOTv709ISIjSZ5s3b2bXrl34+PiolKeCoE3EPWfTKioq4pdffuHUqVMUFBRgZGSEnZ0dvXr1wt/fH0NDQ5YuXSoG4/3/RP5UHxHL2olKiTqUl5ezf/9+QkNDpWUKhQI/Pz+uX7/O2bNnVbY5ffo033zzDZMmTcLb25ubN28SHh6OTCaT5r7du3cvW7ZsYdq0abi5uZGZmUl4eDgtW7aUBnyaPXs2crlc2m9lZSWzZ8+mZ8+e0jIzMzM6derE3r17m8Wbb03GUy6Xs2TJEkxMTPjwww+xsrKisLBQmscewNHREVtbWw4dOvTc8xQ3FV1dXSwsLDSdjL+8jh07MnPmTORyORcvXmTNmjWUl5czdepUTSet2auJLUBhYSGbN29mxYoVfP311xpOmWatXbtW+v///u//Eh4errRMJpNRWlqqiaQBj8rUx8tPbWBtbc3Ro0eZNGmSVGFYVVXFoUOHaNWqlYZT93TaGFOh6Yh7zqaVl5fHZ599hrGxMcHBwTg5OSGTycjOziYpKQlTU1N8fX2fOsvJ43H7KxP5U31ELOsmroB1OH36NAAeHh7SssmTJwOwa9euWjPNoUOH6Nq1KwMHDgTA1taWt956i7i4OAYOHIiOjg6HDh0iICAAX19faZ2srCzi4uKkTGNiYqK0399++43y8nL8/f2Vlnfr1o1t27Zp/QkImo3ngQMHuHfvHp9//rl001fbFF/dunUjJSWl2VRK5OXl8f7777N06VJpZoJTp06xadMm7ty5g6urKwMGDGD16tV89913Ssd87tw5Nm7cSF5eHq6uroSEhGBjY0NZWRmTJk1i0aJFuLu7AxASEkKLFi1YtWoVAGfPnuW///u/2bBhA/r6+iQkJHDgwAFu376NsbExXbp0Ydy4cbRs2ZKysjKmTZtGSEgIPXr0kL7/7NmzLF26lB9++EHrK1YMDAykNPr6+nL+/HlOnDjBlClT2LVrF/v27aOwsBA7OzuGDRvG66+/Lm27ZcsWjh8/zp07d7CwsKBnz54EBQUhk8lq/a47d+6wZMkS6W+ip6fXJMeoKY/H1sLCgiFDhrBs2TIqKiooKipSyd/wqBXBRx99pJSf/moePydqpvF78jypqZTIz89n69atXLp0idatWzNp0iQ6deokrXfjxg2ioqJIT09HJpPRoUMHJk6cKO2vurqan3/+maSkJIqLi7G3t+edd97h1VdfBf6vnPnHP/5BUlISGRkZjBkzhu3bt2vVee3k5MTdu3c5evSodK08deoUBgYGtGvXTqkSJzk5mV27dklvk/r378/gwYPR1X3Uo/XPyjR41PLuxx9/JDU1lYcPH2Jpackbb7zBkCFDgNrz6IwZMxg4cKD0ZisoKIjJkydz/vx5UlNT6d+/P+PHj+fkyZPs2LGDGzduYGFhga+vLyNHjpSuXb///js7duzg1q1byGQyHB0d+fDDD7W+HBX+nLjnbFrr169HV1eXpUuXKrV6tLGxoWvXrtTMAVCf8/bJe0a5XE5kZKTUMtfc3BxfX98Gz5qiTUT+VB8Ry7qJSok6pKen4+Ligo6OTr23qaysxMDAQGmZTCajoKCA/Px8bGxsqKysVHkgkclkZGZm1vmmJCkpCW9vb5W3Pa6urhQWFpKbmys1AdJWmozniRMn8PDwICIighMnTmBiYkLPnj0ZPny4UrxdXV3ZuXMnFRUVdT40arM7d+6wYsUKBg4cSP/+/bl+/TqbNm1SWU8ul/PLL78QEhKCgYEBYWFhrFu3jk8++QRDQ0NcXFxIS0vD3d2d3Nxc7t+/z7179ygqKsLCwkL6rCZ2Ojo6TJw4ERsbG+7cuUNERAQRERHMnDkTQ0NDevXqRXJystIN+v79+/Hx8WmWN9IymYyqqiqio6M5duwYf//732nTpg0ZGRmEh4djYmKCj48PAC1atCAkJAQrKytu3LjBunXr0NfX55133lHZ740bN/jiiy/o0aMH48ePb9C58lfw8OFDjhw5gqOjY7M8/zQlOjqasWPHMmXKFHbu3MmqVav4/vvvMTQ05O7duyxYsAB/f3/GjRtHVVUV27ZtY/ny5SxZsgRdXV3+/e9/Ex8fz9SpU3FxceG3335jxYoVLFu2DGdnZ+l7am5QairLbty4oXXntb+/P8nJydINVs3/b9++La2zb98+YmJimDx5Mi4uLly/fp3w8HD09fWlh4s/K9PgUcyvX7/O3LlzMTc3Jy8vj3v37jU4vT/99BOjRo1i3Lhx6OjocObMGb799lsmTpxIu3btuHPnDuvWraOyspLx48dTVFTEqlWrGD16NK+99hplZWVcvnxZDZETNE3cczadkpISUlNTGTVqVJ3dMP/s7/DkefukX3/9lRMnTjBr1ixsbGwoKCggJydHbenXBJE/1UfEsm5ioMs65OfnY2lp2aBtvL29OXnyJKmpqVRXV5OTk0NCQgLwqO8aQOfOnUlOTiYzMxOFQkFWVhZJSUlUVVVRUlKiss+cnBzS0tIICAhQ+awmffn5+Q09vCanyXjevn2bY8eOIZfLmTdvHsHBwfznP/9h69atSt9naWlJVVUVhYWFajji53fmzBnGjRun9LNgwYI619+7dy+2trZMmDCBNm3a0KNHD/r376+yXlVVFX//+99xdXXFycmJoUOHcuHCBenNQPv27blw4QIAFy5cwNPTEzc3N86fPy8ta9++vbS/IUOG0KFDB2xsbGjfvj1jx47l6NGjVFdXAxAQEEBqaqoU19LSUk6cOEHfvn3VE6gmlJmZSUpKCl5eXiQkJPDee+/h7e2NjY0Nvr6+BAQEsGfPHmn9wMBAPD09sbGxwcfHh7fffpuUlBSV/V6+fJkFCxbQv39/JkyY8MJUSDyexydMmEBaWhr/+Mc/NJ2sZmXIkCF069YNe3t7Ro8eTWlpKdeuXQMelQlOTk6MHTsWBwcHnJyceP/998nMzOTKlSsAxMfHM3ToUHx9fWnTpg3BwcG0a9dOZXC3QYMG0aNHD2xsbLC2ttbK89rX15esrCxu3bpFUVERZ86coU+fPkrr7Ny5k7Fjx0rH0q1bN9566y2l8/ZpZVp+fj4vv/wyrq6utG7dGi8vL6Xmr/X1t7/9jYCAAGxtbbGxsSE2NpahQ4fi7++PnZ0dHTp0YMyYMfznP/9BoVBQWFhIVVWVlHZHR0cCAgKaZeWuoEzcczad3NxcFAoFbdq0UVr+3nvvSdejx7vLPenJ8/ZJ+fn52Nvb065dO1q1aoWHh4fKm+jmRuRP9RGxrJtoKVGH2mqcniYgIIDc3FyWL19OVVUVRkZGDB48mB07dkgPGYGBgRQVFfHZZ5+hUCgwNzfHz8+PXbt21fogkpSUhKWlpfTm9XE16auoqHiGI2xamoynQqHAzMyM9957D11dXVxcXCgtLWXTpk1KNd3aFs927doxbdo0pWXXr19nxYoVta5/8+ZNpWbuAG5ubirrGRgYKF2MLS0tkcvl3L9/HxMTE7y8vEhMTEQul3PhwgW8vLwoLy8nLS2NV199laysLKVmiOfPnyc2NpabN2/y4MEDqqurkcvlFBUVYWVlxSuvvIKjoyMHDhxg+PDhHD58GBMTE7p06fI84WkyNQ/ONcf16quvMnToUI4dO8aXX36ptG5VVRWtW7eWfj927Bi7d+8mNzeXsrIyqqurpQebGoWFhSxevJjAwECVQYv+6h7P46Wlpezdu5cvvviCL774QsMpaz6cnJyk/9fcSBQXFwNw5coV0tPTa22CmZubS5s2bbh7965SM1IAT09PqYlpjSfLFm08r01MTOjevTvJyckYGxvj5eWl9Abo3r17FBQUsHbtWtatWyctr66ulipl4ell2oABA1i5ciVXr16lY8eOdOvWTamitr5cXFyUfr9y5QqZmZnExcVJyxQKhdSdydnZmY4dO/Lxxx/TqVMnOnXqRI8ePZ7a713QfuKeU/M+//xzqqurCQ8Pp7Kyss71njxvn9SnTx+WLFnCrFmz6NSpEz4+Pnh7e0vdw5ojkT/VR8SybqJSog6mpqYNHkhMR0eHsWPHMnr0aIqKijAzM+PcuXMA0gimMpmM6dOn8+6771JcXIylpSX79u3DyMhI5cZCLpdz8OBBAgICau1bXpO+5nBDosl4WlhYoK+vr3RBaNu2LeXl5ZSUlEjraVs8W7RoodJs6v79+8+93ycvjDWFVc3DsqenJ3K5nKysLNLT0xk8eDDl5eWsXbuWS5cuoaenh6urK/CoFnXp0qUEBAQQHByMiYkJV69eZfXq1UqD6fTt25dff/2V4cOHk5ycjJ+fX7O5QNc8OOvp6WFpaYm+vr7UZDo0NFSl2VvNuZqRkcGqVasIDAxkwoQJtGzZkpMnTxIVFaW0vqmpKa1btyYlJYW+ffuq9Pn7K3syj7u4uDBhwgT27dtHv379AJQeFl+UQcUa4vFrw+MVsTX/dunShfHjx6tsZ25urhTbp6ltBHptPK/9/f0JCwvD0NCQ4OBgpc9qyripU6eqVMTUqE+Z1qVLF8LCwjhz5gznzp1j6dKl9OzZk+nTpwOP/g5Pxra2vPtk0/Hq6moCAwNrbXVhZmaGrq4un376KZcvXyY1NZX9+/ezdetWFi5cqNTVRmh+xD1n07Gzs0NHR4ebN28qLa9p9fC02TaeNvOWi4sLYWFhpKamcu7cOcLCwnBycuLTTz/VePn4rET+VB8Ry7o1z7OjCTg7O6sUWPWlq6uLlZUV+vr6pKSk4O7urvKH1dfXx9raGl1dXVJSUvDx8VEprI4fP05JSUmdzWGzs7PR09PD0dHxmdLZlDQZTw8PD3Jzc5XeUN+6dYsWLVooTaeZnZ2NlZVVs20K27ZtW5V5sjMzMxu8n5pxJZKSknjw4AEuLi64ublx584dDh8+rDSeRFZWFnK5nIkTJ+Lu7i69eX1S7969KSgoIDExkatXrzarpow1D86tW7eWjtvBwQEDAwPy8/Oxs7NT+qlpKXHp0iWsrKwIDAzE1dUVe3v7WpvCGRgYEBoaiomJCUuWLFFLxVNzpqurS0VFhXSO1zRNBKRuCUL9vPzyy9y4cYNWrVqp5FMjIyOMjY2xtLTk0qVLSttdvHgRBweHp+5fG8/rjh07oq+vT0lJiTRYZw0LCwssLS25ffu2SjxqKsfqW6aZmZnx+uuvM2PGDEJCQjh48KD0dtXMzExpm6KiIqV8XBcXFxdu3rxZa9pqbhx1dHRwd3dn5MiRLF26FEtLS44cOfLM8RK0g7jnbDqmpqZ06tSJxMREysrKGuU7jIyM6NGjB1OnTmXu3LmcP3+e3NzcRvmupiDyp/qIWNZNtJSog7e3N1u2bKGkpER6cK1pgn337l3kcrl0g+zg4IC+vj737t3j2LFjtG/fHrlcTnJyMkePHmXRokXSfnNycsjMzMTNzY379++TkJBAdnY2M2bMUElDUlISHTp0qHOe2PT0dNq1a9cs5lDWZDwHDBjAnj172LhxI4MGDSIvL4+YmBgGDBig1KQpPT2dzp07N01AGkH//v1JSEggMjKSfv36kZ2dzb59+4A/H7SpNu3btychIYHOnTujq6uLTCbDzc2N3377jcDAQGk9e3t7FAoFu3fv5rXXXiMjI4Pdu3er7K9ly5b06NGDyMhI2rVrh729/fMdrIYZGRkxdOhQoqKiUCgUtG/fnrKyMjIyMtDV1aVfv37Y29tTWFjIb7/9hru7O6mpqbWOJwGParhDQ0P56quvWLJkCZ9++qk00v9fWWVlpfSwVlpaKt0kdu3aVcpzcXFx2Nra8uDBA5VxYIQ/N3DgQJKSkli1ahXDhg3DzMyM27dvc/ToUcaPH4+RkRFvvvkmMTEx2NnZSQNdpqens2zZsqfuXxvPax0dHVasWIFCoVAZGAwejZ4fERGBsbExPj4+yOVyrl69SmFhIW+//Xa9yrTt27fz8ssv89JLL1FVVcXvv/+OjY2N9H1eXl7s2bMHDw8PdHV12bZtW61pedKIESNYtmwZrVu3pmfPnujp6ZGdnU1mZiZjx44lIyODc+fO0blzZywsLLh69SoFBQX1qkAStJu452xaU6ZM4bPPPiM0NJSRI0fi7OyMrq4uV65c4Y8//lCawaihEhISsLCwwNnZGX19fQ4fPoyRkRHW1tZqPIKmJfKn+ohY1k1UStTB0dERV1dXpSki16xZQ1pamrTOnDlzAJSmWzx48KDUPNvd3Z2FCxdKTd3hUfPMhIQEcnJy0NPTw8vLiyVLlqgMlnP79m3Onz/PrFmz6kxjSkoKQUFB6jngRqbJeLZq1YpPPvmEyMhI/vnPf2JhYYG/vz8jRoyQ1qmoqOD48eN88sknjReERta6dWs+/vhjIiMj2bNnD6+88gqBgYH88MMP9bohfpyXlxdxcXFK/aTbt29PWloaXl5e0jInJycmTpxIXFwc0dHReHh4MG7cOGn60Mf17duXQ4cONcsBLmsTHByMubk58fHxrF+/HiMjI5ydnRk2bBjwaEqlN998k40bN1JRUUHnzp0JDg5m/fr1te5PJpMxd+7cF6pi4ty5c7z77rvAo4qeNm3a8OGHH0p5LCQkhPDwcObNm4etrS1Tpkz508FeBWVWVlYsXryYrVu38uWXX1JRUUGrVq3o3LmzVCa88cYbPHz4kC1btlBUVESbNm34+OOP690dQBvPayMjozo/CwgIoEWLFsTHx7Nt2zZkMhkODg7Sdak+ZZqBgQHR0dHk5eVhYGCAu7u70pzz48ePZ82aNSxcuBALCwvGjBlTrzdj3t7ezJ07l507dxIfH4+enh729vbSYJ3GxsZcunSJxMRE7t+/j7W1NSNGjFCahlhonsQ9Z9OytbVl+fLlxMbGsn37dgoKCtDT08PBwYEBAwY819TwhoaGxMfHc+vWLXR0dHB2dmb+/Pla/7D8Z0T+VB8Ry7rpKBrSqfQFc+bMGTZs2MDXX3+tdf3ATp06RVRUFCtWrKi1P5A20uZ4JiYmcvLkST799FNNJ0Wt/v3vf7N9+3Y2btyo8Rkdjhw5wtq1awkPD2/WF2dBEP6POK8FQT20+R6pOd5zCuol8qf6iFjWTm/hwoULm/QbmxE7OzsUCgWWlpZa98by6tWr9OnTR2WQPW2mzfG8du0aAwYMUBpjojlKTEwEHg1ic/r0abZu3crrr7+u0RHxy8vLKSgoICIigp49e9Y60q8gCM2LOK8FQb20+R6pOd5zCuol8qf6iFjWTrSUEIS/kI0bN3L06FFKS0uxsrKiV69eBAYGSgM0akJMTAyxsbF4enryz3/+E2NjY42lRRAE9RDntSAIgiAI6iIqJQRBEARBEARBEARB0Ajt6sgiCIIgCIIgCIIgCMILQ1RKCEIzVFpaytSpU7Vy3uuoqCgiIiI0nYwGEfFUL22O57x58zh27Jimk1Fv2hxLkTfVa+XKlcTHx2s6GYLwl6DN53pzLDsFobGJKUEFoRmKjY2lS5cu2NnZAbBhwwYuXbpEdnY2FhYWhIWFqWxz5MgRYmNjuXXrFmZmZgwaNIg333xTaZ3ExET27NlDXl4erVq1Yvjw4fj5+UmfL1y4UGnaohoODg6sXLkSgGHDhjFz5kyGDBlS5xzI2kbEU70aK56HDx8mLi6OW7duYWRkRMeOHRk/fjwWFhbSOseOHWP79u3cvn0bW1tbRo0aRffu3aXPR4wYQWRkJN27d9e6Ua9rI/KmemkqnvBoNqS9e/eSn5+Pqakp3bp1Y+zYsRgaGgIQGBjIggULCAgIEGN0CMJzEmWnIDQvolJCEJqZ8vJy9u/fT2hoqLRMoVDg5+fH9evXOXv2rMo2p0+f5ptvvmHSpEl4e3tz8+ZNwsPDkclk0jzJe/fuZcuWLUybNg03NzcyMzMJDw+nZcuWdOvWDYDZs2cjl8ul/VZWVjJ79mx69uwpLTMzM6NTp07s3buXcePGNVYY1EbEU70aK54XL17k22+/Zdy4cXTv3p2ioiJ+/PFHvvnmG/71r38BkJGRwapVqwgKCqJ79+4cP36clStXsnjxYtzc3ADw8fEhPDycM2fOaP2MESJvqpcm43n48GE2b97Me++9h6enJ3l5efzwww9UVlYSEhICPJq/3tbWlkOHDkn7FgSh4UTZKQjNj/a/JhIEQcnp06cB8PDwkJZNnjyZN954A3t7+1q3OXToEF27dmXgwIHY2tri4+PDW2+9RVxcHDVj3R46dIiAgAB8fX2xtbWlV69e9OvXj7i4OGk/JiYmWFhYSD8XL16kvLwcf39/pe/r1q0bKSkp6j70RiHiqV6NFc+MjAysra35r//6L2xsbHB3d2fQoEFcvnxZ2s/u3bvx8vJi+PDhODg4MHz4cLy8vNi9e7e0jq6uLl26dOHw4cONcfhqJfKmemkynpcuXcLNzY3XX38dGxsbOnTogJ+fH5mZmUrf15ziKQjaSpSdgtD8iEoJQWhm0tPTcXFxQUdHp97bVFZWYmBgoLRMJpNRUFBAfn6+tI5MJlNZJzMzU6nW/3FJSUl4e3urzGfs6upKYWGhVvblfJKIp3o1Vjw9PT25e/cuJ0+eRKFQcO/ePY4cOUKXLl2kbTIyMujcubPSfjp37kxGRobSMldXV9LT0xt6aE1O5E310mQ8PT09uXbtmpQX79y5w8mTJ5XyLzyKZ2ZmJhUVFQ0+PkEQHhFlpyA0P6JSQhCamfz8fCwtLRu0jbe3NydPniQ1NZXq6mpycnJISEgAoKioCHj08JacnExmZiYKhYKsrCySkpKoqqqipKREZZ85OTmkpaUREBCg8llN+mou5NpMxFO9Giue7u7ufPDBB3z77beMHj2aKVOmoFAoeP/996X9FBUVYW5urrRvc3NzaR81rKysKCwspKqq6lkOscmIvKlemoxnr169GDVqFAsWLGDUqFFMnz4dR0dHxowZo/R9lpaWVFVVUVhYqIYjFoQXkyg7BaH5EWNKCEIzU1tN/dMEBASQm5vL8uXLqaqqwsjIiMGDB7Njxw7pTUJgYCBFRUV89tlnKBQKzM3N8fPzY9euXbW+bUhKSsLS0rLWfvk16WsOb/tEPNWrseJ548YNIiIiGDFiBJ07d+bu3bts3ryZtWvXKlVM1IdMJkOhUFBZWYmenl6Dtm1KIm+qlybjmZaWxs6dO5kyZQpubm7k5uayYcMGYmJiCA4Olr6vOcVTELSVKDsFofkRlRKC0MyYmppSWlraoG10dHQYO3Yso0ePpqioCDMzM86dOwcgjfwsk8mYPn067777LsXFxVhaWrJv3z6MjIwwMzNT2p9cLufgwYMEBATU+lBXk74nt9NGIp7q1VjxjI2NxdXVVRoJ3cnJCUNDQ/71r38xatQorK2tsbCwoLi4WGnfxcXFSrNzwKN4GhgYSLMeaCuRN9VLk/GMjo6mV69e0htTR0dHysrKCA8PJzAwUIptc4qnIGgrUXYKQvMjum8IQjPj7OzMzZs3n2lbXV1drKys0NfXJyUlBXd3d5ULor6+PtbW1ujq6pKSkoKPj4/K1InHjx+npKSEvn371vo92dnZ6Onp4ejo+EzpbEoinurVWPEsLy9XiVvN7zWDkLm7u6uMqn727Fnc3d2Vll2/fh0XF5dnSmNTEnlTvTQZz7ryb03erZGdnY2VlZVKRZogCPUnyk5BaH5ESwlBaGa8vb3ZsmULJSUlmJqaApCbm0tZWRl3795FLpdz7do14NG82Pr6+ty7d49jx47Rvn175HI5ycnJHD16lEWLFkn7zcnJITMzEzc3N+7fv09CQgLZ2dnMmDFDJQ1JSUl06NChzvm109PTadeuHS1atFB/ANRMxFO9Giue3bp1Izw8nL1790rdNzZt2sTLL78sDSA2ePBgFixYwC+//MKrr77K8ePHuXDhAp9//rlSGi9evKgyIKY2EnlTvTQZz65du7J7925eeeUVqfvG9u3b8fHxUXqLmp6e3izypiBoM1F2CkLzIyolBKGZcXR0xNXVlZSUFGnu7DVr1pCWliatM2fOHAC+++47bGxsADh48CBRUVHAozfKCxcuxNXVVdqmurqahIQEcnJy0NPTw8vLiyVLlkjb17h9+zbnz59n1qxZdaYxJSWFoKAg9RxwIxPxVK/GimefPn14+PAhiYmJREZGYmxsTIcOHZQGCvTw8OCDDz4gOjqa7du3Y2dnxwcffICbm5u0TmFhIZcuXWLmzJmNFwQ1EXlTvTQZzxEjRqCjo8P27dspKCjAzMyMrl278s4770jrVFRUcPz4cT755JPGC4IgvABE2SkIzY+O4sm2g4IgaL0zZ86wYcMGvv76a5Umg5p26tQpoqKiWLFihVYPIvg4EU/10uZ4RkVF8eDBA6ZNm6bppNSLNsdS5E31SkxM5OTJk3z66aeaToogNHvafK43x7JTEBqb3sKFCxdqOhGCIDSMnZ0dCoUCS0tLWrZsqenkKLl69Sp9+vRRmZNbm4l4qpc2x/OPP/5gyJAhWj/IZQ1tjqXIm+p17do1BgwYIDU3FwTh2Wnzud4cy05BaGyipYQgCIIgCIIgCIIgCBqhXe2ZBEEQBEEQBEEQBEF4YYhKCUEQBEEQBEEQBEEQNEJUSgiCIAiCIAiCIAiCoBGiUkIQBEEQBEEQBEEQBI0QlRKCIAiCIAiCIAiCIGiEqJQQBEEQBEEQBEEQBEEjRKWEIAiCIAiCIAiCIAga8f8AJ8lyv5c7sbAAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 732 + }, + "id": "8xp79OV8wwdh", + "outputId": "b8a17815-aa7f-4e6e-d818-4f9d3b13ad12" + }, + "source": [ + "plot_components(components_df, 'fc1', ascending=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAAMlCAYAAABEr0kcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3yMd97/8fdkJgdEMiFCJcQp4rCoolFJG4egVZQ2LaVtiFItuna33ZW7a5PqQdR9d1XdSrWNoAfSbYvSIlm0KD2pOBRRshVUKsmgSIhcvz/8MrfpJEgyMg6v5+Oxj+18r+/3e32uz8yVh/nMdX0vk2EYhgAAAAAAAKqZh7sDAAAAAAAANyeKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAwE1ixIgRMplMys7OdncoVVaZY1m3bp1MJpOSkpKqvP/s7GyZTCaNGDGiynPdTLKysjR48GA1aNBAJpNJVqvV3SEBLte9e3eZTCZ3hwEA1w2KEgDgAiaTSSaTSR4eHvrpp5/K7dejRw973/nz51dfgDcBCgWuVZUiTlnvw/nz5zVo0CCtXLlS/fv3V2JioiZNmlThuW02m6ZPn67hw4erTZs2slgsMplMSk9PL3dM9+7d1aRJkwrvC9cuVxYZAQDuZXF3AABwo7BYLCouLtbbb7+tl19+2Wl7VlaW1q1bZ+9X3aZOnapJkyYpODi42vftajfSsdwsDhw4oF27dmn06NF68803Kz1Pdna2/vrXv0qSQkJCFBgYqKNHj7oqTAAAUM24UgIAXKR+/frq3LmzUlJSyiw6vPXWW5KkAQMGVHdokqRbbrlFrVq1kqenp1v270o30rHcLA4fPixJatiwYZXmCQ0NVXp6uvLy8nTw4EHdfffdrggPAAC4CUUJAHCh0aNH65dfftGnn37q0H7u3DnNnz9f3bp1U5s2bcodn5WVpccee0zBwcHy8vJSw4YN9dhjjykrK8uh39ixY2UymbR06dIy59myZYtMJpNiY2PtbZdah2HLli2KjY1VgwYN5OXlpUaNGumJJ56wf5G82P79+zVmzBi1aNFCNWrUUJ06ddSuXTuNHTtWeXl5l0qPpAtfSsu6wiE0NFQmk0kvvPCCQ/tnn30mk8mkf/zjH+UeS1JSkpo2bSpJSk1Ntd8iU95tMj/88IPuvfdeWa1W1axZU9HR0dq0adNlY78SR44c0bhx49SkSRN5eXmpXr16uv/++/Xdd9859Fu1apVMJpOee+45h/a1a9faYz948KDDtiFDhshkMmn//v0O7bt379aIESPUqFEjeXl5qX79+ho2bJj27NnjFN/Ro0f1zDPPKDw8XLVq1ZLValV4eLhGjBhhn3fEiBHq0aOHJOn55593yOe6desqnBOTyaTo6Gin+S6+9P78+fOaM2eOIiMj5e/vrxo1aqhFixZ6/PHHHT7/AQEB6tWrl+rUqVPhOCpi9+7dio+PV5MmTeTt7a2goCDdeeedeuONN5z6ZmRk6O6771adOnXk7e2tli1batKkSTp+/LhT39L1Bs6dO6cpU6aoefPm8vHxUXh4uObNm2fvN2fOHLVr1041atRQSEiIEhMTVVJS4jDXxbcs7d69W4MGDVKdOnVUq1YtRUVFafXq1WUeW1FRkZKTk9WuXTvVrFlTfn5+uvPOO7VkyRKnvhfvIzs7W0OHDlVgYKB8fHzUuXNnp791F3v//ffVo0cPWa1W+fj4qHXr1nrxxRdVVFTk1NdkMql79+46duyYxowZo1tuuUXe3t5q27atUlJSHPpW9vN56NAhmc1mdezYsdw+99xzj0wmk3bs2GFvmz9/vh544AE1a9ZMNWrUkJ+fnyIjI7Vo0aJy5/m9+fPnX/K2vdLj/73i4mLNnj1bXbt2lZ+fn2rWrKmOHTtq1qxZTp8HSVq2bJl69eplz1/Dhg0VHR2t2bNnX3GsAFDduH0DAFzo4Ycf1p///Ge99dZbGjRokL192bJlys3N1bRp07Rv374yx37zzTeKiYnRyZMnNXDgQLVp00a7d+/WokWLtHTpUqWnp6tLly6SpLi4OM2dO1cLFizQfffd5zRXamqqJF3R+grvvPOOxowZI29vbw0cOFCNGjVSVlaW3nrrLS1fvlybN29W48aNJV34wt2lSxedOHFC/fr10wMPPKDCwkIdOHBACxcu1Pjx41W3bt1L7q9nz5569913tXv3brVq1UqStG/fPv3888+SLnzBmzx5sr1/RkaGJKlXr17lztm9e3fZbDa99tpr6tChg0Pub731Voe+3377rV555RXdcccdevzxx/Xzzz/rX//6l3r16qUffvhB4eHhl81ZeQ4cOKCoqCgdPnxYPXv21MMPP6yDBw8qLS1NK1as0L/+9S/1799fknTnnXfKy8tLGRkZeumll5yOt/S/S99DwzC0du1aNWnSRM2aNbP3+fzzz3X//ffr3LlzGjBggFq0aKGcnBx99NFHWrFihdauXavbbrtNknT69GlFRkbqp59+Uu/evTVgwAAZhqH//Oc/Wrp0qWJjY9WsWTN7/lJTUxUdHe3wZakyazMkJiYqOzvbab7S/z979qz69++vNWvWqFGjRho2bJj8/PyUnZ2tjz/+WFFRUQoLC6vwfitrxYoVevDBB1VUVKS7775bDz/8sGw2m7Zt26ZXXnlFTz75pL3v3Llz9eSTT6pWrVp68MEHFRQUpHXr1mnatGlavny5Nm7cWOaCnkOHDtWWLVvUr18/eXp66sMPP9SYMWPk6empzMxMpaamqn///urVq5eWLVumKVOmqGbNmvrb3/7mNNeBAwd0xx13qF27dnriiSd05MgRLV68WPfcc4/ee+89DRkyxN737Nmz6tu3r9avX69WrVpp3LhxOn36tD788EMNGTJEP/zwQ5m3n/3nP//R7bffrmbNmunRRx9Vfn6+Fi9erPvuu0/p6en2IkGp+Ph4paSkKCQkRA888ICsVqs2b96syZMnKyMjQ2vWrJHF4vjPUJvNpsjISHl5eSk2NlZFRUVKS0tTfHy8PDw8FBcXJ0mV/nwGBwcrJiZGq1ev1vbt29WuXTuH7UeOHNGaNWvUqVMn/eEPf7C3P/nkk2rbtq3uuusu3XLLLcrLy9PKlSv16KOPas+ePU6FVFcpPadXrVql8PBwDRs2TD4+Plq7dq0mTJigLVu2aOHChfb+b775pp544gk1aNBAAwYMUGBgoHJzc5WZmamUlBQ99dRTVyVOAKgyAwBQZZKM4OBgwzAMY9SoUYbZbDYOHjxo3963b1/Dz8/POHXqlPHcc88ZkoyUlBT79pKSEqNVq1aGJGPRokUOc3/wwQeGJCM8PNw4f/68vb1ly5aGl5eXkZeX59C/sLDQCAgIMIKCgoxz587Z2+Pi4gxJxoEDB+xte/bsMTw9PY3mzZsbOTk5DvOkp6cbHh4exqBBg+xtM2fONCQZM2bMcMrBb7/9Zpw+ffqyuXr77bcNScasWbPsbXPmzDEkGb179za8vLyMU6dO2bfdeuutRo0aNYyioqJLHsuBAwcMSUZcXFyZ+127dq0hySn3F+//ySefvGz8l9pXnz59DEnGiy++6NC+ceNGw2w2G3Xq1DFOnjxpb7/zzjsNs9ls2Gw2e1vXrl2Njh07GnXr1jUeeeQRe/sPP/xgSDLi4+Ptbfn5+YbVajXq1q1r7Ny502Gf27dvN2rVqmV07NjR3rZs2TJDkjFx4kSnYyoqKjJOnDhhf12ar8TExCvKyeVcar6EhARDkjFgwACjsLDQYVthYaGRm5tb7ryln4U1a9a4JM5ff/3V8PPzMzw9PY1169Y5bb/4vM7Ozja8vLyM2rVrGz/++KNDvyeffNKQZIwePdqhPTo62pBkdO7c2SgoKLC3//TTT4anp6dhtVqNJk2aOJyPBQUFRt26dY3AwECHc7r0cyjJeOaZZxz288033xgWi8WwWq3G8ePH7e0vv/yyIcm45557HOY6evSoERoaakgyNm7cWOY+kpKSHPbx+eef2+e6WEpKiiHJGDx4sNPfhMTExDL/hpTuY9SoUUZxcbG9fefOnYbZbDZat27t0L+yn8/33nvPkGT85S9/cdr2yiuvGJKMmTNnOrTv27fPqW9RUZHRs2dPw2KxOP3tLH2PL1aak9//7SklyYiOjnZoK83V+PHjHXJSXFxsxMfHG5KMTz75xN5+2223GV5eXsbRo0ed5v/111/L3C8AXAu4fQMAXGz06NE6f/683nnnHUkXfmFcs2aNhg8frpo1a5Y5ZtOmTdq9e7fuuOMODR8+3GHbkCFDFBUVpT179mjDhg329ri4OJ09e1bvv/++Q//ly5eroKBAw4cPd/ol8vfeeOMNnTt3Tq+99prTLRW9evXSwIEDtXz5cp08edJhW40aNZzmqlWrVpntv1d6xcPvrwioX7++nn76aZ09e9Z+nHl5edq2bZuioqLk5eV12bmvRGRkpNMVJPHx8bJYLPr6668rPW9OTo5Wr16txo0b2xdiLNWtWzc9/PDDys/P10cffWRv79Wrl86fP6/169dLkk6ePKlvv/1WvXv3Vo8ePfTvf//b3resK0YWLFggm82m559/3um2oD/84Q8aPXq0tm7dql27djlsK+t98vLyUu3atSt59JV3/vx5zZ49WzVq1NCcOXPk7e3tsN3b21v16tWrtnhSU1N14sQJPfnkk/ZbTi4WEhJi/+9Fixbp7NmzGj9+vP2qn1IvvfSSateurYULF5Z5u0JycrLDFRTNmjVTVFSUbDabJk+e7HA+Wq1WDRgwQMeOHdOhQ4ec5vL393e4vUmSOnfurOHDh8tms+njjz+2t7/zzjsymUx69dVXHf4+BAUF2a9QKl3/5mKhoaH6+9//7tDWt29fNW7c2Om8ee2112SxWPTOO+84fdYmT56sunXr6t1333XaR82aNfXqq6/KbDbb29q0aaPIyEj9+OOP+u2335zGVNSgQYPk7++vd999V+fPn3fYlpqaKk9PTz388MMO7c2bN3eax8vLS+PGjVNxcbHD3zJXKSkp0euvv64GDRron//8p0NOzGaz/ud//kcmk8kpjxaLpcy1dgIDA10eIwC4CrdvAICLRUREqF27dnrnnXf097//XW+99ZZKSko0evTocsd8//33ki7c2lCWnj17asOGDdq6davuuusuSdJjjz2myZMnKzU1VePGjbP3rcitG1999ZUkaf369frmm2+ctufm5ur8+fPau3evOnXqpIEDB+q//uu/NG7cOK1atUp9+/ZVZGSk2rRpI5PJdNn9SRe+3DRr1kzr1q1TSUmJ/T7wmJgYRUdHy2KxKCMjQ3369NHatWtlGEa5eamMzp07O7V5enqqfv36KigoqPS8W7dulXThtoyyvhT07NlTixYt0tatW/XYY4/Z25KSkpSRkaGBAwdq/fr1Ki4uVq9evdSkSRN9+OGH+vHHH9W6dWt7geLiXJS+f9u2bSvz0Yh79+6VJP34449q06aNoqOjFRwcrOTkZH3//ffq16+fIiMjdeuttzp86alOu3fv1vHjxxUREVHlRTBdYfPmzZIurC1wOZc6bwMCAtSxY0d98cUX2r17tzp06OCwvazPYenxd+rUyWlbaZEiJydHoaGhDttuu+22MgtK3bt3V2pqqrZu3aq4uDidPHlS+/btU3BwsFMR5eLjKP0sX6y8z0ijRo3sn0Ppwi1C27ZtU2BgoGbMmOHUX7pQaPrxxx+d2sPCwuTn51fmPiSpoKBAvr6+Zc55pWrUqKGHHnpI8+bN06pVq9SvXz9J0nfffaedO3dq8ODBTl/gf/75Z02bNk0ZGRn6+eefdebMGYftZRWKqmrv3r3Kz89XWFiYXnzxxXKP5eI8Dh8+XH/5y1/Upk0bDR06VNHR0YqMjKzWoh4AVAZFCQC4CkaPHq2nn35an332mVJSUtSpU6dLLq5WuiDeLbfcUub20nabzWZvCwkJUa9evbRmzRr7F9fc3Fx9/vnnuvXWW9W+ffvLxlm6MOX06dMv2a/0F8rQ0FB9/fXXSkpK0ueff27/1b9Ro0Z65pln9PTTT192n9KFX/vnzZun77//Xp6envr111/Vq1cv1a5dW126dLH/8ngl60lUVFn390sXfmH8/S+nFVGZ97Br166qVauWw/F6eXkpKirKfm98RkaGwsLC9MUXX6hNmzZq0KCBfXzp+3fxAollKX3//Pz8tHnzZiUmJmrZsmVatWqVpAu/oj711FP6+9//Xu1PNCnNx7XyeNeKxFOZ97yUv7+/U1vplQuX2nbu3DmnbfXr1y9z/6WfldI4qxLvpc6bixdcLCgokGEY+vXXX/X888+XOaY8l9qHpCqdnxcbMWKE5s2bp9TUVHtRorSYW7puRan9+/fr9ttvV0FBge6880716dNH/v7+MpvN9nVSyroSpqpKz+2srKxL5vHiq0f+/Oc/KzAwULNnz9bMmTM1Y8YM+yKz06dPL7MQBgDXAm7fAICr4NFHH1WNGjU0duxYHTp0SGPGjLlk/9IvIb/88kuZ248cOeLQr1TpP6BL/0H97rvvqri42Okf1pfb7/Hjx2UYRrn/u/gy9tatW2vx4sXKy8vTt99+q+TkZJWUlOiPf/yj3n777Svab+kvsunp6U6Fh549e2rr1q3Kz89XRkaG/P397Qs1Xssq8x56enoqKipKO3fu1C+//KKMjAzdcccdqlmzplq2bKmQkBClp6fr66+/1smTJ51+kS+da9u2bZd8/y7+PISEhOjtt99Wbm6uduzYoZkzZ6pu3bqaMmWKpkyZ4tKcXInSL6JX49fmyqhIPJU9b13t6NGjZbaXxlW6/+qIt3Rsx44dL/mZNAyj0vuoqm7duiksLEzLli2TzWbTuXPn9P777yswMNBepCj16quvKi8vT2+//bbWrVunmTNn6oUXXlBSUpL69u17xfv08LjwT+6yHhd9qaLV4MGDL5nDAwcOOIx77LHHtHnzZuXl5WnFihUaNWqUvvjiC/Xt21e//vrrFccLANWJogQAXAVWq1WxsbHKyclRrVq1nO5R/r3SqyjKe5zd2rVrJcnpy/n9998vPz8/LVq0SCUlJUpNTZXFYtGwYcOuKM6uXbtKkr788ssr6n8xi8WiTp066W9/+5t9XYtPPvnkisb27NlTJpNJGRkZ+ve//61mzZrZrwzo1auXSkpKtGDBAmVlZal79+5XdGtBaR9X/ZpaUaXv4YYNG8r84lHee1hajHn//fe1Y8cOh6tCevbsqXXr1mnNmjUOfUtV5f0zmUxq27atJkyYYJ//4vevuvLZqlUrWa1WZWZmlvkI2upWmtPPPvvssn0vdd7abDb98MMP9kdhXk3ff/+907ovF8dVGmft2rXVvHlzHTp0yOkxw1L5n9GK8PX1Vdu2bbVz507l5+dXep7LqernMy4uToWFhVq8eLFWrFihY8eOadiwYU5XCpU+LemBBx5wmqN0LZgrERAQIElOj/mVLjwR6PdKz4vNmzeXeXXM5VitVvXr10/z5s3TiBEjlJ+fry+++KLC8wBAdaAoAQBXyYsvvqiPP/5Yq1atuuwCgpGRkQoPD9eGDRv04YcfOmz78MMP9eWXX6ply5aKiopy2FZ6f/ShQ4f0z3/+U9u2bVO/fv0UFBR0RTGOHz9enp6e+tOf/mRff+BiZ8+edfjC+91339kvAb9Y6S+15S3k+XtBQUFq27atNm7cqC+++MLhy3a3bt3k4+OjqVOnSip/nY3fCwgIkMlksj9atLqFhISod+/eys7OdrqXfsuWLXrvvfcUEBCgwYMHO2wrPb7k5GQZhuFUlDh+/Lhmz54tDw8Ph0cfStLIkSNltVr1/PPPl7lIZ0lJicMX5p07d5b5q3pZ71/po12vdj7NZrOeeuopnTlzRmPHjnW6FP7s2bPV+gtvXFyc/Pz89MYbb5T5JS4nJ8f+34888og8PT31+uuvOz3qd/LkyTpx4oQeeeQRp8U7Xe348eNOV7l8++23evfdd+Xv7+/wmYuPj5dhGHr22WcdvtAfO3bM/mjL+Pj4KsXz5z//WWfPnlV8fHyZVwEUFBTY1+OorKp+Ph977DF5eHhowYIFWrBggaSy1+EpLZb+vvC0atWqMhcELU/nzp3l4eGh9957T6dPn7a35+fnOy2MK10o+k6YMEFHjhzR008/7bSOhXThypaLF7EtXYPn93JzcyVd+d9nAKhurCkBAFdJ48aN1bhx4yvqazKZlJqaqt69e2vIkCG677771KpVK+3Zs0effPKJateurQULFtgvAb5YXFyc3nrrLSUkJNhfX6lWrVrpnXfeUXx8vNq2bau7775bLVu21Llz5/Tzzz/ryy+/VL169bR7925J0sKFCzV37lxFRUWpefPmCggI0E8//aTly5fL29tbEydOvOJ99+rVSzt27LD/dylvb29FRkZWeD0JX19fRURE6Msvv9Tw4cPVsmVLmc1mDRw48IrW13CFOXPmKDIyUs8++6xWr16tzp076+DBg0pLS5OHh4dSUlKcClQdO3ZUQECAcnNzVbt2bd1+++32baXHnpubq86dOzvdc1+3bl19+OGHGjx4sLp27apevXqpbdu2MplMOnjwoL766ivl5eWpsLBQkrRmzRo9++yzuuOOO9SyZUsFBQUpJydHS5culYeHh5599ln73OHh4QoODtYHH3wgT09PhYaGymQy6dFHH3VaaLGqEhMTtWXLFi1fvlwtW7ZU//79Vbt2bR08eFCrV6/W9OnTHb4wPvPMMzp27Jgk2Z/UMn36dC1atEjShScsDBo0qFKxBAYG6r333lNsbKx69Oihe+65R+3bt9eJEyeUmZmpgwcP2i+Zb9KkiWbMmKFx48bptttu00MPPaR69epp/fr1+uqrr9SqVStNmzatCpm5MnfddZfeeustbdmyRZGRkTpy5IgWL16skpISzZ0712HxyGeeeUafffaZli5dqg4dOqhfv346ffq00tLSlJubq7/+9a9Oxc+Kio+P13fffafZs2erefPm9qd05Ofn68CBA/riiy80cuRIzZkzp9L7qOrns1GjRurRo4cyMjJksVjUrl27Mtf9eeqpp5SSkqIHH3xQsbGxatiwoXbs2KHPP/9cDz30kBYvXnxF8d5yyy0aPny4Fi5cqFtvvVX33nuvTpw4oZUrV+quu+4qc3HRyZMna9u2bZozZ46WL1+unj17Kjg4WLm5ucrKytLGjRv10ksv2Z+8M3jwYPn6+qpr165q0qSJDMPQl19+qW+++UadOnVSTEzMFcUKANXuKj9yFABuCpKM4ODgK+r73HPPlfu8+t27dxuPPPKI0aBBA8NisRgNGjQwhg8fbuzevfuSc7Zo0cKQZNSpU8coKioqs09cXJwhyThw4IDTtszMTCMuLs5o3Lix4eXlZQQEBBht27Y1xowZY2RkZNj7bd682Rg7dqzRvn17IyAgwPDx8TGaN29ujBgxwti+ffsVHX+pZcuWGZIMk8lkHD161GHbyy+/bEgy6tevX6FjycrKMvr372/UqVPHMJlMDnleu3atIclITEwsc87Q0FAjNDT0imI/cOCAIcmIi4tz2paTk2OMHTvWaNy4seHp6WnUrVvXuO+++4yvv/663Pnuv/9+Q5LRr18/p20tW7Y0JBl//etfLxnPuHHjjBYtWhje3t5G7dq1jfDwcOORRx4xPv74Y3u/Xbt2GX/605+MTp06GYGBgYaXl5cRGhpqPPDAA8bGjRud5v3666+Nnj17Gn5+fvZ8rl279tLJKcfl8n/u3Dnj9ddfN7p06WLUqlXLqFmzptGiRQtj9OjRRlZWlkPf0NBQQ1K5/ytvHxWxY8cO49FHHzUaNmxoeHp6GkFBQcZdd91lzJ0716nvqlWrjN69extWq9Xw8vIymjdvbjz77LNGQUGBU9/o6GijvH9+XeocTUxMdMr/xZ/DXbt2GQMHDjSsVqtRo0YNo1u3bsbnn39e5n7OnDljvPTSS0bbtm0NHx8fw9fX14iMjDTee+89p76X+qxf7niWL19u3HvvvUa9evUMT09Po379+kaXLl2M5557zvjxxx8d+koyoqOjK5SXqn4+Fy5caP/M/Pd//3e5/TZu3Gj06NHDsFqt9lx9/PHH5X6my8tJYWGh8cwzzxjBwcGGp6en0bx5c+Pll182zp07V+7xl5SUGAsWLDB69uxpBAQEGJ6enkbDhg2NyMhI46WXXjJ+/vlne9833njDGDRokNG0aVOjRo0aRkBAgHHrrbca06ZNM06cOHHFeQGA6mYyDDeuNAQAAIBKyc7OVtOmTRUXF6f58+e7OxwAACqFNSUAAAAAAIBbUJQAAAAAAABuQVECAAAAAAC4BWtKAAAAAAAAt+BKCQAAAAAA4BYWdwfgSocPH3Z3CJcVGBhof7Y6qo58ug65dC3y6Vrk03XIpWuRT9cin65DLl2LfLoW+XSt6yGfDRs2LHcbV0oAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt7ihnr4BAAAAAMDVcv78eRUWFkqSTCaTm6O54OjRoyoqKnLb/g3DkNlslo+PT6XGV6kosWrVKi1btkw2m00hISEaMWKEWrduXW7/Xbt2KTU1VTk5OQoICNDAgQPVp08f+/aPP/5YX3/9tQ4fPiyLxaKwsDANGzZMjRs3rkqYAAAAAABUyfnz53XmzBnVqlXrmilISJLFYpHZbHZrDIWFhTp37pw8PT0rPLbSt29s2rRJ8+fP1+DBgzVt2jSFh4fr5ZdfLvf5qLm5uZo6darCw8M1bdo0DRo0SCkpKdq8ebO9z65du9SnTx+98MILSkxMlNls1gsvvKDffvutsmECAAAAAFBlhYWF11xB4lrh7e2ts2fPVmpspYsSn376qaKjoxUTE6OQkBDFx8crICBAq1evLrP/6tWrFRAQoPj4eIWEhCgmJkbR0dFavny5vc9zzz2nHj16qHHjxmrcuLEmTJigEydOaPfu3ZUNEwAAAAAAl6AgUbaq5KVSRYni4mLt379fHTp0cGhv37699uzZU+aYrKwstW/f3qGtQ4cO2r9/v4qLi8scczhSEC8AACAASURBVObMGRmGIV9f38qECQAAAACAS1CQuLTK5qdSa0qcOHFCJSUl8vf3d2i3Wq3avn17mWNsNpvatWvn0Obv76/z58/r5MmTCggIcBqTkpKiJk2aqGXLlmXOmZ6ervT0dElScnKyAgMDK3M41cpisVwXcV4vyKfrkEvXIp+uRT5dh1y6Fvl0LfLpOuTStcina12v+Tx69KgslmvzWRHXQlze3t6Vel/dH3k5UlNTtWfPHk2ZMkUeHmVf0BETE6OYmBj76/LWs7iWBAYGXhdxXi/Ip+uQS9cin65FPl2HXLoW+XQt8uk65NK1yKdrXa/5LCoqcvuCkmWxWCzl3n1QnYqKisp9Xxs2bFjuuEoVJfz8/OTh4aHjx487tNtsNlmt1jLHWK1W2Ww2h7bjx4/LbDardu3aDu3z58/Xpk2blJiYqPr161cmRAAAAAAArrrzowdW6/7M85ZVeExJSYkmTZqkFStWyGazKS0tTd26dbsK0VVcpdaUsFgsatasmTIzMx3at2/frvDw8DLHhIWFOd3akZmZqWbNmjlcapKSkqKNGzfqH//4h4KDgysTHgAAAAAA+P8yMjK0ZMkSzZ8/X1u3blXnzp0v2d9ms2nChAlq1aqVWrVqpQkTJjhdlOAqlX76Rv/+/bVu3TplZGQoJydHKSkpys/PV+/evSVJs2bN0qxZs+z9+/Tpo/z8fM2fP185OTnKyMjQunXrNGDAAHuft956S+vWrdMf//hH+fr6ymazyWazqbCwsAqHCAAAAADAzSs7O1tBQUHq0qWLgoKC5OXldcn+48eP144dO7Ro0SItWrRIO3bs0NNPP31VYqv0mhLdunXTyZMn9dFHH6mgoECNGjVSQkKC6tWrJ8l5fYegoCAlJCQoNTXV/njQkSNHqmvXrvY+pY8TnTJlisPY2NhYPfTQQ5UNFQAAAACAm9LEiROVlpYmSQoODlZISIg2b96suXPnauHChTp8+LDq1Kmj2NhYJSQkKCsrS2vXrtUnn3xiv6Ji2rRpGjx4sPbt26cWLVq4NL4qLXTZt29f9e3bt8xtSUlJTm1t2rTRtGnTyp1vyZIlVQkHAAAAAABcZMqUKQoJCdEHH3yglStXymw2Kzk5WQsWLFBiYqIiIiKUl5enHTt2SJK+++471apVy+EWjy5duqhmzZr67rvvrq2iBAAAAAAAuHb5+fnJ19dXZrNZQUFBOnXqlObNm6ekpCQNHTpUktS0aVN7ESI3N1d169aVyWSyz2EymRQYGKjc3FyXx1fpNSUAAAAAAMD1Ze/evSoqKlJUVJS7Q5FEUQIAAAAAAPx/QUFBysvLk2EY9jbDMHTs2DEFBQW5fH8UJQAAAAAAuEmEhYXJ29tbGzZsKHN7p06ddOrUKX377bf2tm+//VanT59Wp06dXB4Pa0oAAAAAAHCT8PX11ahRo5ScnCxvb29FRESooKBAmZmZiouLU1hYmHr06KFJkybZH1QxadIkxcTEuHyRS4miBAAAAAAAlWaet8zdIVRYQkKC/P39NWPGDB05ckSBgYGKjY21b581a5YmT56s4cOHS5L69OmjF1988arEYjIuvlHkOnf48GF3h3BZgYGBOnbsmLvDuGGQT9chl65FPl2LfLoOuXQt8ula5NN1yKVrkU/Xul7zefr0adWsWdPdYTixWCwqLi52dxiXzE/Dhg3LHceaEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALnr5xjQval3DZPrktplZDJDeGy+WTXFYM+XQdznXXIp+uxbnuWuTTdTjXXYvPpmuRT+DKcKUEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKFLgEAAAAAqKT73t1drftbOrxVhceUlJRo0qRJWrFihWw2m9LS0tStW7erEF3FUZQAAAAAAOAGlpGRoSVLligtLU2hoaGyWq2X7P/aa6/p3//+t3bu3KkzZ87o0KFDVy02bt8AAAAAAOAGlp2draCgIHXp0kVBQUHy8vK6ZP+zZ8/qnnvu0eOPP37VY+NKCQAAAAAAblATJ05UWlqaJCk4OFghISHavHmz5s6dq4ULF+rw4cOqU6eOYmNjlZCQIEl69tlnJUmffvrpVY+PogQAAAAAADeoKVOmKCQkRB988IFWrlwps9ms5ORkLViwQImJiYqIiFBeXp527NjhlvgoSgAAAAAAcIPy8/OTr6+vzGazgoKCdOrUKc2bN09JSUkaOnSoJKlp06bq3LmzW+JjTQkAAAAAAG4Se/fuVVFRkaKiotwdiiSKEgAAAAAAwE0oSgAAAAAAcJMICwuTt7e3NmzY4O5QJLGmBAAAAAAANw1fX1+NGjVKycnJ8vb2VkREhAoKCpSZmam4uDhJ0qFDh1RQUKCcnBxJsi+C2bRpU9WqVcul8VCUAAAAAABcNcsX237X4vh6wBBr9QVzFSwd3qra9mXLL5Ytv/h3rY6vrXUu/zU/ISFB/v7+mjFjho4cOaLAwEDFxsbat0+fPt3+GFFJ6tu3ryQpLS1N3bp1q/wBlIGiBAAAAAAAN7CxY8dq7Nix9tceHh4aP368xo8fX2b/GTNmaMaMGdUSG2tKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3sLg7AAAAAAAArlfLF9uqdX939vat8JiSkhJNmjRJK1askM1mU1pamrp163YVoqs4ihIAAAAAANzAMjIytGTJEqWlpSk0NFRWq7XcvgcPHtSMGTO0adMm5ebmKigoSAMHDtTEiRNVo0YNl8dGUQIAAAAAgBtYdna2goKC1KVLl8v23bdvn86fP6+pU6eqadOmysrK0t/+9jcVFBTolVdecXlsrCkBAAAAAMANauLEiUpKStKhQ4cUHBysiIgIGYahOXPmKDIyUk2bNlWnTp00depUSVKPHj00Y8YMde/eXaGhoYqJidGECRO0YsWKqxIfV0oAAAAAAHCDmjJlikJCQvTBBx9o5cqVMpvNSk5O1oIFC5SYmKiIiAjl5eVpx44d5c7x22+/XfKWj6qgKAEAAAAAwA3Kz89Pvr6+MpvNCgoK0qlTpzRv3jwlJSVp6NChkqSmTZuqc+fOZY7PycnRnDlzNGHChKsSH7dvAAAAAABwk9i7d6+KiooUFRV12b6//vqrhg8frrvuuktjxoy5KvFQlAAAAAAAAA5yc3P14IMPKjw8XDNnzpTJZLoq+6EoAQAAAADATSIsLEze3t7asGFDuX2OHj2q2NhYhYWFafbs2bJYrt7KD6wpAQAAAADATcLX11ejRo1ScnKyvL29FRERoYKCAmVmZiouLk6//PKLYmNj1aBBAyUlJSk/P98+tm7dujKbzS6Nh6IEAAAAAACVNGDI1XkqRVls+cUumSchIUH+/v6aMWOGjhw5osDAQMXGxkqS1q9frwMHDujAgQO6/fbbHcZt3rxZjRo1ckkMpShKAAAAAABwAxs7dqzGjh1rf+3h4aHx48dr/PjxTn2HDBmiIUOGVFtsrCkBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANzC4u4AAAAAAAC4Xs2cObNa9/fYI09VeExJSYkmTZqkFStWyGazKS0tTd26dbsK0VUcRQkAAAAAAG5gGRkZWrJkidLS0hQaGiqr1Vpu35KSEsXHx2vnzp3Ky8uTv7+/oqKi9F//9V+65ZZbXB4bt28AAAAAAHADy87OVlBQkLp06aKgoCB5eXldsn9kZKTmzJmjL774Qm+++ab+85//6PHHH78qsXGlBAAAAAAAN6iJEycqLS1NkhQcHKyQkBBt3rxZc+fO1cKFC3X48GHVqVNHsbGxSkhIkIeHh0aPHm0fHxISovHjx2vkyJEqLCyUj4+PS+OjKAEAAAAAwA1qypQpCgkJ0QcffKCVK1fKbDYrOTlZCxYsUGJioiIiIpSXl6cdO3aUOb6goEAfffSROnbs6PKChFTFosSqVau0bNky2Ww2hYSEaMSIEWrdunW5/Xft2qXU1FTl5OQoICBAAwcOVJ8+fRy2L1++XPv371dBQYGeeuopde/evSohAgAAAABw0/Lz85Ovr6/MZrOCgoJ06tQpzZs3T0lJSRo6dKgkqWnTpurcubPDuJdeekkpKSk6c+aMbrvtNi1YsOCqxFfpNSU2bdqk+fPna/DgwZo2bZrCw8P18ssv69ixY2X2z83N1dSpUxUeHq5p06Zp0KBBSklJ0ebNm+19CgsL1ahRI40cOfKy97gAAAAAAICK2bt3r4qKihQVFXXJfk8++aRWrVql999/X2azWRMmTJBhGC6Pp9JXSnz66aeKjo5WTEyMJCk+Pl4//PCDVq9erWHDhjn1X716tQICAhQfHy/pwn0p+/bt0/Lly9W1a1dJ0m233abbbrtNkvS///u/lQ0NAAAAAABUQZ06dVSnTh01b95cLVq0UJcuXfT1118rIiLCpfupVFGiuLhY+/fv14ABAxza27dvrz179pQ5JisrS+3bt3do69Chg9avX6/i4mJZLBUPJT09Xenp6ZKk5ORkBQYGVniO6maxWCoW577Ld7kejvtqcXU+yWUFj598lotz3bXIp+twrrsW+XQtznXX4bPpWuSzqmyX3Hq95OLo0aOV+t5a3cqK0cPDQyaTSRaLRa1bt5a3t7e++uortWzZ8orm9PC4cJPFpb67e3t7V+q9rFRGT5w4oZKSEvn7+zu0W61Wbd++vcwxNptN7dq1c2jz9/fX+fPndfLkSQUEBFQ4jpiYGPuVGpLKvXXkWhIYGFihOIOuoM/1cNxXi6vzSS4rdvzks3yc665FPl2Hc921yKdrca67Dp9N1yKfV9f1kouioiKZzWZ3h3FZxcXFTm0lJSUyDEPFxcXy8fHRqFGj9NJLL8lisSgiIkIFBQXKzMxUXFycvv32W+3YsUNdunSRv7+/srOzNX36dDVq1EidOnUqc37pQn7Key8bNmxYbrzXfpkHAAAAAIBr1NNPP11t+7Lll10QqKiEhAT5+/trxowZOnLkiAIDAxUbGytJ8vHx0aeffqrp06frzJkzCgoKUvfu3fXGG29cO0/f8PPzk4eHh44fP+7QbrPZZLVayxxjtVplszletnP8+HGZzWbVrl27MmEAAAAAAIDLGDt2rMaOHWt/7eHhofHjx2v8+PFOff/whz/oww8/rLbYKvX0DYvFombNmikzM9Ohffv27QoPDy9zTFhYmNOtHZmZmWrWrNl1cV8OAAAAAABwrUo/ErR///5at26dMjIylJOTo5SUFOXn56t3796SpFmzZmnWrFn2/n369FF+fr7mz5+vnJwcZWRkaN26dQ6LZRYWFio7O1vZ2dkyDEPHjh1Tdnb2dXOPEQAAAAAAuHKVvkShW7duOnnypD766CMVFBSoUaNGSkhIUL169SQ5L1YSFBSkhIQEpaam2h8POnLkSPvjQCXpp59+0vPPP29/vWTJEi1ZskTR0dEaN25cZUMFAAAAAADXoCrdN9G3b1/17du3zG1JSUlObW3atNG0adPKna9t27ZasmRJVUICAAAAAADXiUrfvgEAAAAAAFAVFCUAAAAAAIBbUJQAAAAAAABuQVECAAAAAAC4BUUJAAAAAADgFlV6+gYAAAAAADezoH0J1bcvSXvrvFDhcSUlJZo0aZJWrFghm82mtLQ0devWzfUBVgJXSgAAAAAAcAPLyMjQkiVLNH/+fG3dulWdO3e+onGFhYWKiYlRcHCwtm3bdlVioygBAAAAAMANLDs7W0FBQerSpYuCgoLk5eV1ReNeeOEF3XLLLVc1NooSAAAAAADcoCZOnKikpCQdOnRIwcHBioiIkGEYmjNnjiIjI9W0aVN16tRJU6dOdRi3atUqbdq0Sf/4xz+uanysKQEAAAAAwA1qypQpCgkJ0QcffKCVK1fKbDYrOTlZCxYsUGJioiIiIpSXl6cdO3bYxxw+fFgJCQlauHChfHx8rmp8FCUAAAAAALhB+fn5ydfXV2azWUFBQTp16pTmzZunpKQkDR06VJLUtGlT+zoT58+f14QJEzRmzBi1bdtWBw8evKrxcfsGAAAAAAA3ib1796qoqEhRUVFlbp85c6Y8PT31xBNPVEs8XCkBAAAAAAAkSRs3btSWLVsUGhrq0D5gwAANHDhQs2bNcun+KEoAAAAAAHCTCAsLk7e3tzZs2KBmzZo5bX/11Vd1+vRp++ujR49q2LBhev3119WlSxeXx0NRAgAAAACAm4Svr69GjRql5ORkeXt7KyIiQgUFBcrMzFRcXJwaN27s0L9WrVqSpCZNmqhhw4Yuj4eiBAAAAAAAlZTbYurlO7mILb/YJfMkJCTI399fM2bM0JEjRxQYGKjY2FiXzF1RFCUAAAAAALiBjR07VmPHjrW/9vDw0Pjx4zV+/PjLjm3UqJEOHTp01WLj6RsAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAABchmEY7g7hmlbZ/FCUAAAAAADgMsxmswoLCylOlKG4uFgeHpUrL/BIUAAAAAAALsPHx0fnzp3T6dOnJUkmk6naYzj6y9nL9vHy8aqGSP6PYRjy8PCQj49PpcZTlAAAAAAA4Ap4enrK09PTbfvfu/3yRYnwNjWrIRLX4fYNAAAAAADgFhQlAAAAAACAW1CUAAAAAAAAbkFRAgAAAAAAuAVFCQAAAAAA4BY8fQMAAAAAcM0K2pdw2T65LaZWQyQ3hsvls7pzyZUSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANzCUpXBq1at0rJly2Sz2RQSEqIRI0aodevW5fbftWuXUlNTlZOTo4CAAA0cOFB9+vSp0pwAAAAAAOD6VOkrJTZt2qT58+dr8ODBmjZtmsLDw/Xyyy/r2LFjZfbPzc3V1KlTFR4ermnTpmnQoEFKSUnR5s2bKz0nAAAAAAC4flW6KPHpp58qOjpaMTExCgkJUXx8vAICArR69eoy+69evVoBAQGKj49XSEiIYmJiFB0dreXLl1d6TgAAAAAAcP2q1O0bxcXF2r9/vwYMGODQ3r59e+3Zs6fMMVlZWWrfvr1DW4cOHbR+/XoVFxdLUoXnTE9PV3p6uiQpOTlZgYGBlTmcch0d3O2S2+/v/spl53jc0uB3LTaHVweOLrjk+ClT5l12H6496qunqvl0zqXk6nzeKLmUKpPPiuVSunnyybleMdfDuS5dH/nkXHeta+Fcl8jnxfjbeUF1nOsS/066GH87r5wrzvWNf4xyeG2xWOzfASXpH//4xyXH3yjnulT1fP4+l5Lr81nduaxUUeLEiRMqKSmRv7+/Q7vVatX27dvLHGOz2dSuXTuHNn9/f50/f14nT56UYRgVnjMmJkYxMTH21zfibR434jG5E/l0LfLpOuTStcina5FP1yKfrkMuXYt8uhb5/D+/z0VgYGCF8kMu/09Zubge8tmwYcNyt/H0DQAAAAAA4BaVulLCz89PHh4eOn78uEO7zWaT1Wotc4zVapXN5njZ0/Hjx2U2m1W7dm1JqvCcAAAAAADg+lWpKyUsFouaNWumzMxMh/bt27crPDy8zDFhYWFOt2FkZmaqWbNmslgslZoTAAAAAABcvyp9+0b//v21bt06ZWRkKCcnRykpKcrPz1fv3r0lSbNmzdKsWbPs/fv06aP8/HzNnz9fOTk5ysjI0Lp16xwWtrzcnAAAAAAA4MZRqds3JKlbt246efKkPvroIxUUFKhRo0ZKSEhQvXr1JDkvnhEUFKSEhASlpqbaHw86cuRIde3a9YrnBAAAAAAAN45KFyUkqW/fvurbt2+Z25KSkpza2rRpo2nTplV6TgAAAAAAcOPg6RsAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANzC4u4AbnZPP/20u0O4oZBP1yGXrkU+XYt8ug65dC3y6Vrk07XIp+uQS8B1uFICAAAAAAC4BUUJAAAAAADgFhQlAAAAAACAW1CUAAAAAAAAbkFRAgAAAAAAuAVFCQAAAAAA4BY8EvQSzPOWXbrDu7urJ5AbBPl0ncvmUiKfFcBn07XIp+twrrsWn03XIp+uw7nuWuQTuL5wpQQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt6AoAQAAAAAA3IKiBAAAAAAAcAuKEgAAAAAAwC0oSgAAAAAAALegKAEAAAAAANyCogQAAAAAAHALihIAAAAAAMAtKEoAAAAAAAC3oCgBAAAAAADcgqIEAAAAAABwC4oSAAAAAADALShKAAAAAAAAt7BUZpBhGEpLS1NGRoZ+++03hYWFadSoUWrUqNElx23evFmLFy/W0aNHVb9+fT388MO6/fbb7du3bNmi9PR07d+/XydPnlRiYqLatm1bmRABAAAAAMA1rlJXSixdulSffvqpRo4cqalTp8rPz08vvviizpw5U+6YvXv3asaMGbrzzjv1yiuv6M4779Srr76qrKwse5+ioiK1bNlScXFxlQkLAAAAAABcRypclDAMQytXrtSgQYPUtWtXNW7cWOPHj9eZM2e0YcOGcsetWLFCbdu21f3336+QkBDdf//9atu2rVasWGHvc9ddd+nBBx/UrbfeWrmjAQAAAAAA140K376Rm5srm82m9u3b29u8vLzUunVr7dmzR7179y5z3N69e3XPPfc4tHXo0EGff/55RUOwS09PV3p6uiQpOTlZgYGBlZ7LXa7HmK9l5NN1yKVrkU/XIp+uQy5di3y6Fvl0LfLpOjdTLo+6YI7f58tisVQohzdSvquaz7Jycb3ns8JFCZvNJkmyWq0O7f7+/iooKLjkOH9/f6cxpfNVRkxMjGJiYuyvjx07Vum53OV6jPlaRj5dh1y6Fvl0LfLpOuTStcina5FP1yKfrkMuK+b3+QoMDKxQDsn3/ykrF9dDPhs2bFjutssWJb788ku9+eab9tcJCQmuiQoAAAAAANzULluU6Ny5s8LCwuyvz507J+nClQ8XX/Zx/PhxpyshLma1WnX8+HGHtuPHjztdcQEAAAAAAG4Ol13oskaNGmrQoIH9fyEhIbJarcrMzLT3OXv2rHbv3q3w8PBy52nZsqXDGEnKzMxUy5YtqxA+AAAAAAC4XlX46Rsmk0n9+vXT0qVLtWXLFv3888+aPXu2fHx8FBUVZe83ZcoUvffee/bX/fr1044dO/TJJ5/o0KFD+vjjj7Vz507de++99j6//fabsrOzdfDgQUnSL7/8ouzs7CqtOwEAAAAAAK5NFV7oUpLuu+8+nT17Vm+//bZOnTqlFi1a6LnnntP/Y+/uw60q67yBf4+8CKRwkCMgHMGUFxUFcxDPGJgapkEqD9VxUEkNtVQ0pqaufBzNCPOlUtIBk9RpnkpBbRQTHscANc2w0VQeUMSXDEWFUM9BEEVenj+82OPhnePGBfb5XBfXxVp73Wvf+7fvvfba332vfVq2bFnaZuHChWnXrl1puWfPnhk1alQmTpyYSZMmpWPHjhk1alSDS0MeffTRjB8/vrR8/fXXJ0m+9KUvpba2tjFdBQAAALZTjQolKioqUltbu8mgYNy4ceutq6mpSU1NzUbbHHHEETniiCMa0yUAAABgB7PVl28AAAAAlINQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAoRNOiOwAAAMDfr/PPP7/oLnys7Gj1NFMCAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAFS62QAAAIABJREFUKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKETTxjRas2ZNbrvttkyfPj1Lly5N9+7dM2LEiOy5556bbDdz5sxMmjQpCxcuTIcOHTJs2LD069cvSbJy5cpMnDgxTzzxRBYuXJiWLVumV69eOfnkk1NVVdWYbgIAAADbsUbNlJg8eXLuvvvunH766bnsssvSunXrjBkzJsuXL99om3nz5mXs2LEZMGBArrzyygwYMCBXXXVVnn322STJihUr8pe//CVDhw7NFVdcke985zt5/fXXc+mll2bVqlWNe3QAAADAdmurQ4k1a9Zk6tSpGTJkSGpqatKlS5eMHDkyy5cvz0MPPbTRdlOmTEmvXr0ydOjQVFdXZ+jQoenVq1emTJmSJGnVqlUuuuiiHHbYYenUqVO6deuWs846KwsWLMiCBQsa/wgBAACA7dJWX76xaNGi1NXVpXfv3qV1zZs3z3777ZdnnnkmRx999AbbzZs3L5///OcbrOvTp0/uueeejd7X22+/nST5xCc+scHbp02blmnTpiVJLr/88u3yMo/Tz+3WYLlp06ZZuXJlQb3Zsa1by0Q9Pwxjs7zUs3y81svL2Cwv9Swv9SwftSwv9fwfC8uwj3U/pzVt2nS7/Oz2Ufiw9dxQ3Xb0em51KFFXV5ckqaysbLC+TZs2efPNNzfZrk2bNuu1Wbu/da1cuTK//OUv8w//8A9p167dBrcZOHBgBg4cWFpevHjxFj2Gj9K6faqqqtou+7kj2FDd1LPxjM3yUs/y8VovL2OzvNSzvNSzfNSyvNSzvNSzfHbU86ROnTpt9LbNhhIPPvhgJkyYUFq+4IILytOrTVi1alWuueaaLFu2LN/5zne2+f0BAAAAH73NhhJ9+/ZN9+7dS8vvvfdekvdnPnxwikh9ff16MyE+qLKyMvX19Q3W1dfXrzfjYtWqVfnpT3+a+fPn55JLLsmuu+66ZY8EAAAA2KFs9ocuW7ZsmY4dO5b+VVdXp7KyMrNmzSpts2LFisydOzc9e/bc6H569OjRoE2SzJo1Kz169Cgtr1y5MldffXX++te/5nvf+956gQUAAADw8bHVf32joqIigwYNyuTJk/PII49k/vz5GT9+fFq0aJH+/fuXths9enRuvvnm0vKgQYMye/bs3HnnnVmwYEHuuOOOzJkzJ4MHD07y/gyJtX8i9Bvf+EYqKipSV1eXurq6rFixogwPFQAAANiebPUPXSbJCSeckBUrVuTGG2/MsmXL0q1bt1x44YVp2bJlaZuFCxc2+IHKnj17ZtSoUZk4cWImTZqUjh07ZtSoUaVLQ15//fU8+uijSZLvfve7De7vnHPOyRFHHNGYrgIAAADbqUaFEhUVFamtrU1tbe1Gtxk3btx662pqalJTU7PB7du3b59bb721Md0BAAAAdkBbffkGAAAAQDkIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCCCUAAACAQgglAAAAgEIIJQAAAIBCNG1MozVr1uS2227L9OnTs3Tp0nTv3j0jRozInnvuucl2M2fOzKRJk7Jw4cJ06NAhw4YNS79+/Uq3T5w4MTNnzszrr7+epk2b5pOf/GROPPHE9OzZszHdBAAAALZjjZopMXny5Nx99905/fTTc9lll6V169YZM2ZMli9fvtE28+bNy9ixYzNgwIBceeWVGTBgQK666qo8++yzpW06deqUESNG5Mc//nFGjx6d9u3b54c//GHq6uoa000AAABgO7bVocSaNWsyderUDBkyJDU1NenSpUtGjhyZ5cuX56GHHtpouylTpqRXr14ZOnRoqqurM3To0PTq1StTpkwpbXP44YfnwAMPTIcOHbLnnnvmK1/5SpYvX54XX3yxUQ8OAAAA2H5tdSixaNGi1NXVpXfv3qV1zZs3z3777Zdnnnlmo+3mzZuXPn36NFjXp0+fzJs3b4Pbr1y5MtOmTUvLli2z1157bW03AQAAgO3cVv+mxNpLKSorKxusb9OmTd58881NtmvTps16bda9NOOxxx7L2LFjs2LFilRWVuaiiy5a777WmjZtWqZNm5Ykufzyy1NVVbW1D2ebW7dPTZs23S77uSPYUN3Us/GMzfJSz/LxWi8vY7O81LO81LN81LK81PN/LCzDPtTzf3zYen4cz5M2G0o8+OCDmTBhQmn5ggsu2KYd6tWrV370ox9lyZIlmT59eq6++uqMGTMmbdu2XW/bgQMHZuDAgaXlxYsXb9O+Nca6faqqqtou+7kj2FDd1LPxjM3yUs/y8VovL2OzvNSzvNSzfNSyvNSzvNSzfHbU86ROnTpt9LbNhhJ9+/ZN9+7dS8vvvfdekvdnPnwwjamvr19vJsQHVVZWpr6+vsG6+vr69WZBtGjRIh07dkzHjh3To0ePnH/++Zk+fXq+9KUvba6rAAAAwA5ks78p0bJly1JI0LFjx1RXV6eysjKzZs0qbbNixYrMnTt3k3+6s0ePHg3aJMmsWbPSo0ePTd7/mjVrsnLlys11EwAAANjBbPUPXVZUVGTQoEGZPHlyHnnkkcyfPz/jx49PixYt0r9//9J2o0ePzs0331xaHjRoUGbPnp0777wzCxYsyB133JE5c+Zk8ODBSZK33347EydOzLPPPpvFixfnhRdeyPjx4/P666/nH//xH8vwUAEAAIDtyVb/0GWSnHDCCVmxYkVuvPHGLFu2LN26dcuFF16Yli1blrZZuHBh2rVrV1ru2bNnRo0alYkTJ2bSpEnp2LFjRo0aVbo0pEmTJnnppZdy33335a233squu+6affbZJ9///vfTtWvXD/kwAQAAgO1No0KJioqK1NbWpra2dqPbjBs3br11NTU1qamp2eD2O++8c7797W83pjsAAADADmirL98AAAAAKAehBAAAAFCIRl2+AQAAwMdfk5/ftekNfj33o+nIx4R6rs9MCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEEIJAAAAoBBCCQAAAKAQQgkAAACgEE0b02jNmjW57bbbMn369CxdujTdu3fPiBEjsueee26y3cyZMzNp0qQsXLgwHTp0yLBhw9KvX78NbjthwoRMmzYtp5xySo4//vjGdBMAAADYjjVqpsTkyZNz99135/TTT89ll12W1q1bZ8yYMVm+fPlG28ybNy9jx47NgAEDcuWVV2bAgAG56qqr8uyzz6637cyZM/Pcc8+lbdu2jekeAAAAsAPY6lBizZo1mTp1aoYMGZKampp06dIlI0eOzPLly/PQQw9ttN2UKVPSq1evDB06NNXV1Rk6dGh69eqVKVOmNNjub3/7W/793/89559/fpo2bdREDgAAAGAHsNWhxKJFi1JXV5fevXuX1jVv3jz77bdfnnnmmY22mzdvXvr06dNgXZ8+fTJv3rzS8qpVq/LTn/40X/ziF1NdXb21XQMAAAB2IFs9FaGuri5JUllZ2WB9mzZt8uabb26yXZs2bdZrs3Z/SXLrrbdm1113zec+97kt6su0adMybdq0JMnll1+eqqqqLWr3UVq3T02bNt0u+7kj2FDd1LPxjM3yUs/y8VovL2OzvNSzvNSzfNSyvNSzvNSzfD6O50mbDSUefPDBTJgwobR8wQUXbJOOzJkzJ/fff39+9KMfbXGbgQMHZuDAgaXlxYsXb4uufSjr9qmqqmq77OeOYEN1U8/GMzbLSz3Lx2u9vIzN8lLP8lLP8lHL8lLP8lLP8tlRz5M6deq00ds2G0r07ds33bt3Ly2/9957Sd6f+fDBNKa+vn69mRAfVFlZmfr6+gbr6uvrSzMu5syZk7q6upx11lml21evXp1f//rXmTp1an72s59trqsAAADADmSzoUTLli3TsmXL0vKaNWtSWVmZWbNmpVu3bkmSFStWZO7cuTnllFM2up8ePXpk1qxZDf6856xZs9KjR48kyTHHHJOampoGbS699NJ8+tOfbjAbAgAAAPh42OofuqyoqMigQYMyefLkPPLII5k/f37Gjx+fFi1apH///qXtRo8enZtvvrm0PGjQoMyePTt33nlnFixYkDvuuCNz5szJ4MGDk7z/+xJdunRp8K9p06aprKzc5FQPAAAAYMfUqL+5ecIJJ2TFihW58cYbs2zZsnTr1i0XXnhhgxkVCxcuTLt27UrLPXv2zKhRozJx4sRMmjQpHTt2zKhRoxpcGgIAAAD8/WhUKFFRUZHa2trU1tZudJtx48att66mpma9SzQ2ZUP7AAAAAD4etvryDQAAAIByEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhWjamEZr1qzJbbfdlunTp2fp0qXp3r17RowYkT333HOT7WbOnJlJkyZl4cKF6dChQ4YNG5Z+/fqVbh83blweeOCBBm26d++eSy+9tDHdBAAAALZjjQolJk+enLvvvjvnnHNOOnXqlNtvvz1jxozJ2LFj07Jlyw22mTdvXsaOHZva2tr069cvf/rTn3LVVVflBz/4Qbp3717a7sADD8x55533Px1s2qguAgAAANu5rb58Y82aNZk6dWqGDBmSmpqadOnSJSNHjszy5cvz0EMPbbTdlClT0qtXrwwdOjTV1dUZOnRoevXqlSlTpjTYrlmzZqmsrCz922WXXbb+UQEAAADbva0OJRYtWpS6urr07t27tK558+bZb7/98swzz2y03bx589KnT58G6/r06ZN58+Y1WDd37tycccYZ+cY3vpGf/exnqa+v39ouAgAAADuArb42oq6uLklSWVnZYH2bNm3y5ptvbrJdmzZt1muzdn9JctBBB+XQQw9N+/bts2jRokyaNCmjR4/O5ZdfnmbNmq23z2nTpmXatGlJkssvvzxVVVVb+3C2uXX71LRp0+2ynzuCDdVNPRvP2Cwv9Swfr/XyMjbLSz3LSz3LRy3LSz3LSz3L5+N4nrTZUOLBBx/MhAkTSssXXHDBNuvMpz/96dL/u3Tpkr333jvnnntu/vznP+fQQw9db/uBAwdm4MCBpeXFixdvs7411rp9qqqq2i77uSPYUN3Us/GMzfJSz/LxWi8vY7O81LO81LN81LK81LO81LN8dtTzpE6dOm30ts2GEn379m3wQ5TvvfdekvdnPnwwjamvr19vJsQHVVZWrncpRn19/XozLj5ot912y2677ZZXX311c90EAAAAdjCb/U2Jli1bpmPHjqV/1dXVqayszKxZs0rbrFixInPnzk3Pnj03up8ePXo0aJMks2bNSo8ePTbaZsmSJXnjjTfStm3bLXksAAAAwA5kq3/osqKiIoMGDcrkyZPzyCOPZP78+Rk/fnxatGiR/v37l7YbPXp0br755tLyoEGDMnv27Nx5551ZsGBB7rjjjsyZMyeDBw9Okrzzzjv5P//n/2TevHlZtGhR5syZkyuuuCJt2rRJv379yvBQAQAAgO3JVv/QZZKccMIJWbFiRW688cYsW7Ys3bp1y4UXXpiWLVuWtlm4cGHatWtXWu7Zs2dGjRqViRMnZtKkSenYsWNGjRpVujRkp512yksvvZTf//73WbZsWdq2bZtevXrln//5nxvsFwAAAPh4aFQoUVFRkdra2tTW1m50m3Hjxq23rqamJjU1NRvcvnnz5rnwwgsb0x0AAABgB7TVl28AAAAAlINQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAACiEUAIAAAAohFACAAAAKIRQAgAAAChE08Y0WrNmTW677bZMnz49S5cuTffu3TNixIjsueeem2w3c+bMTJo0KQsXLkyHDh0ybNiw9OvXr8E2r7zySm6++ebMnj07K1euTOfOnXPeeeelurq6MV0FAAAAtlONmikxefLk3H333Tn99NNz2WWXpXXr1hkzZkyWL1++0Tbz5s3L2LFjM2DAgFx55ZUZMGBArrrqqjz77LOlbRYtWpSLLroo7du3z8UXX5yf/OQnOfHEE9OiRYvGdBMAAADYjm11KLFmzZpMnTo1Q4YMSU1NTbp06ZKRI0dm+fLleeihhzbabsqUKenVq1eGDh2a6urqDB06NL169cqUKVNK29xyyy3p06dPvvKVr2TvvfdOhw4dcvDBB6eqqqpxjw4AAADYbm11KLFo0aLU1dWld+/epXXNmzfPfvvtl2eeeWaj7ebNm5c+ffo0WNenT5/MmzcvSbJ69eo89thjqa6uzqWXXpoRI0bkggsuyMMPP7y1XQQAAAB2AFv9mxJ1dXVJksrKygbr27RpkzfffHOT7dq0abNem7X7W7JkSd55553ccccdOfHEE3PyySdn9uzZueaaa9KiRYscfPDB6+1z2rRpmTZtWpLk8ssv3y5nVKzbp6ZNm26X/dwRbKhu6tl4xmZ5qWf5eK2Xl7FZXupZXupZPmpZXupZXupZPh/H86TNhhIPPvhgJkyYUFq+4IILtklHVq9enSTp27dvvvCFLyRJ9tprrzz//PO55557NhhKDBw4MAMHDiwtL168eJv07cNYt09VVVXbZT93BBuqm3o2nrFZXupZPl7r5WVslpd6lpd6lo9alpd6lpd6ls+Oep7UqVOnjd622VCib9++6d69e2n5vffeS/L+zIcPpjH19fXrzYT4oMrKytTX1zdYV19fX5px0bp16zRp0mS9v7LRuXNnl3AAAADAx9Bmf1OiZcuW6dixY+lfdXV1KisrM2vWrNI2K1asyNy5c9OzZ8+N7qdHjx4N2iTJrFmz0qNHjyTvTznZZ5998sorrzTY5tVXX83uu+++VQ8KAAAA2P5t9Q9dVlRUZNCgQZk8eXIeeeSRzJ8/P+PHj0+LFi3Sv3//0najR4/OzTffXFoeNGhQZs+enTvvvDMLFizIHXfckTlz5mTw4MGlbY4//vg8/PDDmTZtWl577bVMmzYtDz/8cI455pgP+TABAACA7c1W/9BlkpxwwglZsWJFbrzxxixbtizdunXLhRdemJYtW5a2WbhwYdq1a1da7tmzZ0aNGpWJEydm0qRJ6dixY0aNGtXg0pB+/frla1/7Wu644478+7//e/bYY4+ce+65G/w9CQAAAGDH1qhQoqKiIrW1tamtrd3oNuPGjVtvXU1NTWpqaja57yOOOCJHHHFEY7oFAAAA7EC2+vINAAAAgHIQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIUQSgAAAACFEEoAAAAAhRBKAAAAAIVoWnQHAAAA2DFNPnnforvwsfL3WE8zJQAAAIBCCCUAAACAQlSsWbNmTdGdKJdXXnml6C5sVlVVVRYvXlx0Nz421LN81LK81LO81LN81LK81LO81LN81LK81LO81LO8doR6durUaaO3mSkBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABRCKAEAAAAUQigBAAAAFEIoAQAAABSiYs2aNWuK7gQAAADw98dMiY/Yd7/73aK78LGinuWjluWlnuWlnuWjluWlnuWlnuWjluWlnuWlnuW1o9dTKAEAAAAUQigBAAAAFKLJJZdccknRnfh7s/feexfdhY8V9SwftSwv9Swv9SwftSwv9Swv9SwftSwv9Swv9SyvHbmefugSAAAAKITLNwAAAIBCCCXgY27+/Pm58847s3LlyqK7AgAA0IBQokC1tbWZOXNm2fd7//33Z/jw4WXfb2MsWrQotbW1ef755z/UNn/vbr311nzrW9/a6nZvv/12fvKTn6RDhw5p2rTph+7HnDlzUltbmyVLlnyobT4uttVrmG3H8Qb+/owbNy6XX3550d0APkJ33XVXzj333KK7URZbcu5y7rnn5q677voIe1VeH/5TynbqiiuuyLvvvpuLL754vdtefvnlfPOb38yFF16YPn36FNC7902YMCGf+MQnyr7fww47LJ/61KfKvt911dbWbvL2z3zmM/nyl7+82f1UVVVlwoQJ2XXXXcvVtUYZN25c3nrrrY/s7/yOGzcuDzzwQI488sicffbZDW771a9+lbvuuisHH3xwvvvd7+b444/P5z//+a2+j/Hjx+eYY47JP/7jP5ar29uVtTVMkiZNmqRdu3bp169famtr06JFi4J7Vz6XXHJJ9txzz4wYMaLormyRrRnbjXX//ffnxhtvzC9/+csP293t1kdRx+3dx6UGt956ax555JH85Cc/abB+yZIlOeOMM/K9730vvXr1Ktv9zZkzJ9///vdzww03pHXr1lvV9oUXXsgFF1yQHj165Ac/+EHZ+rQlPky/N+X000/P9vITah/VmF6wYEFuu+22zJkzJ8uWLUvbtm1z6KGHZujQodlll10+1L63R9vD+cCG3qu31Zj+MOrq6nLHHXfkz3/+c15//fXsuuuu6dq1a4499tgcfPDBRXfv79qHOf5edtll2XnnnbdRz7a9j20ocdRRR+XHP/5xFi1alPbt2ze4bcaMGdl9991z4IEHFtS791VWVm7y9pUrVzbq2+3mzZunefPmje3WFpswYULp/4899liuv/76BuuaN2+epUuXbnY/O+2002Zr8XHVrl27/PGPf8zpp59eetO2lcVAAAAgAElEQVRctWpVfv/736eqqqq0XYsWLRr1pvov//IvW7RdY8fa9uDAAw/Meeedl5UrV2bu3Ln52c9+lnfffTdnnnlm0V37u7alY7sx/p4uRdqWddxRqMFHa8aMGTnmmGPywAMP5OWXX051dXXRXfrQWrVqVXQXGtjWY/q5557L6NGjs//+++fb3/52dtttt/z1r3/Nr371qzz++OMZM2bMNvlSrGjOBzZv0aJFueiii9KyZcsMGzYse+21V1avXp3Zs2fn5z//ea677rpG7XdHPo/cnnyY4+/mQq/t/Tnafnv2IR188MFp06ZN7r///gbf6K9cuTIPPvhgjjnmmCTJddddl9mzZ6euri7t2rXLZz/72Rx33HHZaaf3r2xZ++35vvvumylTpmTFihX53Oc+l2HDhuX222/Pvffem4qKigwePDhDhgwp3U9tbW2++tWv5vHHH8+cOXPSunXr/NM//VMOP/zwBtt885vfTE1NTRYtWpSRI0fm/PPPz/Tp0zNv3rwMHz48xx57bO67777cddddWbRoUaqqqnL00Udn0KBBpT6ua91vENd+QzN06NBMnDgx9fX1OeCAA/L1r3/9Q6W2HwwS1r65rRsurA0l/va3v+Xmm2/OM888k9133z2nn356evfunSSlx37ZZZdln332KaXKF110UW655ZbMnz8/1dXVOeussxr8qZsZM2bktttuy1tvvZUDDjggn/rUp3LjjTfm1ltvTZIsXrw4N910U55++um89957qaqqype//OV8+tOfXu+x3HrrraWEfe14WfvN1fz58/Mf//EfmTt3bpo3b56+ffvm9NNPL8tJTteuXfPmm2/mj3/8Y4488sgkyZ///Oc0a9Ys++23X6l+637LNn/+/PziF7/I888/n9WrV6djx4459dRTc8ABByR5fzbQL3/5yzz99NNp3rx5DjjggJx22mml5+eD4/qee+7JypUrc8MNN+T3v/99/u///b9ZsGBBmjdvnv333z+nnXZadttttwb9fvbZZzNx4sS88sorqa6uzte+9rVN/hmiZ555JjfffHOef/75fOITn0jfvn1z8sknl2r41FNP5de//nXmz5+fnXbaKZ06dcrZZ5+dLl26bLaGzZo1Kz2u/v37Z/bs2fnv//7vnHbaafn1r3+dP/zhD3n77bez1157Zfjw4dl3331LbRcsWJBf/epXeeqpp7J69ep06dIlX/va19KlS5c899xzmThxYv7yl79k5cqV6dKlS4YPH54ePXo0uP+lS5fmqquuyuOPP542bdqktra2wet8c+Nn7XPRu3fvTJ48OStWrMghhxySESNGZOedd864cePy1FNP5amnnsp//dd/JUn+7d/+LVVVVbn++us3efza3DjZlrZ0bK9evTr/+Z//menTp6e+vj577LFH/umf/imHHHJIkmz02HjTTTcl+Z/X65e+9KXU1tZu8RjeUZS7jt/85jfzu9/9boPH4mTTx46nnnoqP/jBD3Ldddc1ONbfcssteeyxx/LjH/+40BpsTR3Wvt+s9cH34yS5/fbbM2PGjNTV1eUTn/hE+vTpk5EjRyZJ1qxZk7vuuivTpk3LG2+8kY4dO+aEE05o8LpvrC3p3+aey0WLFuX73/9+kuSMM85I8v7MxS2ZxrxixYo89NBDGT16dN59993MmDEjX/nKVxr0bVNjaPXq1Y0+LrVv336j/X7iiSfyn//5n3nppZeSJN26dcupp57a4IR9U8/ZujMht2R/29KWjunGvObWrFmT6667LnvssUe+853vlOpeVVWVT37ykzn//PNzyy235Iwzzsi9996bqVOnZuzYsUmSWbNmZcyYMTnppJNK57TXXHNNmjdvnq9//eul88vvfOc7+cUvfpFFixalW7duOfvss9f7ArAIjT0f2NLzzk2dy2zsvXpjY3pbHkc25cYbb0ySXH755Q2+7Kqurs6AAQOSvD/LdsmSJQ1m66xevTrnnntuBg8enC984Qu55JJL0rlz5+y888554IEH0r59+1x22WW5++67c//992fhwoVp1apVPvWpT2X48OGlzwlbOoZ+97vf5a677srixYtTVVWVE044IQMHDtzkY5s8eXLuvvvuvPPOOzn00EM3OCa39jPVR2lTx991rV69OjfddFMef/zx/Ou//mv22GOPnHvuuTnmmGNy/PHHJ/mfz6KzZ8/Ok08+maOPPjqnnHLKZo/RRfnYhhJNmjTJZz7zmdx///350pe+VCr0Y489liVLluSII47I6tWrs9tuu+Wf//mf07p16zz33HOlywiOOuqo0r6efvrp7Lbbbrnkkkvyl7/8Jddee21efPHFfPKTn8zo0aMze/bs3HDDDendu3eDg9ett96aYcOG5dRTT83MmTMzbty4dO7cucGJxrpuueWWDB8+PGeffXaaNGmSadOm5dZbb81Xv/rV7L333pk/f36uv/76NG3aNMcee+wW12PRokV5+OGH8y//8i959913M3bs2EycODFnnXVWI6q79SZOnJhTTjklZ5xxRn7zm99k7NixGT9+/Ca//b/55ptz8sknp23btvnFL36Ra6+9NldddVUqKioyb968XH/99Rk2bFj69euXp556KrfcckuD9jfccEPee++9fO9730urVq3yyiuvbPS+jj/++CxYsCBLly7NeeedlyTZZZdd8s477+TSSy/NPvvsk8suuyxLly7N9ddfn/Hjx2/xLITNOfLII3PfffeVTkzW/n/hwoUbbfPTn/40Xbt2zQ9/+MM0adIk8+fPL82OefPNN/O9730vRx55ZIYPH55Vq1bllltuyZVXXpkxY8aUXgtPPfVUWrVqlf/9v/93ab8rV67Ml7/85XTu3DlvvfVWfv3rX+enP/1p6U11rV/+8pelD3q33357Lr/88lx77bUbnDY2f/78jBkzJrW1tfn617+epUuX5he/+EWuu+66fOtb38qqVavyox/9KEceeWTOO++8rFq1Kn/5y18afXBs3rx5Vq1alV/96lf54x//WHqju/vuu3PppZfmmmuuSdu2bfPGG2/k4osvTs+ePXPRRRelVatWee6557J69eokyTvvvJPDDz88p512WioqKnLPPffksssuyzXXXNPgUqPbb789J510Uk466aTMmDEj1113Xfbff/9UVVVt8fh5+umnU1lZmYsuuiivv/56rr766uyxxx75X//rf+X000/Pq6++mk6dOuWkk05K8n4aviXHr02Nk4/CloztqVOn5re//W3OPPPM7L333nnwwQfz4x//OFdccUX22muv0nYfPDbutNNOWb16dW655ZZce+21SVI6lmzpGN6RlLOOmzoWb+7Ysf/++6dDhw554IEHcsIJJyR5/8To97//fY477rjCa7A1ddiUmTNn5re//W2+8Y1vpEuXLqmvr8+zzz5bun3ixImZOXNmRowYkU6dOpXej3bZZZePdOrzxp7LqqqqfOtb38pPfvKTXHXVVdlll122+HU/c+bM7L777unSpUsOP/zwXH311TnppJMafLu2qTH0YY5Lm+r3O++8k0GDBqVr165ZsWJFfvOb3+SKK67I1VdfnaZNm272OVvX5vb3UdiSMd2Y19yLL76Yl156Keeff/5676O77bZb+vfvnz/84Q8ZMWJEevXqlRtuuCF1dXWl4HHXXXfNnDlzSqHE008/nWHDhpX2sXLlytx55505++yz06xZs4wbNy4///nPc+GFF5a1PuWwpecDa23qvHNz5zIbe6/e2Jgu4jiydOnSPPHEEznxxBM3eP69NjgYOHBgLr744rz55pul+syaNSt1dXUNQpMHH3wwAwcOzOjRo0uXR1VUVOS0005L+/btS18O3nTTTaVz62TzY+hPf/pTbrrpppx66qnp3bt3nnzyydx4442prKxM3759N/jYHn744UycODFf/epX06tXr8ycOTOTJ09ucKlSuT5TbStbcvxN3q/fv/3bv+Wll17KD37wg01+6XL77bdn2LBhGT58eCoqKrb4s28Rio+FtqGjjjoqixcvzv/7f/+vtG7GjBnp06dPqqqq0rRp05x44onp1q1b2rdvn8MOOyxHH310/vCHPzTYT6tWrXLGGWekc+fO6d+/fz75yU+mrq4uJ510Ujp16pTPfe5z2X333TN79uwG7fr165ejjz46nTp1ytChQ3PAAQdkypQpm+zzsccem5qamrRv3z7t2rXLb37zm5xyyimldX379s2QIUNKKeyWWptwdu3aNT169MjAgQMb1GVbGzx4cPr27Zs99tgjJ510UpYuXZoXX3xxk21OPPHEHHDAAencuXO++MUvZsGCBXnjjTeSvH/i2bt37wwZMiSdOnXKwIED069fvwbtFy9enH333Td77bVX2rdvn4MOOigHHXTQBu+rRYsWad68eSllr6ysTNOmTfPQQw/lnXfeyXnnnZcuXbpk//33z1lnnZU//elPee2118pSm/79++f555/Pq6++mrq6ujzxxBM54ogjNtlm8eLF6d27dzp37pyOHTumX79+pW/w77333nTt2jWnnHJKqqur07Vr14wcOTLPPfdcXnjhhdI+mjVrVpqNsHZGwlFHHZWDDz44HTp0SLdu3XLGGWfk6aefzuuvv97g/r/4xS/moIMOSpcuXXLOOeeU0t0Nueuuu3LYYYfluOOOyx577JHu3bvnzDPPzCOPPJL6+vosX748y5YtS9++fdOxY8fS66wx31g999xz+cMf/pBevXrl3nvvzcknn5yDDz649I1HZWVl6bXzX//1X9l5553zzW9+M926dUunTp1y+OGHlz64HHDAATn88MNTXV2dzp0756tf/WqaNWuWxx9/vMF9Hn744Tn88MPTsWPHnHjiiWnSpEmeeuqpJNni8dOqVaucddZZqa6uTp8+fVJTU1M6nrRq1SpNmzbNzjvvXBqbO+200xYdvzY1Tj4KWzK2f/vb3+a4445L//7906lTp5x44onZb7/91vuxpg8eG6uqqkozTdbWZO0J1paO4R1JOeu4qWPxlhw7jjrqqNx///2l/T355JOpr68vfcO2rWzpcXJL67ApixcvTmVlZXr37p2qqqrss88+pRPWd955J3fffXe+/vWv56CDDkr79u3Tv3//fPazn93s+/LLL7+c4cOHN/j3YX6EbWPP5U477VQ6EW/dunUqKyu3eGbfjBkzSs/l/vvvn5133jmPPvroFt1vkg91XNpUv2tqalJTU5M99tgjXbt2zTnnnJNFixblueeeK+1zY8/Zhmxufx+FLR3TW/uae/XVV5MknTt33uDt1dXVWbZsWZYsWZLOnTunsrKy9H4zZ86cHHfccZk7d25WrVqV1157La+//nqD3ztZtWpVRowYkW7duqVr16457rjjMmfOnO3mNzvW2przgbU2dd65uXOZjb1Xb2hMf5jjyIfx2muvZc2aNZs9v+rRo0c6d+5cmkGcvB+a9e3bt8EM6/bt2+crX/lKOnfuXNrn4MGDc8ABB6R9+/bZf//9c8opp+SPf/xj6QufZPNj6Le//W0GDBiQY489Np06dcrnP//59O/fP5MnT95on6dOnZrPfOYzDT53devWrcE25fpMta1syfH33XffzRVXXJG//e1v+f73v7/ZWaCHHXZYPvvZz6ZDhw5p3779Fn/2LcLHdqZEkuyxxx7Zf//9c99996VPnz5544038uSTT2bUqFGlbe69997MmDEjf/vb37JixYqsWrUqu+++e4P9VFdXN0ib27Rps961eG3atFnvrw2se/LfvXv39T7QrOuDsyiWLFmS119/PRMmTMjPf/7z0vrVq1dv9cH/gyfxSdK2bduP9K8jdO3atcF9J0l9ff0Wt1n7oquvr0+7du3yyiuv5B/+4R8abN+9e/dMnz69tDxo0KD8/Oc/zxNPPJEDDzww/fr12+QlBhuyYMGCdO3aNS1btiyt69mzZyoqKvLyyy+nY8eOW7W/Ddlll13Sr1+/3HfffWnVqlV69eq12WtKBw8enOuvvz4PPPBADjzwwBx66KGlE5AXXnghTz/99Ab/Astrr71WOkh36dIlzZo1a3D7Cy+8kNtvvz0vvvhili5dWhpnixcvTrt27UrbfXBst2jRIl26dMnLL7+8wb6+8MILee211/Lwww+vd9vChQvTo0ePHHHEEbn00ktzwAEH5MADD0xNTc0WX1f7xBNPZPjw4Vm9enVWrlyZQw45JMcee2xmzpyZnj17lrbbaaed0r1791I/X3zxxey7774b/Vasvr4+kyZNypw5c1JXV5fVq1dnxYoVWbx4cYPtPniJSZMmTdK6devSa2tLx8+6x5jddttti06ON3f82tQ4+Shsbmy//fbbefPNNxs8T0my7777rnes3NQMsw/a0jG8IylnHTd1LN6SY8cRRxyRiRMn5plnnknPnj1z33335ZBDDtnmP1S8JcfJranDptTU1GTq1KkZOXJk+vTpk4MOOih9+/ZNs2bN8vLLL+e9997LD3/4wwZtNnTusK6OHTvmggsuaLBu6dKlDWarbY3GvK9uymuvvZa5c+fm/PPPT/L+N579+/fPjBkzSpe1bMn9bovj0muvvZZJkyblueeey5IlS0rnQWuPx5t6zhqzv4/Clr73b+vX3P7775+nnnoqhxxySJ5//vl861vfyu9+97s8//zzeemll9KhQ4cGx85mzZqlU6dOpeW2bdtm5cqVWbZsWeE/oNnY84G1NnXeublzmTZt2mxxPz/MceTD2JrPDmsDkiFDhmTp0qV59NFH15shvKFz6tmzZ+eOO+7IggUL8vbbb5eei7q6ulJNNzeGXn755dIMorX23Xff9T6gf9CCBQvW+6a/e/fupS+AyvmZalvY0uPvtddem8rKynzve9/bot+a29BztCWffYvwsQ4lkvcT5uuvvz5Lly7N/fffn1122aU09efhhx/Of/zHf5SuE2/VqlXuueee/Pd//3eDfTRp0qTBckVFxQbXfTAFbKwPTn9fu78zzzxzvZOsrbXuB69y9XdLfbBeFRUVSTZ/cFy3xlvS5oOOOuqo9OnTJ48//nhmzZqVf/3Xf82QIUM2+1dDinDkkUdm3LhxadGiRU488cTNbl9bW5sBAwbk8ccfz5NPPpnbbrstZ555Zo466qisWbMmn/rUpzZ4HdoH3zTXvdRi7aUGBx54YEaOHJk2bdrkrbfeysUXX/yhflhwzZo1Oeqoo/KFL3xhvdvWvkGdc845GTRoUJ544ok8+uijueWWW/Ltb397ozNbPmi//fbL1772tTRp0iRt27ZN06ZN89e//rXR/V1r3Lhxqa+vz6mnnprdd989zZo1y/9n787jqqj+x4+/WAQF2ZFdQAI03FBJzVRKTM2iTcNyt1JzqyzN+vrxI6RpLtmiWFimuaJW5lKpHxEVsDTNnUWvimxeUBFFERC4vz94MD+vgIAC96rv5+Ph46Fzz5w5czwzZ+bMWT799NNyeVEb19a9lPXq3L/uVk7qS03LdmWqM6N0XZVhfVBb+Xi3e3F17h2WlpYEBAQQHR2Ni4sLBw8eZMqUKfecnpqojTwoa/y7/fq6s2zY29vz1VdfceLECY4dO8aKFSv4+eef+eyzz5T9pkyZUu4FsqLr+HbGxsblGrLv/DhQnfRVdLzq1qt3ExUVRUlJCWPHjlW23d6oV53j1tV9ac6cOdja2jJy5EhsbW0xMjLigw8+UPLmbv9nFT24VxVffalOma7pNefs7AyUvvg2a9as3O9paWmYm5srX7z9/Pz4/fffSUpKwsnJCWtra/z8/Dhx4gRpaWnlVoW5c0hIWRmoz2fKytzv88Dd6uLqPMtU1/3cR+6Hs7Oz8lHkzt7Fd+revTurV68mMTGRc+fOYWlpWW7Fwjvr5YsXLzJ79myCgoIYMGAAjRs35ty5c3z99dda19a9lqGycPeiNt+p6kJ177/t2rVj7969JCUlVWsFyTvvf9V999WFh75RonPnzvz444/s3buX6OhounfvrrxEJCYm4u3trdXF727j+Gvq9OnTWpXs6dOna/SV0traGhsbGzIzMwkMDKy1dD0MXFxcyq3VW9GXZTs7O3r27EnPnj357bff+PPPPyttlDA2Ni53Q3R1dSU6OpqbN28qX7uTkpKq1f2tJlq3bo2xsTG5ubnKhGxVcXZ2xtnZWekRsmvXLnr06EGzZs3466+/lCFK1ZWRkUFubi4DBw5UJgfav39/hWFPnz6No6MjUPoimJqaWunkTM2aNatWrxJPT088PT15+eWXmTVrFnv27KlWo4SpqWm5uB0dHTE2NlYesqC0Qjp9+rQy0amnpycxMTGVzkacmJjIiBEjlLGdOTk5XLlypcr03K62yk9FZbO696/Kykl9uVvZNjMzw8bGhqSkJK3VkBITE6vMn4rypCZl+EFTV/l4u+reO4KCgliwYAEODg5YW1vX20pWVd0nq5MPZS9iOTk5yu8VDSU0MTGhffv2tG/fnpdffplRo0aRlJSEr68vDRo04OLFi3UyYWx101eVsv+/6r4oFhcXs2fPHgYOHFhuPPuiRYvYvXt3tSbgu9/7UkXpzs3NJT09nbfeekvJ87Nnz1JcXKwVZ2X/Z3c+uFc3vvpQ3bq/Jtecp6cnrq6ubN26laeeekrrBTA7O5vY2Fiefvpp5QWvbF6J2NhY/Pz8lG0xMTFkZGRozSeh7+71eaA6qvMsU1G9VFGZdnNzq9P7SGUaN25M27Zt2b59O3379i33wnrjxg2lJ3hZT55du3aRnJxMYGBglXN9nTlzhqKiIoYPH66E/ffff2ucTjc3NxITE7WeVaqqz1xdXSt87yqjz+9UNbn/BgUF4eXlxbx58/joo4+0Jqqujrp+970fD/WcElBaSXXt2pUNGzaQmZmpVVidnZ05d+4chw8f5sKFC/z888/KWPDacODAAXbu3MmFCxfYuHEjJ06coG/fvjWKIyQkRJlNNiMjg5SUFPbs2cPGjRtrLZ0Por59+3L06FE2b97MhQsX2LVrFwcOHNAKs2zZMo4cOUJmZibJyckcPXr0rje0Jk2akJqaSkZGBteuXaOoqIhu3bphamrKokWLSElJIT4+niVLltCxY8daGbpRxsDAgPnz57No0aJKu5uWKSws5IcffuDkyZNkZWVx+vRprZt17969ycvL46uvvuL06dNkZmZy7NgxIiIiuHnzZqXx2tvb06BBA7Zt20ZmZib//vsv69atqzDsL7/8wrFjx0hNTeXbb7/F2NiYrl27Vhj2pZdeUibSOXfuHGq1mkOHDinLx2ZlZbF69WqSkpK4ePEiJ06c4Pz58/fV6NOwYUN69erF6tWr+ffff0lLS+P7778nJydHWXmnd+/e5Ofns2DBAlQqFWq1mtjYWOUFwNnZmZiYGNLS0lCpVHz99dc1ngCttspPkyZNUKlUZGVlKV2Nq7p/VVVO6ktVZfvFF19ky5YtxMbGkpGRwbp160hISKhy4sQmTZpw69Ytjh07xrVr1ygoKKhRGX7Q1FU+3q669442bdrQuHFjfv75Z55++ul6m7G7OvfJqvLBxMQEHx8fNm3aRGpqKklJScpKVWV2795NVFQUKSkpZGVlsXv3boyMjHB2dqZRo0YEBwezcuVKdu3ahVqtJjk5mR07drBz5877PsfqpK86mjRpgoGBAf/++y/Xrl0jPz//ruH//fdfcnNzCQoKUuYZKvvTpUsXoqOjq3Xc+70vVZRuc3NzLCwsiIqKQq1WEx8fz/fff6/1Rflu/2d3qk589aW6dX9NrjkDAwPGjBlDRkYGc+fO5dSpU1y6dIl///2XGTNm0KRJE15//XUlfNm8EjExMcoLctmQjjvnk3gQVed5oDqqepaBiuvqisp0Xd9H7uatt95Co9Hw8ccf89dff5GRkUF6ejo7duwoNzwjKCiI2NhYzp8/X244RUWcnZ3RaDT8/vvvZGVlERsbW+VcehUJDg4mJiaGbdu2ceHCBf78809iY2OVVSUq0rdvX/bs2aP13nXnx0p9faeq6f23Z8+eDBs2jHnz5nHs2LEaHauu333vx0PfUwJKu/Hv2LGD5s2baz2QP/vssyQnJ/PNN9+g0Wjo1KkTwcHB1a58q/Laa6+xf/9+li1bhqWlJWPGjCk36UpVgoKCMDU1ZcuWLaxduxYTExPc3Nz0YpZYXfL19WX06NFs2LCBdevW0bp1a1566SUiIyOVMBqNhh9//JHLly/TsGFDWrduXenSOlB6kcfHx/Pxxx+Tn5+vLAk6depUli9fzieffKK1pGNtu33egbsxNDTkxo0bLF68mCtXrmBhYUH79u2VceC2trbMmDGDNWvWMGvWLAoLC7G3t6dt27Z3feixtLRk3LhxrF27lu3bt+Pu7s7QoUPLjXkEGDRoECtWrCAjI4OmTZsyZcqUSse2eXh4EBYWRmRkJKGhoZSUlODg4KB0HTQxMeHChQssWLCA3NxcrKys6NatmzLT+L0aNGgQULrs740bN2jWrBlTp05VxkDb2toSFhbGqlWrCAsLw8DAAHd3d2VFmjFjxrBkyRKmTJmCra0tr732Wo3nYTE1Na2V8hMcHEx4eDgffPABhYWFLFq0qMr7V1XlpD7drWw/99xz3Lx5k9WrV5OTk4OLiwsffvhhlSslNG/enGeffZavv/6a3NxcZUnQ6pbhB1Fd5OPtqnvvMDAw4JlnnmHDhg1VTspb26q6T1YnH8aMGUNERASffPIJjo6OvP3220yfPl353czMjE2bNrFy5UqKi4txc3Nj0qRJSu+bAQMGYGVlxZYtW/jhhx9o1KgRnp6e933Pqm76qqPsnhUZGUlERATdu3e/66Sau3btomXLlhXOU/Dkk0+yZs2aaj383u99qbJ0T5w4kWXLlvHhhx/i5OTEkCFDlCWyoer/s9sZGhpWGV99qk7dX9NrztfXl9mzZ/Pzzz8zd+5cbty4gUfpTeMAACAASURBVK2tLR07dqRfv37l5n7w8/Pjr7/+UnpKODg4YGtri6Gh4QM7F8/tqnoeqI6qnmWg4rrawcGhwjJd1/eRyjg6OjJnzhw2btzI6tWryc7OxsLCAg8PD0aPHq0VtmXLltjZ2WFvb6/0jr0bDw8Phg8fzqZNm4iMjKR58+YMGTJEWXK2ujp27MiIESPYsmULP/30E/b29rz11luVrrwBpRM6ZmZmEhkZSUFBAQEBATz//PNak3Xq6zvVvdx/n332WTQaDfPmzWPy5MnV7jFR1+++98NAow+zezyE7lzzXNS95cuXc/z4cZ09WAghxKPi+++/R61WM23aNF0nRYhHglxzor4VFhYyevRo3nzzzTpfYUmIh374hnh4bd68meTkZNRqNTt27OB///tftbqXCSGEuDd5eXmcOnWKvXv38vzzz+s6OUI89OSaE/WtpKSEq1ev8ssvv2BiYsKTTz6p6ySJR8AjMXxDPJzOnDnDli1byMvLw8HBgYEDB9Z4zg4hhBDVN3fuXFQqFT169Cg3IZcQovbJNSfq26VLlxg/fjx2dnaMHTu2xvNpCXEvZPiGEEIIIYQQQgghdEKGbwghhBBCCCGEEEInpFFCB8LDw/n8888r/X337t06mSH/YRIaGsrSpUt1nQzxgLh+/TojR45ErVbrOinlLFiwgC1btug6GXrl5MmThISE1Hg1FFG5quqlh4U+X+srV67kxx9/1HUyakTyUzcelev1fuhz2ZR6XUj5LO+RHSQUHh6utUyMhYUFPj4+DBkyBFdXVx2m7NFwe/4bGRlhbm5O06ZN6dSpEz179rzv8WuTJk3SyZrjD4Lw8HByc3P5+OOPdZ0UvbFx40batWuHk5MTAMuWLSMpKYnU1FSsra0JDw8vt8++ffvYuHEjFy5cwNLSkj59+pRbQ3vbtm1s376drKws7O3tefXVVwkMDFR+3717N4sXLy4X96pVqzAxMQGgf//+TJ8+naCgIMzMzGrztGtFReXp0KFDfPnll7zwwgu8/vrrOkzd3YWGhtK0aVPeeustXSel2kJCQu76e2Bg4F2XfqzMiBEjeBRGc+rqWg8NDa1wLXg3NzcWLFgAwEsvvcSECRN4/vnnq7X8nj6Q/Lx/dz4P2dnZ0bFjR0JCQipdarsurtdx48bRu3fvcv8XDypdlU0onZw0MjKS/fv3k5ubi52dHW+88QZdunQB9K9el3ei+ldX5TM2NpZNmzZx4cIFGjVqROvWrRk6dCjW1tZKmD/++IMdO3Zw8eJFLCwsCAgIYPDgwcr9Rlfl85FtlABo3bo1EyZMACA7O5tVq1Yxf/58vvzyywrDFxUVyWQvtags/0tKSrh27RonTpxgw4YNxMTEMG3atEor4+q4cw1uISpTUFDArl27mDJlirJNo9EQGBhISkpKubWhAQ4fPsw333zDiBEj8Pf3Jz09nYiICExMTJT1rnfs2MHq1asZPXo0Pj4+qFQqIiIiMDc311pr29TUlIULF2rFX9YgAeDu7o6joyN79+7V+Vra1bF3716+++47Bg8eLBPP1oElS5Yofz906BARERFa224vO1D9eksfHozrmi6v9UmTJlFUVKTEe+vWLSZNmqQ1q72lpSVt2rRhx44dD0RvScnP2lP2PFRUVERiYiLfffcdBQUFjBw5UitccXExhoaGj8T1ej90WTaLioqYOXMmjRs3ZuLEidja2pKdna11H9bHer2m70RVKSurBgYGtZnMOo+7PtRV+UxMTGThwoUMGTKEjh07kpOTw9KlS/nmm2/473//C5Q2WqxatYp33nmHFi1akJWVxbfffsutW7cYM2YMoLvy+Ui/YTdo0EBpObK2tub5559nzpw5FBYWkpOTw/jx43n33XeJiori1KlTDBkyhF69evHrr78SFRXF1atXcXZ25vXXX+eJJ55Q4k1JSeGnn34iMTERExMTAgICGDFiRKWVSHJyMrNmzeKZZ57hjTfe0PotKyuLCRMmMGvWLB577DFl+86dO1m7di0REREkJSURFhbGtGnTWLt2LSkpKbi5uTFq1Ci8vLzqIOdqx+35b2tri6enJ23atGHKlCls3rxZ+SJ4/fp1li9fzqFDhygsLKRFixYMHz6cpk2bVhr3nV9Ax40bR48ePbh8+TJxcXE0atSIvn37PjRfBGpTWloaK1euJCEhARMTE1q1asXw4cOV/6uUlBSWL1/OmTNnKCkpwcnJiWHDhtGqVSsdp/zeHD58GIDmzZsr2958802gdNnZiiqHvXv30qFDB3r37g2Ao6MjL7/8Mps2baJ3794YGBiwd+9egoKC6Nq1qxLmzJkzbNq0SatRAtBqwa5IQEAAcXFxevPwUpnff/+d1atX884779C9e3cA9u/fz/r167lw4QJWVlY8++yzvPLKK8rDxLhx43jmmWfIzMxk//79mJubM2TIENq2bcv333/PoUOHsLGx4a233qJt27Zaxzt9+jSRkZFkZGTg5ubG6NGjlXtebm4uS5cuJTExkdzcXBwdHQkODlaWDQ4PDyc+Pp74+Hi2b98OwKJFi3BwcKiv7Lont5cVc3NzrW1ZWVmMGjWqXL311FNP3TUvoHyPl9DQUNzc3DAzMyMqKgoDAwO6d+/O4MGDMTR8MEd+6vJav7OhPCYmhoKCgnLLWAcEBLB27Vq9f4kGyc/adPvzUNeuXTlx4gT//PMPVlZW7N+/n+DgYH755ReysrL46aefWLp0qXK97ty5k3Xr1hEREaF1bX799dfk5+czZcoU1Go1K1as4PTp0+Tn5+Pi4kJISAgdOnQASq/3ixcvsmrVKlatWgXA+vXrAUhKSmLNmjWcOXNGefkeNGiQXjeM6LJs7t69m2vXrvHpp58qDREV1Sv6Vq/f7Z3IxMSE7OxsVqxYwdGjRwHw9fVl+PDhODs7A6XlpaKymp2dTUREBCqVCnt7e4YNG8aXX37JW2+9xdNPPw1wz3EnJiby66+/kpqaCoC3tzfDhg3Dzc1NOa+yBpYjR45QWFiIs7Ozzp9Z66p8njp1Cjs7O1544QWgtNz16dNHaxhbUlISPj4+yjOag4MDgYGB7N+/X+t4uiifD+aTRR24efMm+/btw93dXetL09q1a+nduzdffvklTzzxBH/88Qdbtmxh0KBBzJ8/n44dOzJ//nySk5MByM/P57PPPsPU1JTZs2czefJkTp06VWEXbYCEhATCwsJ48cUXyzVIQGlhadOmDdHR0Vrbo6Oj6datm1bL65o1axg4cCBz5szBwsKChQsXPnDdcd3d3fH399e6OBYvXoxKpWLy5MnMnj0bExMTZs2aRWFhYY3i/v3333F3d2fOnDm89NJLrFq1ilOnTtX2KTzQrly5wvTp02natCmzZs1i2rRp5OfnM3fuXEpKSoDSBx1ra2tmzZrFvHnzeO2118p9nX2QJCQk4OXlVaMW91u3btGgQQOtbSYmJly+fJmLFy8qYe7MFxMTE1QqldYXvsLCQsaOHcs777zD559/zrlz58odz9vbG5VKVeMyX58iIyNZu3YtkyZNUiq7s2fPsmDBAjp16sT8+fMZOHAgGzduZNu2bVr7/v7773h7ezNnzhyefPJJwsPD+eabb2jXrh3z5s3j8ccfZ+HCheXOf+XKlQwaNIjZs2fj6OjI559/TkFBAVCa/15eXnz88ccsWLCAvn37smTJEo4fPw6Udn/29fXl6aefZsmSJSxZsgR7e/t6yKm6d2e9VVVeVCYmJgYjIyNmzJjBm2++yR9//MG+ffvq6Sxqn66v9dtFRUXh7+9frsx5e3uTnZ2tl+OM7yT5WXdMTEwoLi4GShsbY2NjmThxIvPmzSuXf507dyYvL0/rRSY/P5+DBw/SrVs35d/+/v5MmzaNefPmKffk9PR0oLTniZ2dHf3791fuh1D6EWLmzJkEBAQwb948Jk2aRHJyMt9++219ZMM902XZ/Oeff2jevDk//vgjI0eOZOLEiaxfv75c2dXnev3Od6KCggLCwsJo0KABoaGhzJw5ExsbG2bMmKHUuVC+rBobGzN//nyMjIz47LPPGDduHD///LNWXtxr3A0aNCA/P5++ffsya9YsQkNDadSoEXPmzFHiz8/PVxrcJk+ezPz58+nfv3/9ZWQl6qp8tmjRgitXrnDw4EE0Gg3Xrl1j3759tGvXTtmnRYsWJCcnK+8/ly5d4uDBg1phQDfl85FulDhy5AhDhgxhyJAhDBs2jPj4eN59912tMH369KFz5844ODhgZ2fHli1bCA4OpmvXrri4uDBgwAAef/xxNm/eDJR2i8nPz2fChAm4u7vj5+fHqFGjOHDgQLlK8dChQ3z++ecMHz5cadWqSFBQEHFxcUrBSEtL4/Tp0/To0UMr3IABA2jVqhWurq7069eP9PR0srOzayOr6pWbmxuZmZkAXLhwgYMHDzJq1Cj8/Pxwd3dnwoQJ5OXlERMTU6N427RpQ58+fXBycuK5557DycmpyofyR82OHTvw8PBg8ODBuLm54eHhwfjx41GpVJw9exYovYG1adMGV1dXnJyc6NixI76+vjpO+b27ePEiNjY2NdrH39+fgwcPcvToUUpKSsjIyGDr1q0A5OTkANC2bVuio6NRqVRoNBrOnDlDVFQUxcXF5ObmAuDi4sKYMWP46KOPeO+992jQoAHTpk3jwoULWsezsbGhuLhYb6/nY8eO8euvv/LBBx/Qvn17ZfvWrVvx8/MjJCQEFxcXunXrRnBwMJs2bdLav23btvTu3RtnZ2dCQkK4desWjo6OBAYG4uTkRL9+/bh27ZryNaRMv3798Pf3x93dnbFjx1JYWEhsbCxQ2vvqxRdfxNPTE0dHR3r27EmnTp2Ii4sDSocrGBsbY2pqirW1NdbW1g9sD4A73VlvVZUXlXFzc2PAgAG4uLjQpUsXWrZsyYkTJ+rpLGqfLq/122VkZBAfH09QUFC538rSV/aQqc8kP+uGSqUiLi5O+ZJbVFTE+PHj8fLywt3dvdx8WY0bN6Zdu3Zaz0QHDhzA0NBQ+Xrv6elJr169cHd3x8nJiVdffRUvLy/+/vtvJQ5DQ0MaNmyo3A+h9Kttly5dCA4OxtnZGR8fH0aOHMn+/fu5evVqfWTHPdFl2czMzOTvv/+mqKiITz75hAEDBvC///2PNWvWaB1P3+r1u70TxcXFodFoGDt2LB4eHri6ujJq1Cjy8/M5dOiQEsedZfXEiRNkZGQwfvx4PD098fX1ZdiwYUqD2/3EbWRkROfOnencuTPOzs54eHgwduxYsrKyUKlUQOk7WU5ODpMnT+bxxx/HycmJTp066bxnb12VT19fX95//30WLlzIwIEDefvtt9FoNIwfP16J56mnnuKNN95g+vTpvPHGG4wdOxZ3d3cGDRqkdTxdlM9HevjG448/zujRo4HSIQI7duzgs88+47PPPlPC3D5kIi8vjytXrmh1t4HSVqeyrjjp6el4eHjQqFEj5ffmzZtjYGBAWlqaMqHJ2bNnmT9/Pu+++67WGMiKBAQEsHTpUg4cOEDXrl2Jjo7G29sbd3d3rXAeHh7K321tbQG4evUqdnZ21c4TfaDRaJTWw/T0dAwMDLRees3MzHB3dyctLa1G8d6eP1B6welzpaoLZ8+eJSEhocKurmq1Gm9vb55//nkiIiLYs2cPrVu3plOnTg/0REgVffmoSlBQEGq1mrlz51JcXKwMB9qwYYNSdvv3709OTg7Tpk1Do9FgZWVFYGAgmzdvVsL4+vpqle3mzZszefJk/vzzT6UrH/z/eQL08YsKQNOmTcnLy2PDhg00b95cGVaQnp5ervW9RYsW/Pzzz+Tl5Sndf2+/Nhs2bIipqanW/a3sAfnO6/X2vGvYsKHWfaGkpITffvuNffv2kZ2dza1btygqKqJly5a1eOb66fZ6C+49Lx62e6Yur/XbRUVFYWNjo9WAV0bfr/XbSX7WnrIXwpKSEoqKinjiiSd488032b59O7a2tlUO8evWrRvh4eEUFBRgampKbGwsnTp1Us4/Pz+fn3/+mUOHDpGTk0NRURG3bt0q9xx5p7Nnz6JWqyvsIZWZmYmVldW9n3Qd0mXZ1Gg0WFpa8s4772BoaIiXlxfXr1/np59+YsiQIUo4fSubd3snOnv2LFlZWQwdOlRrn8LCQuUjIlCurGZkZGBjY6O8k0Bp/XT7dXyvcUPpc+m6detQqVRcu3aNkpISNBoNly5dAkqHx3t4eGBpaXmv2VIn6qp8pqWl8eOPP9KvXz/atm3LlStXWLVqFUuWLFEaJuLj4/nll194++238fHxQa1Ws2zZMtavX8+AAQOU4+mifD7SjRKmpqZKIwGAl5cXw4YNY+fOnUovBFNT0zo5toODA1ZWVuzevZuAgIByXXJuZ2xsTPfu3YmOjubJJ59k7969WgWnTEWrTTxowzeg9KKqzrjumk5wc2f+GBgYPJD5U5c0Gg3t2rUrVzkAysNHSEgI3bp14/Dhwxw9epQNGzYwcuTIcj13HhQWFhZcv369RvsYGBgwePBgBg4cSE5ODpaWlkqvm7JZ3k1MTBg7diyjRo3i6tWr2NjYsHPnTho1alRpBWloaMhjjz1WrldVWfr0rWItY2Njw5QpUwgLC2PGjBn85z//qXKy2duv34ruXRVNzliT63Xz5s1s2bKFESNG4O7uTsOGDVmzZs0jsYzonfXWvebFw3bP1IdrvaioiD179hAUFFRhudf3a/12kp+1p+yF0MjICBsbG637X3Um/W7fvj1GRkb8888/tG7dmuPHjzN16lTl95UrVyoNH87OzpiamrJo0aJKh8OU0Wg09OjRo8LevLe/aOobXZZNa2trjI2NtXreubq6UlBQQG5urhJO38rm3d6JNBoNnp6evP/+++X2u72uv5cJ6u8n7jlz5mBra8vIkSOxtbXFyMiIDz74oMpyrWt1VT43btyIt7e3Ml+eh4cHDRs25L///S9vvPEGdnZ2REZG8tRTTyk9y9zd3cnPzyciIoL+/fsr91FdlM+Ho69qLTI0NKy0VcjMzAwbGxuSkpK0ticmJiqTqri6upKSksLNmzeV35OSktBoNFoTrzRu3Jhp06aRnZ3N/PnzuXXr1l3TFRQUxIkTJ9i+fTv5+fnKskIPm5SUFI4ePUrnzp2B0vzUaDRacz/k5eUpk3mK2tWsWTPS0tKwt7fHyclJ68/tvX+cnZ3p27cvn3zyCT169GDXrl06TPX98fT0VMbV1pShoSG2trYYGxsTFxeHr69vuRu4sbExdnZ2GBoaEhcXR/v27SsdJqDRaDh//ny5rwGpqanV+lqmS7a2toSGhlJQUMCMGTPIzc3F1dW1wvulnZ2dVnm6V6dPn1b+np+fT2pqqtJrJzExkQ4dOtC9e3dl2MKdw2KMjY2VuVIeZtXJi0eBPlzrBw4cIDc3t9JG3NTUVIyMjKr8gq0PJD9rT9kLYZMmTe5plbcGDRrQuXNnYmNj2bdvH9bW1vj5+Sm/JyYmEhgYSOfOnfHw8MDW1lbrKzRUfD8seya483nAyclJr+eS0mXZbN68OWq1WisvL1y4gKmpKRYWFsq2B6FeL3snatasGWq1GgsLi3Ll4G4fIFxcXLhy5YrWEICzZ89qNW7fa9y5ubmkp6fzyiuv0KZNG9zc3Lh586bW0BBPT0/Onz+vdx8j6qp8FhQUlLtHlv27LM8rC3PnBwddlM9HulHi1q1b5OTkkJOTo3R5yc/PV2YjrsiLL77Ili1biI2NJSMjg3Xr1pGQkEBwcDBQ2oWurAU6JSWF+Ph4lixZQseOHbVaIKG09WnatGlcvny5yoYJFxcXWrRowapVq+jUqZNez3pcXWX5n52dTXJyMlu3biUsLAwvLy8lP52dnQkICOD7778nISGBlJQUFi5ciJmZmTL7sai5mzdvkpycrPUnKyuL3r17k5eXx1dffcXp06fJzMzk2LFjREREcPPmTQoLC/nhhx84efIkWVlZnD59WqtR7kHk7+9PWlqa1lhltVpNcnIyV65coaioSMmjstb3a9eusWPHDtLS0khOTmbZsmX89ddfDB8+XIkjIyODvXv3cuHCBVQqFV999RWpqalaE9pu2LCBI0eOkJmZqUwelpKSQq9evbTSmJCQUG7lCX1kY2PD9OnTKSoq4tNPP6Vv377Ex8ezfv16MjIyiImJYevWrbW26s0vv/zCsWPHSE1N5dtvv8XY2Fi5L7i4uHDixAkSExNJT09n6dKlZGVlae3fpEkTVCoVWVlZStfPh1F18uJRoMtrvUxUVBStWrVSvmzdKSEhgccff7zOemnWJslP/dKtWzeOHj3K//73P5566imtFw9nZ2cOHDjA2bNnleeoOz/ANWnShMTERLKzs5WXuJdeegmVSsWSJUs4d+4carWaQ4cOaS1DrI90WTZ79eqlrBqXkZHBkSNHWL9+Pb169dLqIahv9frd3om6deuGlZUVc+fOJT4+nqysLOLj41mxYsVdG7jbtGmDi4sL4eHhyuSKP/30k1avpnuN29zcHAsLC6KiolCr1cTHx/P9999rxd21a1esrKyYN28eCQkJZGZmcvDgQZ3PjVRX5TMgIICDBw+yY8cOMjMzSUxMZNmyZTRr1kyZBLhDhw5ERUURFxdHVlYWx44dY926dUpvqzK6KJ+P9PCN48ePM2rUKAAaNWqEi4sLEydOpGXLlpU+sD333HPcvHmT1atXk5OTg4uLCx9++CGenp5AaWv31KlTWb58OZ988onWkqAVsbS05L///S+ffvopX3zxBR9++GGl6e3RowcJCQkPbDf5O5Xlv6GhIebm5jRt2pTXXnuNnj17an0pGDt2LMuXL2fu3LnKkqD/93//p9et9Lqye/duFi9eXOXShgkJCXz00Uda2zp16sSHH37IjBkzWLNmjbLCib29PW3btlWGGN24cYPFixdz5coVLCwsaN++vd4vt3Y37u7ueHt7ay199N133xEfH6+EKcur2/N1z549rFy5Eiid2yA0NBRvb29ln5KSErZu3UpGRgZGRka0bNmSmTNnav2/3LhxgyVLlpCTk4OZmRnNmjUjLCxMK57CwkIOHDig1RVXn1lbWzN9+nRmzJjBt99+y3vvvccvv/zCxo0bsba25uWXX661JaYGDRrEihUryMjIoGnTpkyZMkXp4vnqq6+SlZXFrFmzMDEx4emnn6Zbt25ac9EEBwcTHh7OBx98QGFh4QOxJOi9qE5ePAp0ea1D6Rj8EydO8N5771Waxri4OGU5bH0n+alfHn/8cWxtbUlLSyuXJ8OGDeO7775j+vTpmJub07dv33IfwkJCQvj++++ZMGECt27dYv369Xh4eBAWFkZkZCShoaGUlJTg4OBAx44d6/PUakyXZdPe3p6pU6eyYsUKJk+ejLW1Nc888wz9+vVTwuhjvX63dyKAsLAw1qxZw4IFC8jLy8PGxoaWLVsqc0hVxNDQkEmTJhEREcH//d//0aRJE4YOHcr8+fOVZ3hTU9N7jnvixIksW7aMDz/8ECcnJ4YMGcIXX3yhhGnYsCGhoaGsWLFCWZXDxcWFYcOG1UaW3bO6Kp9PP/00N2/eZNu2baxYsQIzMzNatWqlNYllv379MDAwYN26dVy+fBlLS0s6dOjA66+/roTRVfk00DzIA0QfMb/99hvR0dF8/fXXuk6K0FPr16/n77//Zt68eRWOrxUVO3LkCMuWLePLL7/UuxUYtm3bxsGDB/nPf/6j66QI8cDT52v933//ZeXKlcoSeg8CyU+hr/S5bD7K9XpycjIfffQRn3/+OV5eXrpOjs5I+SzPKDQ0NLRejyhqLD8/nwsXLvDjjz8SHBys1SomxO1WrVrF8OHDK+3KKirm5OSERqPBxsbmri3zupCcnEyvXr20xqIKIe6NPl/r586d4+mnn1a62T4IJD+FvtLnsvko1esHDhzg0qVLGBoacu7cOX744QesrKwICQmp8YT1DxMpn+VJT4kHQHh4OHFxcQQEBPDee+9Ji78QQgghhBBCr+3Zs4dff/2VS5cu0bhxY/z8/Bg2bJheT/ApdEMaJYQQQgghhBBCCKET+jWIRQghhBBCCCGEEI8MaZS4i+vXrzNy5EjUarWuk1LOggUL2LJli66TIXREyqbQZ1I+a4/kpRDiTkuXLqUmU8JlZWUREhLCmTNn6i5R4qGmz3XRJ598wt9//63rZIj79EgvCVqVjRs30q5dO5ycnABYtmwZSUlJpKamYm1tTXh4eLl99u3bx8aNG7lw4QKWlpb06dOHF198USvMtm3b2L59O1lZWdjb2/Pqq68SGBio/F62rOOdVq1apSyh079/f6ZPn05QUBBmZma1edq1Jjw8nD179gBgZGSkLPvZqVOncst+ipqRsin0mZTP2iN5KcSjJzw8nNzcXD7++GNdJ0UIQHd1EcDff//NunXryMzMxNHRkTfeeENrWdp+/fqxYsUKOnbsqHcrWYjqk7fCShQUFLBr1y6mTJmibNNoNAQGBpKSksKxY8fK7XP48GG++eYbRowYgb+/P+np6URERGBiYqKsQ7tjxw5Wr17N6NGj8fHxQaVSERERgbm5OQEBAUpcpqamLFy4UCv+sgdBKF3j1tHRkb179ypx66PWrVszYcIESkpKuHbtGidOnGDDhg3ExMQwbdo0GjZsqOskVqq4uBhDQ0O9mx1YyqbQZ1I+a4/kpRBCCF3TZV106tQpvvrqK0JCQujYsSMHDhxgwYIFzJgxAx8fHwDat29PREQER44coX379vWQI6IuSKNEJQ4fPgxA8+bNlW1vvvkmAJs3b67wAty7dy8dOnSgd+/eADg6OvLyyy+zadMmevfujYGBAXv37iUoKIiuXbsqYc6cOcOmTZu0HgaBKmemDQgIIC4uTq8fBhs0aKCch62tLZ6enrRp04YpU6awefNmQkJCKCoqIjIyktjYWK5fv07Tpk0ZMGAAoHpbUgAAHjpJREFU/v7+AJw8eZKwsDCmTZvG2rVrSUlJwc3NjVGjRuHl5UVeXh4jR45k4sSJWnl49OhRPv/8c7777jusrKzIzs5mxYoVHD16FABfX1+GDx+Os7MzAOvXr2f//v0EBwfzyy+/kJWVxU8//aR3DSdSNoU+k/JZeyQvhRAlJSWsWrWK6OhoAAIDAykpKdEKc+TIEX799VdSU1MB8Pb2ZtiwYbi5uWmFu3jxImvWrCEpKYkmTZowYsQI2rRpo/weHx/PqlWrOH/+PGZmZjz11FMMHjxYerY+4nRZF/3++++0bNmSV199FQA3NzdOnjzJ77//zvvvvw+AoaEh7dq1IzY2VholHmDSx6USCQkJeHl51egr+a1bt2jQoIHWNhMTEy5fvszFixeVMLd/aSoLo1KpKCoqUrYVFhYyduxY3nnnHT7//HPOnTtX7nje3t6oVCoKCwtrcmo65+7ujr+/P/v37wdg8eLFJCQk8O677/LFF18QGBjInDlzSE5O1tpvzZo1DBw4kDlz5mBhYcHChQvRaDSYmZnRoUMHYmNjtcLHxMTQpk0brKysKCgoICwsjAYNGhAaGsrMmTOxsbFhxowZFBQUKPtkZWURGxvLxIkTmTdvXrn/T30gZVPoMymftUfyUgixZcsWoqKiGDlyJDNnzqSkpKTc805+fj59+/Zl1qxZhIaG0qhRI+bMmaN1PQNERkby3HPPMW/ePB577DG++uor8vPzAcjOzmb27Nl4enoyZ84c3nnnHeLi4lizZk29navQT7qsi06dOkXbtm21wrRt25ZTp05pbfP29iYhIaHa6RP6RxolKnHx4kVsbGxqtI+/vz8HDx7k6NGjlJSUkJGRwdatWwHIyckBSi+k6OhoVCoVGo2GM2fOEBUVRXFxMbm5uQC4uLgwZswYPvroI9577z0aNGjAtGnTuHDhgtbxbGxsKC4uJjs7uxbOuH65ubmRmZmJWq0mLi6OiRMn4ufnh6OjI3369KFdu3bs3LlTa58BAwbQqlUrXF1d6devH+np6cq5d+/enYMHD3Lz5k2g9GH6n3/+oVu3bgDExcWh0WgYO3YsHh4euLq6MmrUKPLz8zl06JByjKKiIsaPH4+Xlxfu7u4YGRnVU45Un5RNoc+kfNYeyUshxB9//MFLL71Ely5dcHV1Zfjw4eV6MHXu3JnOnTvj7OyMh4cHY8eOJSsrC5VKpRXu+eefJyAgAGdnZwYOHMj169eVD0Dbt2/HxsaGt99+Gzc3Nzp06MCgQYPYtm2b1scb8ejRZV2Uk5ODlZWVVtxWVlZKHGVsbW3Jzs6muLj4Xk9T6Jj0x6pERa13VQkKCkKtVjN37lyKi4tp1KgRffv2ZcOGDUrrYv/+/cnJyWHatGloNBqsrKwIDAxk8+bNShhfX198fX2VeJs3b87kyZP5888/le5S8P/H9j6IX6g0Gg0GBgacO3cOjUbDxIkTtX4vKiqiVatWWts8PDyUv9va2gJw9epV7Ozs8Pf3x9TUlAMHDhAYGMjBgwfRaDQ88cQTAJw9e5asrCyGDh2qFWdhYSGZmZla8VbVXVnXpGwKfSbls/ZIXgrxaMvLy+PKlSta16KhoSHe3t5cvnxZ2aZWq1m3bh0qlYpr165RUlKCRqPh0qVLWvHd/hxV9pJ59epVANLT0/Hx8dGaKLBFixYUFRWhVqu19hWPFl3WRdVlYmKCRqPh1q1bevlBUVRNGiUqYWFhwfXr12u0j4GBAYMHD2bgwIHk5ORgaWnJ8ePHgdJxUlB60YwdO5ZRo0Zx9epVbGxs2LlzJ40aNcLS0rLCeA0NDXnsscfKLcNTlr7K9tNnaWlpODg4KI0Ts2fPLjdm8c4bYEU3GY1GA4CxsTFPPvkksbGxBAYGEhMTQ8eOHTE1NVXCeXp6KuPPbte4cWPl7/o2f0RFpGwKfSbls/ZIXgohqmPOnDnY2toycuRIbG1tMTIy4oMPPig3fOP256iyl76y56i70bcJv0X90mVdZG1trTSclbl69Wq5D4jXr1+nQYMGD8RzvKiYDN+ohKenJ+np6fe0r6GhIba2thgbGxMXF4evr2+5BzZjY2Ps7OwwNDQkLi6O9u3bV7qMjUaj4fz58+UuwNTU1Afiy/6dUlJSOHr0KJ07d8bT0xONRkNOTg5OTk5af8p6Q1RXt27dOH78OGlpaRw5ckQZugHQrFkz1Go1FhYW5Y5ze6PEg0DKptBnUj5rj+SlEI82MzMzbGxstMbPazQarWEZubm5pKen88orr9CmTRvc3Ny4efNmjbuxu7q6cvr0aa1JNBMTEzE2NlZeIsWjSZd1ka+vb7mJNI8dO6bVewhK3y28vLzuKY1CP0hPiUr4+/uzevVqcnNzsbCwAEq7x+Xn53PlyhWKioqUcXhubm4YGxtz7do1/v77b/z8/CgqKiI6Opq//vqLsLAwJd6MjAxUKhU+Pj7cuHGDrVu3kpqayrhx45QwGzZswMfHB2dnZ27evMkff/xBSkoKI0eO1EpjQkJCuclf9M2tW7fIycnRWhJ048aNeHl5ERwcTMOGDenatSuLFy9m6NChNGvWjOvXr3Py5EkcHR3p1KlTtY/VvHlzmjRpwtdff42lpSWtW7dWfuvWrRtbtmxh7ty5DBgwAHt7ey5dusTBgwd59tlnlRU4HgRSNoU+k/JZeyQvhRDPPfccv/32Gy4uLri7u7N9+3ZycnKU4Rfm5uZYWFgQFRWFvb092dnZrFy5ssZd2Hv37s0ff/zBDz/8QN++fcnKymL16tX06dNH6XUqHk26rIv69u3L9OnT+e2333jiiSc4cOAAJ0+e5NNPP9VKY2JiotRFDzhplKiEu7s73t7eWkudfffdd8THxythPvroIwAWLVqEg4MDAHv27GHlypVAaeteaGgo3t7eyj4lJSVs3bqVjIwMjIyMaNmyJTNnzlT2B7hx4wZLliwhJycHMzMzmjVrRlhYmFY8hYWFHDhwgKlTp9ZdJtSC48ePM2rUKAwNDTE3N6dp06a89tpr9OzZUxmuMXbsWH799VdWrVrF5cuXady4Md7e3uXmlKiOrl278ssvv/D8889rffEzNTUlLCyMNWvWsGDBAvLy8rCxsaFly5aYm5vX2vnWBymbQp9J+aw9kpdCiODgYHJycvjuu++A0om9u3btqny5NjQ0ZOLEiSxbtowPP/wQJycnhgwZwhdffFGj49ja2vLJJ5+watUqPvroI8zNzXnqqad44403av2cxINFl3VR8+bNef/994mMjGTdunU4OTnx/vvv4+Pjo4TJzs4mKSmJCRMm1F0miDpnoKnOYLJH1JEjR1i2bBlffvllpV1adWXbtm0cPHiQ//znP7pOitABKZtCn0n5rD2Sl0IIIXRNn+uilStXkpeXx+jRo3WdFHEfjEJDQ0N1nQh95eTkhEajwcbGRu++picnJ9OrVy+lG5V4tEjZFPpMymftkbwUQgiha/pcF50/f57nn39eJrl8wElPCSGEEEIIIYQQQuiEfvW/EUIIIYQQQgghxCNDGiWEEEIIIYQQQgihE9IoIfTayZMnCQkJ4dq1a7pOihBCCCGEEEKIWiaNEkIIIYQQQujQ9evXGTlyJGq1WtdJKWfBggVs2bJF18kQQjzEjHWdACGEEEIIIR5lGzdupF27djg5OQGwbNkykpKSSE1NxdramvDw8HL77Nu3j40bN3LhwgUsLS3p06cPL774olaYbdu2sX37drKysrC3t+fVV18lMDBQ+T01NZX169dz7tw5srKy6N+/PyEhIVpx9O/fn+nTpxMUFISZmVkdnL0Q4lEnjRKi3h05coQFCxawbNkyjIyMUKvVvPvuu/Ts2ZNRo0YBEBkZyenTp3n11VeB0uV+1q5dS0pKCm5ubowaNQovLy8lzqSkJNasWcOZM2cwNzcnICCAQYMGKZVnaGgobm5umJmZERUVhYGBAd27d2fw4MF6t96yEEIIIR4dBQUF7Nq1iylTpijbNBoNgYGBpKSkcOzYsXL7HD58mG+++YYRI0bg7+9Peno6ERERmJiY0KdPHwB27NjB6tWrGT16ND4+PqhUKiIiIpTnpLJjN2nShE6dOhEZGVlh+tzd3XF0dGTv3r1K3EIIUZvkbUzUuxYtWnDr1i3OnDkDlM4bYWFhQXx8vBLm5MmT+Pn5Kf9es2YNAwcOZM6cOVhYWLBw4ULKVrNNSUlh5syZBAQEMG/ePCZNmkRycjLffvut1nFjYmIwMjJixowZvPnmm/zxxx/s27evHs5YCCGEEKJihw8fBqB58+bKtjfffJPnnnsOZ2fnCvfZu3cvHTp0oHfv3jg6OtK+fXtefvllNm3apDwf7d27l6CgILp27YqjoyNPPfUUPXv2ZNOmTUo83t7eDB06lK5du2JqalppGgMCAoiLi6uN0xVCiHKkUULUu4YNG+Ll5cXJkyeB0gaIPn36cPHiRa5cuUJBQQFnzpyhZcuWyj4DBgygVatWuLq60q9fP9LT08nOzgZg8+bNdOnSheDgYJydnfHx8WHkyJHs37+fq1evKnG4ubkxYMAAXFxc6NKlCy1btuTEiRP1e/JCCCGEELdJSEjAy8sLAwODau9z69YtGjRooLXNxMSEy5cvc/HiRSWMiYlJuTAqlYqioqIapdHb2xuVSkVhYWGN9hNCiOqQ4RtCJ/z8/IiPj+eVV14hISGBvn37cvLkSU6ePImlpSVGRkZ4e3uTlJQEgIeHh7Kvra0tAFevXsXOzo6zZ8+iVqsr7PWQmZmJlZVVuTgAbGxstBothBBCCCHq28WLF7GxsanRPv7+/ixfvpyjR4/SunVr1Go1W7duBSAnJwcHBwfatm1LdHQ0HTt25LHHHuPs2bNERUVRXFxMbm5ujY5pY2NDcXEx2dnZyrwXQghRW6RRQuhEy5Yt2bZtG2lpaeTl5eHl5YWfnx8nT57EysoKX19fjI3/f/E0MjIqF0dZ90SNRkOPHj144YUXyoUpa8CoKA4DAwMlDiGEEEIIXaioR0NVgoKCUKvVzJ07l+LiYho1akTfvn3ZsGGD0uOif//+5OTkMG3aNDQaDVZWVgQGBrJ58+Ya9coAlPRJTwkhRF2QRgmhEy1atKCoqIjNmzfTokULDA0NadmyJREREVhZWeHv71/tuJo1a0ZaWpq03AshhBDigWNhYcH169drtI+BgQGDBw9m4MCB5OTkYGlpyfHjxwFwdHQEShsSxo4dy6hRo7h69So2Njbs3LmTRo0aYWlpWaPjlaWvpvsJIUR1yJwSQifK5pWIiYlR5o7w8fHh8uXLnD59Wms+iaq89NJLqFQqlixZwrlz51Cr1Rw6dIglS5bUVfKFEEIIIWqFp6cn6enp97SvoaEhtra2GBsbExcXh6+vb7mGA2NjY+zs7DA0NCQuLo727dvXeOWx1NRUbG1tsba2vqd0CiHE3UhPCaEzfn5+Wg0QJiYm+Pj4cObMGby9vasdj4eHB2FhYURGRhIaGkpJSQkODg507NixrpIuhBBCCFEr/P39Wb16Nbm5uVhYWACgVqvJz8/nypUrFBUVkZycDJRO2m1sbMy1a9f4+++/8fPzo6ioiOjoaP766y/CwsKUeDMyMlCpVPj4+HDjxg22bt1Kamoq48aNU8IUFRWRlpYGlA7NyMnJITk5mYYNG2r1QE1ISKBt27b1kBtCiEeRgUYG1QshhBBCCKEzU6dOpVu3bvTp0weA0NBQraXSyyxatAgHBweuXbvGnDlzSElJAcDX15fXX38dHx8fJWxaWhrffPMNGRkZGBkZ0bJlSwYPHoyLi4sSJisri/Hjx5c7jp+fH6GhoUBpY8XIkSOZOnUqvr6+tXnaQggBSKOEEEIIIYQQOnXkyBGWLVvGl19+WeOhFXVt27ZtHDx4kP/85z+6TooQ4iGlX3c9IYQQQgghHjH+/v707t2by5cv6zop5RgbG/Pmm2/qOhlCiIeY9JQQQgghhBBCCCGETkhPCSGEEEIIIYQQQuiENEoIIYQQQgghhBBCJ6RRQtSLb775hsmTJ1NUVKS1/fjx47zxxhskJSXdV/zr168nPDz8vuIQQgghhBBCCFG/pFFC1Iu33nqL69evs2HDBmVbXl4e3377LcHBwTRv3rxOj39nY4gQQgghhBBCCN0z1nUCxKPB3NycMWPGMHv2bJ544gm8vb356aefMDc3p0uXLsyYMYPExERMTEwICAhgxIgRmJmZARAeHk5ubi4ff/yxEt/69evZv38/X3zxRYXHCw0NxdXVFVNTU/bs2YODgwOzZ88mLS2NlStXkpCQgImJCa1atWL48OFYW1vXSz4IIYQQQgghhPj/pKeEqDdt2rTh2WefJTw8nL///pvY2FjGjRvH7NmzMTU1Zfbs2UyePJlTp06xePHi+z5eTEwMAJ9++injxo3jypUrTJ8+naZNmzJr1iymTZtGfn4+c+fOpaSk5L6PJ4QQQgghhBCiZqRRQtSrwYMHo9Fo+PLLLxkwYAAqlYr8/HwmTJiAu7s7fn5+jBo1igMHDqBWq6sdb0hICOPGjdPa5uDgwNChQ3F1dcXNzY0dO3bg4eHB4MGDcXNzw8PDg/Hjx6NSqTh79mxtn6oQQgghhBBCiCrI8A1Rr0xMTAgODmbZsmW88MILrFy5Eg8PDxo1aqSEad68OQYGBqSlpeHk5HTPx/Ly8tL699mzZ0lISGDIkCHlwqrVary9ve/5WEIIIYQQQgghak4aJUS9MzIywsDAAEPD6nXUMTAwQKPRaG0rLi6ucj9TU1Otf2s0Gtq1a8fQoUPLhbWysqpWWoQQQgghhBBC1B5plBA65erqSnR0NDdv3lR6SyQlJaHRaHBzcwPA0tKS8+fPa+2XnJxc42M1a9aMv/76C3t7e4yNpegLIYQQQgghhK7JnBJCp7p164apqSmLFi0iJSWF+Ph4lixZQseOHZWhG61ateLcuXPs2rULtVrNpk2bSEpKqvGxevfuTV5eHl999RWnT58mMzOTY8eOERERwc2bN2v71IQQQgghhBBCVEE+FwudMjU1ZerUqSxfvpxPPvlEa0nQMv7+/vTv35/IyEgKCgro1q0bvXr14tChQzU6lq2tLTNmzGDNmjXMmjWLwsJC7O3tadu2LQ0aNKjtUxNCCCGEEEIIUQUDzZ2D9YUQQgghhBBCCCHqgQzfEEIIIYQQQgghhE5Io4QQQgghhBBCCCF0QholhBBCCCGEEEIIoRPSKCGEEEIIIYQQQgidkEYJIYQQQgghhBBC6IQ0SohH0tKlSwkNDdV1MoQQQgghhBDikWas6wSIR1N4eDh79uwBwMjICHNzc5o2bUqnTp3o2bMnxsZSNIUQQgghhBDiYSdvfkJnWrduzYQJEygpKeHatWucOHGCDRs2EBMTw7Rp02jYsKGukyiEEEIIIYQQog5Jo4TQmQYNGmBtbQ2Ara0tnp6etGnThilTprB582ZCQkIoKioiMjKS2NhYrl+/TtOmTRkwYAD+/v4AlJSUEBERwYkTJ8jJycHOzo6goCCCg4MxNDRUwqxatYro6GgAAgMDKSkp0c1JCyGEEEIIIYRQSKOE0Cvu7u74+/uzf/9+QkJCWLx4MZmZmbz77rvY2dlx+PBh5syZw+zZs/H09KSkpARbW1smTpyIpaUlKpWKJUuWYGFhQY8ePQDYsmULUVFRjB49Gg8PD7Zv305sbCzNmjXT8dkKIYQQQgghxKNNJroUesfNzY3MzEzUajVxcXFMnDgRPz8/HB0d6dOnD+3atWPnzp0AGBsbM2DAALy9vXFwcKBLly48++yzxMXFKfH98ccfvPTSS3Tp0gVXV1eGDx+u9NAQQgghhBBCCKE70lNC6B2NRoOBgQHnzp1Do9EwceJErd+Liopo1aqV8u8dO3awa9cuLl68SGFhIcXFxTRp0gSAvLw8rly5gq+vrxLe0NAQb29vLl++XD8nJIQQQgghhBCiQtIoIfROWloaDg4OSuPE7Nmzy63GYWJiAsC+ffv46aefGDJkCL6+vpiZmbFt2zb++ecfXSRdCCGEEEIIIUQNSKOE0CspKSkcPXqUV199FU9PTzQaDTk5OVo9I26XmJiIt7c3ffr0UbZlZmYqfzczM8PGxoZTp04pcWg0GlQqFTY2NnV7MkIIIYQQQggh7koaJYTO3Lp1i5ycHK0lQTdu3IiXlxfBwcE0bNiQrl27snjxYoYOHUqzZs24fv06J0+exNHRkU6dOuHs7Mzu3bs5fPgwTk5OxMXFER8fT+PGjZXjPPfcc/z222+4uLjg7u7O9u3bycnJkUYJIYQQQgghhNAxA41Go9F1IsSjJzw8nD179gClczyYm5vTtGlTOnfuTM+ePZXhGkVFRfz666/s3buXy5cv07hxY7y9vXnttdfw8vKiqKiI77//ngMHDqDRaOjUqRNNmjQhOjqa8PBwAIqLi1m5ciW7d+8GoHv37hQXF5Oenk5oaKguTl8IIYT4f+3dX0hTfRzH8Y9rLjRczsJlhGhtE+rCZd4V7GKBq4uIEILoJsEiu+gquygpoasuCioiu/GivJIQa5RFZQmrCMH+QBIsKIdzJNZKpZrbznMRHZ49M+p5Htt5eHq/YDdnv+9333M4N/vyPecHAABEUwIAAAAAAFiELUEBAAAAAIAlaEoAAAAAAABL0JQAAAAAAACWoCkBAAAAAAAsQVMCBTMzM6PW1lYlEgmrS8lz6tQpXbt2zeoyAAAAAOC3Yre6APw++vr6tH79eq1YsUKS1N3drZcvXyoWi6m8vNzcwvPPHjx4oL6+Pk1MTMjpdCoUCmnbtm05awYGBnTz5k29fftWy5cv144dOxQIBMzv7927p/Pnz+flvnz5shwOhySpublZx44dUzAYVGlp6UKeNgAAAADgO2hKoCC+fPmiu3fv6vDhw+YxwzAUCAQ0NjamZ8+e5cWMjIzozJkz2rNnj/x+v8bHx9XV1SWHw6FQKCRJunXrlnp6erRv3z55vV5Fo1F1dXVpyZIlamxsNHMtXrxYZ8+ezcn/rSEhSdXV1XK73RoaGjJzAwAAAAB+LR7fQEGMjIxIkurq6sxjLS0t2rJli6qqquaNGRoa0oYNG9TU1CS3262GhgZt375d/f39MgzDXBMMBrVp0ya53W5t3LhRmzdvVn9/f16+8vLynM9fNTY2KhKJLMTpAgAAAAB+ApMSKIjR0VGtXr1aRUVFPx0zNzen4uLinGMOh0NTU1OanJxUZWWl5ubmciYevq2JRqNKp9Oy27/e4qlUSm1tbcpms6qpqdHOnTtVW1ubE+fxeHTlyhWlUqm8nAAAAACAhcekBApicnJSLpfrb8X4/X4NDw/r6dOnymazisfjCofDkqRkMilJqq+v1+DgoKLRqAzD0KtXr3Tnzh1lMhlNT09LklauXKn9+/ervb1dBw8eVHFxsTo6OjQxMZHzey6XS5lMRu/evVuAMwYAAAAA/AiTEiiI+SYafiQYDCqRSOjkyZPKZDIqKSnR1q1b1dvba05cNDc3K5lMqqOjQ4ZhaOnSpQoEArp69aq5xufzyefzmXnr6up06NAh3bhxQy0tLebxb/WlUql/e7oAAAAAgJ9AUwIFUVZWppmZmb8VU1RUpN27d2vXrl1KJpNyOp16/vy5JMntdkv62khoa2vT3r179eHDB7lcLt2+fVslJSVyOp3z5rXZbFqzZk3e1qTf6vteHAAAAABgYfH4BgqipqZG4+Pj/yjWZrOpoqJCdrtdkUhEPp8vr3Fgt9u1bNky2Ww2RSIRNTQ0yGab//Y2DENv3rzJe9llLBZTRUXFvC/BBAAAAAAsPCYlUBB+v189PT2anp5WWVmZJCmRSOjz5896//690um0Xr9+LUlatWqV7Ha7Pn78qEePHmnt2rVKp9MaHBzUw4cP1dnZaeaNx+OKRqPyer2anZ1VOBxWLBbTgQMHzDW9vb3yer2qqqrSp0+fdP36dY2Njam1tTWnxtHRUdXX1//6iwEAAAAAkERTAgVSXV0tj8ejSCSiUCgkSbpw4YJevHhhrmlvb5cknTt3TpWVlZKk+/fv69KlS5K+vhvi+PHj8ng8Zkw2m1U4HFY8HteiRYu0bt06nThxwoyXpNnZWV28eFHJZFKlpaWqra1VZ2dnTp5UKqXHjx/ryJEjv+4iAAAAAAByFBmGYVhdBH4PT548UXd3t06fPv3dRyusMjAwoOHhYR09etTqUgAAAADgt/Hf+meI/zW/36+mpiZNTU1ZXUoeu92esxMHAAAAAODXY1ICAAAAAABYgkkJAAAAAABgCZoSAAAAAADAEjQlAAAAAACAJWhKAAAAAAAAS9CUAAAAAAAAlqApAQAAAAAALPEHH8VjFbSZjLsAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6zcqiJUEwwdQ" + }, + "source": [ + "Note that cosine annealing scheduler is a bit different from other schedules as soon as it starts with `base_lr` and gradually decreases it to the minimal value while triangle schedulers increase the original rate." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "27Y_3GcPgTfS" + }, + "source": [ + "## MF with PyTorch on ML-100k" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2jqJNaQhK8ji" + }, + "source": [ + "### Utils" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vsby4vwWGlWJ", + "outputId": "52308560-91b9-4db7-daea-3dc502d34bb5" + }, + "source": [ + "%%writefile utils.py\n", + "\n", + "import os\n", + "import requests\n", + "import zipfile\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import scipy.sparse as sp\n", + "\n", + "\"\"\"\n", + "Shamelessly stolen from\n", + "https://github.com/maciejkula/triplet_recommendations_keras\n", + "\"\"\"\n", + "\n", + "\n", + "def train_test_split(interactions, n=10):\n", + " \"\"\"\n", + " Split an interactions matrix into training and test sets.\n", + " Parameters\n", + " ----------\n", + " interactions : np.ndarray\n", + " n : int (default=10)\n", + " Number of items to select / row to place into test.\n", + "\n", + " Returns\n", + " -------\n", + " train : np.ndarray\n", + " test : np.ndarray\n", + " \"\"\"\n", + " test = np.zeros(interactions.shape)\n", + " train = interactions.copy()\n", + " for user in range(interactions.shape[0]):\n", + " if interactions[user, :].nonzero()[0].shape[0] > n:\n", + " test_interactions = np.random.choice(interactions[user, :].nonzero()[0],\n", + " size=n,\n", + " replace=False)\n", + " train[user, test_interactions] = 0.\n", + " test[user, test_interactions] = interactions[user, test_interactions]\n", + "\n", + " # Test and training are truly disjoint\n", + " assert(np.all((train * test) == 0))\n", + " return train, test\n", + "\n", + "\n", + "def _get_data_path():\n", + " \"\"\"\n", + " Get path to the movielens dataset file.\n", + " \"\"\"\n", + " data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),\n", + " 'data')\n", + " if not os.path.exists(data_path):\n", + " print('Making data path')\n", + " os.mkdir(data_path)\n", + " return data_path\n", + "\n", + "\n", + "def _download_movielens(dest_path):\n", + " \"\"\"\n", + " Download the dataset.\n", + " \"\"\"\n", + "\n", + " url = 'http://files.grouplens.org/datasets/movielens/ml-100k.zip'\n", + " req = requests.get(url, stream=True)\n", + "\n", + " print('Downloading MovieLens data')\n", + "\n", + " with open(os.path.join(dest_path, 'ml-100k.zip'), 'wb') as fd:\n", + " for chunk in req.iter_content(chunk_size=None):\n", + " fd.write(chunk)\n", + "\n", + " with zipfile.ZipFile(os.path.join(dest_path, 'ml-100k.zip'), 'r') as z:\n", + " z.extractall(dest_path)\n", + "\n", + "\n", + "def read_movielens_df():\n", + " path = _get_data_path()\n", + " zipfile = os.path.join(path, 'ml-100k.zip')\n", + " if not os.path.isfile(zipfile):\n", + " _download_movielens(path)\n", + " fname = os.path.join(path, 'ml-100k', 'u.data')\n", + " names = ['user_id', 'item_id', 'rating', 'timestamp']\n", + " df = pd.read_csv(fname, sep='\\t', names=names)\n", + " return df\n", + "\n", + "\n", + "def get_movielens_interactions():\n", + " df = read_movielens_df()\n", + "\n", + " n_users = df.user_id.unique().shape[0]\n", + " n_items = df.item_id.unique().shape[0]\n", + "\n", + " interactions = np.zeros((n_users, n_items))\n", + " for row in df.itertuples():\n", + " interactions[row[1] - 1, row[2] - 1] = row[3]\n", + " return interactions\n", + "\n", + "\n", + "def get_movielens_train_test_split(implicit=False):\n", + " interactions = get_movielens_interactions()\n", + " if implicit:\n", + " interactions = (interactions >= 4).astype(np.float32)\n", + " train, test = train_test_split(interactions)\n", + " train = sp.coo_matrix(train)\n", + " test = sp.coo_matrix(test)\n", + " return train, test" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Writing utils.py\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d9wghzkqLR2i" + }, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EWmOo0AqKiEN", + "outputId": "5b4b9d36-218e-42ed-da99-f5af0b4b93f9" + }, + "source": [ + "%%writefile metrics.py\n", + "\n", + "import numpy as np\n", + "from sklearn.metrics import roc_auc_score\n", + "from torch import multiprocessing as mp\n", + "import torch\n", + "\n", + "\n", + "def get_row_indices(row, interactions):\n", + " start = interactions.indptr[row]\n", + " end = interactions.indptr[row + 1]\n", + " return interactions.indices[start:end]\n", + "\n", + "\n", + "def auc(model, interactions, num_workers=1):\n", + " aucs = []\n", + " processes = []\n", + " n_users = interactions.shape[0]\n", + " mp_batch = int(np.ceil(n_users / num_workers))\n", + "\n", + " queue = mp.Queue()\n", + " rows = np.arange(n_users)\n", + " np.random.shuffle(rows)\n", + " for rank in range(num_workers):\n", + " start = rank * mp_batch\n", + " end = np.min((start + mp_batch, n_users))\n", + " p = mp.Process(target=batch_auc,\n", + " args=(queue, rows[start:end], interactions, model))\n", + " p.start()\n", + " processes.append(p)\n", + "\n", + " while True:\n", + " is_alive = False\n", + " for p in processes:\n", + " if p.is_alive():\n", + " is_alive = True\n", + " break\n", + " if not is_alive and queue.empty():\n", + " break\n", + "\n", + " while not queue.empty():\n", + " aucs.append(queue.get())\n", + "\n", + " queue.close()\n", + " for p in processes:\n", + " p.join()\n", + " return np.mean(aucs)\n", + "\n", + "\n", + "def batch_auc(queue, rows, interactions, model):\n", + " n_items = interactions.shape[1]\n", + " items = torch.arange(0, n_items).long()\n", + " users_init = torch.ones(n_items).long()\n", + " for row in rows:\n", + " row = int(row)\n", + " users = users_init.fill_(row)\n", + "\n", + " preds = model.predict(users, items)\n", + " actuals = get_row_indices(row, interactions)\n", + "\n", + " if len(actuals) == 0:\n", + " continue\n", + " y_test = np.zeros(n_items)\n", + " y_test[actuals] = 1\n", + " queue.put(roc_auc_score(y_test, preds.data.numpy()))\n", + "\n", + "\n", + "def patk(model, interactions, num_workers=1, k=5):\n", + " patks = []\n", + " processes = []\n", + " n_users = interactions.shape[0]\n", + " mp_batch = int(np.ceil(n_users / num_workers))\n", + "\n", + " queue = mp.Queue()\n", + " rows = np.arange(n_users)\n", + " np.random.shuffle(rows)\n", + " for rank in range(num_workers):\n", + " start = rank * mp_batch\n", + " end = np.min((start + mp_batch, n_users))\n", + " p = mp.Process(target=batch_patk,\n", + " args=(queue, rows[start:end], interactions, model),\n", + " kwargs={'k': k})\n", + " p.start()\n", + " processes.append(p)\n", + "\n", + " while True:\n", + " is_alive = False\n", + " for p in processes:\n", + " if p.is_alive():\n", + " is_alive = True\n", + " break\n", + " if not is_alive and queue.empty():\n", + " break\n", + "\n", + " while not queue.empty():\n", + " patks.append(queue.get())\n", + "\n", + " queue.close()\n", + " for p in processes:\n", + " p.join()\n", + " return np.mean(patks)\n", + "\n", + "\n", + "def batch_patk(queue, rows, interactions, model, k=5):\n", + " n_items = interactions.shape[1]\n", + "\n", + " items = torch.arange(0, n_items).long()\n", + " users_init = torch.ones(n_items).long()\n", + " for row in rows:\n", + " row = int(row)\n", + " users = users_init.fill_(row)\n", + "\n", + " preds = model.predict(users, items)\n", + " actuals = get_row_indices(row, interactions)\n", + "\n", + " if len(actuals) == 0:\n", + " continue\n", + "\n", + " top_k = np.argpartition(-np.squeeze(preds.data.numpy()), k)\n", + " top_k = set(top_k[:k])\n", + " true_pids = set(actuals)\n", + " if true_pids:\n", + " queue.put(len(top_k & true_pids) / float(k))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Writing metrics.py\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7N1Rl15-LJAj" + }, + "source": [ + "### Model" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "NUlsa6LeLKqu", + "outputId": "6326b89c-3659-4008-8801-f8d3b991b6fc" + }, + "source": [ + "%%writefile torchmf.py\n", + "\n", + "import collections\n", + "import os\n", + "\n", + "import numpy as np\n", + "from sklearn.metrics import roc_auc_score\n", + "import torch\n", + "from torch import nn\n", + "import torch.multiprocessing as mp\n", + "import torch.utils.data as data\n", + "from tqdm import tqdm\n", + "\n", + "import metrics\n", + "\n", + "\n", + "# Models\n", + "# Interactions Dataset => Singular Iter => Singular Loss\n", + "# Pairwise Datasets => Pairwise Iter => Pairwise Loss\n", + "# Pairwise Iters\n", + "# Loss Functions\n", + "# Optimizers\n", + "# Metric callbacks\n", + "\n", + "# Serve up users, items (and items could be pos_items, neg_items)\n", + "# In this case, the iteration remains the same. Pass both items into a model\n", + "# which is a concat of the base model. it handles the pos and neg_items\n", + "# accordingly. define the loss after.\n", + "\n", + "\n", + "class Interactions(data.Dataset):\n", + " \"\"\"\n", + " Hold data in the form of an interactions matrix.\n", + " Typical use-case is like a ratings matrix:\n", + " - Users are the rows\n", + " - Items are the columns\n", + " - Elements of the matrix are the ratings given by a user for an item.\n", + " \"\"\"\n", + "\n", + " def __init__(self, mat):\n", + " self.mat = mat.astype(np.float32).tocoo()\n", + " self.n_users = self.mat.shape[0]\n", + " self.n_items = self.mat.shape[1]\n", + "\n", + " def __getitem__(self, index):\n", + " row = self.mat.row[index]\n", + " col = self.mat.col[index]\n", + " val = self.mat.data[index]\n", + " return (row, col), val\n", + "\n", + " def __len__(self):\n", + " return self.mat.nnz\n", + "\n", + "\n", + "class PairwiseInteractions(data.Dataset):\n", + " \"\"\"\n", + " Sample data from an interactions matrix in a pairwise fashion. The row is\n", + " treated as the main dimension, and the columns are sampled pairwise.\n", + " \"\"\"\n", + "\n", + " def __init__(self, mat):\n", + " self.mat = mat.astype(np.float32).tocoo()\n", + "\n", + " self.n_users = self.mat.shape[0]\n", + " self.n_items = self.mat.shape[1]\n", + "\n", + " self.mat_csr = self.mat.tocsr()\n", + " if not self.mat_csr.has_sorted_indices:\n", + " self.mat_csr.sort_indices()\n", + "\n", + " def __getitem__(self, index):\n", + " row = self.mat.row[index]\n", + " found = False\n", + "\n", + " while not found:\n", + " neg_col = np.random.randint(self.n_items)\n", + " if self.not_rated(row, neg_col, self.mat_csr.indptr,\n", + " self.mat_csr.indices):\n", + " found = True\n", + "\n", + " pos_col = self.mat.col[index]\n", + " val = self.mat.data[index]\n", + "\n", + " return (row, (pos_col, neg_col)), val\n", + "\n", + " def __len__(self):\n", + " return self.mat.nnz\n", + "\n", + " @staticmethod\n", + " def not_rated(row, col, indptr, indices):\n", + " # similar to use of bsearch in lightfm\n", + " start = indptr[row]\n", + " end = indptr[row + 1]\n", + " searched = np.searchsorted(indices[start:end], col, 'right')\n", + " if searched >= (end - start):\n", + " # After the array\n", + " return False\n", + " return col != indices[searched] # Not found\n", + "\n", + " def get_row_indices(self, row):\n", + " start = self.mat_csr.indptr[row]\n", + " end = self.mat_csr.indptr[row + 1]\n", + " return self.mat_csr.indices[start:end]\n", + "\n", + "\n", + "class BaseModule(nn.Module):\n", + " \"\"\"\n", + " Base module for explicit matrix factorization.\n", + " \"\"\"\n", + " \n", + " def __init__(self,\n", + " n_users,\n", + " n_items,\n", + " n_factors=40,\n", + " dropout_p=0,\n", + " sparse=False):\n", + " \"\"\"\n", + "\n", + " Parameters\n", + " ----------\n", + " n_users : int\n", + " Number of users\n", + " n_items : int\n", + " Number of items\n", + " n_factors : int\n", + " Number of latent factors (or embeddings or whatever you want to\n", + " call it).\n", + " dropout_p : float\n", + " p in nn.Dropout module. Probability of dropout.\n", + " sparse : bool\n", + " Whether or not to treat embeddings as sparse. NOTE: cannot use\n", + " weight decay on the optimizer if sparse=True. Also, can only use\n", + " Adagrad.\n", + " \"\"\"\n", + " super(BaseModule, self).__init__()\n", + " self.n_users = n_users\n", + " self.n_items = n_items\n", + " self.n_factors = n_factors\n", + " self.user_biases = nn.Embedding(n_users, 1, sparse=sparse)\n", + " self.item_biases = nn.Embedding(n_items, 1, sparse=sparse)\n", + " self.user_embeddings = nn.Embedding(n_users, n_factors, sparse=sparse)\n", + " self.item_embeddings = nn.Embedding(n_items, n_factors, sparse=sparse)\n", + " \n", + " self.dropout_p = dropout_p\n", + " self.dropout = nn.Dropout(p=self.dropout_p)\n", + "\n", + " self.sparse = sparse\n", + " \n", + " def forward(self, users, items):\n", + " \"\"\"\n", + " Forward pass through the model. For a single user and item, this\n", + " looks like:\n", + "\n", + " user_bias + item_bias + user_embeddings.dot(item_embeddings)\n", + "\n", + " Parameters\n", + " ----------\n", + " users : np.ndarray\n", + " Array of user indices\n", + " items : np.ndarray\n", + " Array of item indices\n", + "\n", + " Returns\n", + " -------\n", + " preds : np.ndarray\n", + " Predicted ratings.\n", + "\n", + " \"\"\"\n", + " ues = self.user_embeddings(users)\n", + " uis = self.item_embeddings(items)\n", + "\n", + " preds = self.user_biases(users)\n", + " preds += self.item_biases(items)\n", + " preds += (self.dropout(ues) * self.dropout(uis)).sum(dim=1, keepdim=True)\n", + "\n", + " return preds.squeeze()\n", + " \n", + " def __call__(self, *args):\n", + " return self.forward(*args)\n", + "\n", + " def predict(self, users, items):\n", + " return self.forward(users, items)\n", + "\n", + "\n", + "def bpr_loss(preds, vals):\n", + " sig = nn.Sigmoid()\n", + " return (1.0 - sig(preds)).pow(2).sum()\n", + "\n", + "\n", + "class BPRModule(nn.Module):\n", + " \n", + " def __init__(self,\n", + " n_users,\n", + " n_items,\n", + " n_factors=40,\n", + " dropout_p=0,\n", + " sparse=False,\n", + " model=BaseModule):\n", + " super(BPRModule, self).__init__()\n", + "\n", + " self.n_users = n_users\n", + " self.n_items = n_items\n", + " self.n_factors = n_factors\n", + " self.dropout_p = dropout_p\n", + " self.sparse = sparse\n", + " self.pred_model = model(\n", + " self.n_users,\n", + " self.n_items,\n", + " n_factors=n_factors,\n", + " dropout_p=dropout_p,\n", + " sparse=sparse\n", + " )\n", + "\n", + " def forward(self, users, items):\n", + " assert isinstance(items, tuple), \\\n", + " 'Must pass in items as (pos_items, neg_items)'\n", + " # Unpack\n", + " (pos_items, neg_items) = items\n", + " pos_preds = self.pred_model(users, pos_items)\n", + " neg_preds = self.pred_model(users, neg_items)\n", + " return pos_preds - neg_preds\n", + "\n", + " def predict(self, users, items):\n", + " return self.pred_model(users, items)\n", + "\n", + "\n", + "class BasePipeline:\n", + " \"\"\"\n", + " Class defining a training pipeline. Instantiates data loaders, model,\n", + " and optimizer. Handles training for multiple epochs and keeping track of\n", + " train and test loss.\n", + " \"\"\"\n", + "\n", + " def __init__(self,\n", + " train,\n", + " test=None,\n", + " model=BaseModule,\n", + " n_factors=40,\n", + " batch_size=32,\n", + " dropout_p=0.02,\n", + " sparse=False,\n", + " lr=0.01,\n", + " weight_decay=0.,\n", + " optimizer=torch.optim.Adam,\n", + " loss_function=nn.MSELoss(reduction='sum'),\n", + " n_epochs=10,\n", + " verbose=False,\n", + " random_seed=None,\n", + " interaction_class=Interactions,\n", + " hogwild=False,\n", + " num_workers=0,\n", + " eval_metrics=None,\n", + " k=5):\n", + " self.train = train\n", + " self.test = test\n", + "\n", + " if hogwild:\n", + " num_loader_workers = 0\n", + " else:\n", + " num_loader_workers = num_workers\n", + " self.train_loader = data.DataLoader(\n", + " interaction_class(train), batch_size=batch_size, shuffle=True,\n", + " num_workers=num_loader_workers)\n", + " if self.test is not None:\n", + " self.test_loader = data.DataLoader(\n", + " interaction_class(test), batch_size=batch_size, shuffle=True,\n", + " num_workers=num_loader_workers)\n", + " self.num_workers = num_workers\n", + " self.n_users = self.train.shape[0]\n", + " self.n_items = self.train.shape[1]\n", + " self.n_factors = n_factors\n", + " self.batch_size = batch_size\n", + " self.dropout_p = dropout_p\n", + " self.lr = lr\n", + " self.weight_decay = weight_decay\n", + " self.loss_function = loss_function\n", + " self.n_epochs = n_epochs\n", + " if sparse:\n", + " assert weight_decay == 0.0\n", + " self.model = model(self.n_users,\n", + " self.n_items,\n", + " n_factors=self.n_factors,\n", + " dropout_p=self.dropout_p,\n", + " sparse=sparse)\n", + " self.optimizer = optimizer(self.model.parameters(),\n", + " lr=self.lr,\n", + " weight_decay=self.weight_decay)\n", + " self.warm_start = False\n", + " self.losses = collections.defaultdict(list)\n", + " self.verbose = verbose\n", + " self.hogwild = hogwild\n", + " if random_seed is not None:\n", + " if self.hogwild:\n", + " random_seed += os.getpid()\n", + " torch.manual_seed(random_seed)\n", + " np.random.seed(random_seed)\n", + "\n", + " if eval_metrics is None:\n", + " eval_metrics = []\n", + " self.eval_metrics = eval_metrics\n", + " self.k = k\n", + "\n", + " def break_grads(self):\n", + " for param in self.model.parameters():\n", + " # Break gradient sharing\n", + " if param.grad is not None:\n", + " param.grad.data = param.grad.data.clone()\n", + "\n", + " def fit(self):\n", + " for epoch in range(1, self.n_epochs + 1):\n", + "\n", + " if self.hogwild:\n", + " self.model.share_memory()\n", + " processes = []\n", + " train_losses = []\n", + " queue = mp.Queue()\n", + " for rank in range(self.num_workers):\n", + " p = mp.Process(target=self._fit_epoch,\n", + " kwargs={'epoch': epoch,\n", + " 'queue': queue})\n", + " p.start()\n", + " processes.append(p)\n", + " for p in processes:\n", + " p.join()\n", + "\n", + " while True:\n", + " is_alive = False\n", + " for p in processes:\n", + " if p.is_alive():\n", + " is_alive = True\n", + " break\n", + " if not is_alive and queue.empty():\n", + " break\n", + "\n", + " while not queue.empty():\n", + " train_losses.append(queue.get())\n", + " queue.close()\n", + " train_loss = np.mean(train_losses)\n", + " else:\n", + " train_loss = self._fit_epoch(epoch)\n", + "\n", + " self.losses['train'].append(train_loss)\n", + " row = 'Epoch: {0:^3} train: {1:^10.5f}'.format(epoch, self.losses['train'][-1])\n", + " if self.test is not None:\n", + " self.losses['test'].append(self._validation_loss())\n", + " row += 'val: {0:^10.5f}'.format(self.losses['test'][-1])\n", + " for metric in self.eval_metrics:\n", + " func = getattr(metrics, metric)\n", + " res = func(self.model, self.test_loader.dataset.mat_csr,\n", + " num_workers=self.num_workers)\n", + " self.losses['eval-{}'.format(metric)].append(res)\n", + " row += 'eval-{0}: {1:^10.5f}'.format(metric, res)\n", + " self.losses['epoch'].append(epoch)\n", + " if self.verbose:\n", + " print(row)\n", + "\n", + " def _fit_epoch(self, epoch=1, queue=None):\n", + " if self.hogwild:\n", + " self.break_grads()\n", + "\n", + " self.model.train()\n", + " total_loss = torch.Tensor([0])\n", + " pbar = tqdm(enumerate(self.train_loader),\n", + " total=len(self.train_loader),\n", + " desc='({0:^3})'.format(epoch))\n", + " for batch_idx, ((row, col), val) in pbar:\n", + " self.optimizer.zero_grad()\n", + "\n", + " row = row.long()\n", + " # TODO: turn this into a collate_fn like the data_loader\n", + " if isinstance(col, list):\n", + " col = tuple(c.long() for c in col)\n", + " else:\n", + " col = col.long()\n", + " val = val.float()\n", + "\n", + " preds = self.model(row, col)\n", + " loss = self.loss_function(preds, val)\n", + " loss.backward()\n", + "\n", + " self.optimizer.step()\n", + "\n", + " total_loss += loss.item()\n", + " batch_loss = loss.item() / row.size()[0]\n", + " pbar.set_postfix(train_loss=batch_loss)\n", + " total_loss /= self.train.nnz\n", + " if queue is not None:\n", + " queue.put(total_loss[0])\n", + " else:\n", + " return total_loss[0]\n", + "\n", + " def _validation_loss(self):\n", + " self.model.eval()\n", + " total_loss = torch.Tensor([0])\n", + " for batch_idx, ((row, col), val) in enumerate(self.test_loader):\n", + " row = row.long()\n", + " if isinstance(col, list):\n", + " col = tuple(c.long() for c in col)\n", + " else:\n", + " col = col.long()\n", + " val = val.float()\n", + "\n", + " preds = self.model(row, col)\n", + " loss = self.loss_function(preds, val)\n", + " total_loss += loss.item()\n", + "\n", + " total_loss /= self.test.nnz\n", + " return total_loss[0]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Writing torchmf.py\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5KpaDgMwLNI5" + }, + "source": [ + "### Trainer" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HxKI9a2dLDDy", + "outputId": "538238ee-aa22-49e9-aa44-68eb30d5e92a" + }, + "source": [ + "%%writefile run.py\n", + "\n", + "import argparse\n", + "import pickle\n", + "\n", + "import torch\n", + "\n", + "from torchmf import (BaseModule, BPRModule, BasePipeline,\n", + " bpr_loss, PairwiseInteractions)\n", + "import utils\n", + "\n", + "\n", + "def explicit():\n", + " train, test = utils.get_movielens_train_test_split()\n", + " pipeline = BasePipeline(train, test=test, model=BaseModule,\n", + " n_factors=10, batch_size=1024, dropout_p=0.02,\n", + " lr=0.02, weight_decay=0.1,\n", + " optimizer=torch.optim.Adam, n_epochs=40,\n", + " verbose=True, random_seed=2017)\n", + " pipeline.fit()\n", + "\n", + "\n", + "def implicit():\n", + " train, test = utils.get_movielens_train_test_split(implicit=True)\n", + "\n", + " pipeline = BasePipeline(train, test=test, verbose=True,\n", + " batch_size=1024, num_workers=4,\n", + " n_factors=20, weight_decay=0,\n", + " dropout_p=0., lr=.2, sparse=True,\n", + " optimizer=torch.optim.SGD, n_epochs=40,\n", + " random_seed=2017, loss_function=bpr_loss,\n", + " model=BPRModule,\n", + " interaction_class=PairwiseInteractions,\n", + " eval_metrics=('auc', 'patk'))\n", + " pipeline.fit()\n", + "\n", + "\n", + "def hogwild():\n", + " train, test = utils.get_movielens_train_test_split(implicit=True)\n", + "\n", + " pipeline = BasePipeline(train, test=test, verbose=True,\n", + " batch_size=1024, num_workers=4,\n", + " n_factors=20, weight_decay=0,\n", + " dropout_p=0., lr=.2, sparse=True,\n", + " optimizer=torch.optim.SGD, n_epochs=40,\n", + " random_seed=2017, loss_function=bpr_loss,\n", + " model=BPRModule, hogwild=True,\n", + " interaction_class=PairwiseInteractions,\n", + " eval_metrics=('auc', 'patk'))\n", + " pipeline.fit()\n", + "\n", + "\n", + "if __name__ == '__main__':\n", + " parser = argparse.ArgumentParser(description='torchmf')\n", + " parser.add_argument('--example',\n", + " help='explicit, implicit, or hogwild')\n", + " args = parser.parse_args()\n", + " if args.example == 'explicit':\n", + " explicit()\n", + " elif args.example == 'implicit':\n", + " implicit()\n", + " elif args.example == 'hogwild':\n", + " hogwild()\n", + " else:\n", + " print('example must be explicit, implicit, or hogwild')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Writing run.py\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "40lNybzWLtRP" + }, + "source": [ + "### Explicit Model Training" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0i4BoW9HLHSb", + "outputId": "b2a2a2bc-c5c8-4c6e-f576-6a68326d5a30" + }, + "source": [ + "!python run.py --example explicit" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Making data path\n", + "Downloading MovieLens data\n", + "( 1 ): 100% 89/89 [00:01<00:00, 74.87it/s, train_loss=7.64] \n", + "Epoch: 1 train: 14.61587 val: 8.83048 \n", + "( 2 ): 100% 89/89 [00:00<00:00, 112.92it/s, train_loss=2.5]\n", + "Epoch: 2 train: 4.20514 val: 4.05539 \n", + "( 3 ): 100% 89/89 [00:00<00:00, 112.48it/s, train_loss=1.57]\n", + "Epoch: 3 train: 1.86044 val: 2.45105 \n", + "( 4 ): 100% 89/89 [00:00<00:00, 113.76it/s, train_loss=1.15]\n", + "Epoch: 4 train: 1.20612 val: 1.82121 \n", + "( 5 ): 100% 89/89 [00:00<00:00, 110.97it/s, train_loss=0.966]\n", + "Epoch: 5 train: 0.98724 val: 1.51758 \n", + "( 6 ): 100% 89/89 [00:00<00:00, 112.02it/s, train_loss=0.89]\n", + "Epoch: 6 train: 0.89150 val: 1.35180 \n", + "( 7 ): 100% 89/89 [00:00<00:00, 112.91it/s, train_loss=0.906]\n", + "Epoch: 7 train: 0.83810 val: 1.25295 \n", + "( 8 ): 100% 89/89 [00:00<00:00, 108.44it/s, train_loss=0.873]\n", + "Epoch: 8 train: 0.80769 val: 1.18821 \n", + "( 9 ): 100% 89/89 [00:00<00:00, 113.67it/s, train_loss=0.83]\n", + "Epoch: 9 train: 0.78222 val: 1.15017 \n", + "(10 ): 100% 89/89 [00:00<00:00, 113.89it/s, train_loss=0.777]\n", + "Epoch: 10 train: 0.76105 val: 1.11414 \n", + "(11 ): 100% 89/89 [00:00<00:00, 113.23it/s, train_loss=0.73]\n", + "Epoch: 11 train: 0.74182 val: 1.08541 \n", + "(12 ): 100% 89/89 [00:00<00:00, 112.17it/s, train_loss=0.64]\n", + "Epoch: 12 train: 0.72437 val: 1.06774 \n", + "(13 ): 100% 89/89 [00:00<00:00, 107.08it/s, train_loss=0.733]\n", + "Epoch: 13 train: 0.70896 val: 1.05505 \n", + "(14 ): 100% 89/89 [00:00<00:00, 111.47it/s, train_loss=0.702]\n", + "Epoch: 14 train: 0.69648 val: 1.03989 \n", + "(15 ): 100% 89/89 [00:00<00:00, 114.82it/s, train_loss=0.69]\n", + "Epoch: 15 train: 0.68401 val: 1.03105 \n", + "(16 ): 100% 89/89 [00:00<00:00, 113.08it/s, train_loss=0.772]\n", + "Epoch: 16 train: 0.67320 val: 1.02541 \n", + "(17 ): 100% 89/89 [00:00<00:00, 112.42it/s, train_loss=0.624]\n", + "Epoch: 17 train: 0.66667 val: 1.01918 \n", + "(18 ): 100% 89/89 [00:00<00:00, 113.76it/s, train_loss=0.671]\n", + "Epoch: 18 train: 0.65996 val: 1.01878 \n", + "(19 ): 100% 89/89 [00:00<00:00, 113.38it/s, train_loss=0.667]\n", + "Epoch: 19 train: 0.65364 val: 1.01307 \n", + "(20 ): 100% 89/89 [00:00<00:00, 110.37it/s, train_loss=0.745]\n", + "Epoch: 20 train: 0.64888 val: 1.01569 \n", + "(21 ): 100% 89/89 [00:00<00:00, 113.61it/s, train_loss=0.671]\n", + "Epoch: 21 train: 0.64512 val: 1.01603 \n", + "(22 ): 100% 89/89 [00:00<00:00, 108.82it/s, train_loss=0.623]\n", + "Epoch: 22 train: 0.64155 val: 1.01564 \n", + "(23 ): 100% 89/89 [00:00<00:00, 112.19it/s, train_loss=0.677]\n", + "Epoch: 23 train: 0.63771 val: 1.01452 \n", + "(24 ): 100% 89/89 [00:00<00:00, 114.23it/s, train_loss=0.739]\n", + "Epoch: 24 train: 0.63746 val: 1.00893 \n", + "(25 ): 100% 89/89 [00:00<00:00, 114.42it/s, train_loss=0.766]\n", + "Epoch: 25 train: 0.63591 val: 1.01990 \n", + "(26 ): 100% 89/89 [00:00<00:00, 112.95it/s, train_loss=0.586]\n", + "Epoch: 26 train: 0.63194 val: 1.01370 \n", + "(27 ): 100% 89/89 [00:00<00:00, 111.70it/s, train_loss=0.734]\n", + "Epoch: 27 train: 0.63205 val: 1.01533 \n", + "(28 ): 100% 89/89 [00:00<00:00, 112.88it/s, train_loss=0.733]\n", + "Epoch: 28 train: 0.63321 val: 1.01158 \n", + "(29 ): 100% 89/89 [00:00<00:00, 107.37it/s, train_loss=0.645]\n", + "Epoch: 29 train: 0.63266 val: 1.01819 \n", + "(30 ): 100% 89/89 [00:00<00:00, 112.43it/s, train_loss=0.683]\n", + "Epoch: 30 train: 0.63357 val: 1.01789 \n", + "(31 ): 100% 89/89 [00:00<00:00, 109.35it/s, train_loss=0.7]\n", + "Epoch: 31 train: 0.63155 val: 1.01247 \n", + "(32 ): 100% 89/89 [00:00<00:00, 113.49it/s, train_loss=0.68]\n", + "Epoch: 32 train: 0.63328 val: 1.01842 \n", + "(33 ): 100% 89/89 [00:00<00:00, 112.89it/s, train_loss=0.68]\n", + "Epoch: 33 train: 0.63136 val: 1.01667 \n", + "(34 ): 100% 89/89 [00:00<00:00, 113.90it/s, train_loss=0.752]\n", + "Epoch: 34 train: 0.63255 val: 1.01864 \n", + "(35 ): 100% 89/89 [00:00<00:00, 113.94it/s, train_loss=0.716]\n", + "Epoch: 35 train: 0.63282 val: 1.01362 \n", + "(36 ): 100% 89/89 [00:00<00:00, 112.76it/s, train_loss=0.618]\n", + "Epoch: 36 train: 0.63292 val: 1.01480 \n", + "(37 ): 100% 89/89 [00:00<00:00, 113.63it/s, train_loss=0.666]\n", + "Epoch: 37 train: 0.63206 val: 1.02341 \n", + "(38 ): 100% 89/89 [00:00<00:00, 107.43it/s, train_loss=0.652]\n", + "Epoch: 38 train: 0.63254 val: 1.02066 \n", + "(39 ): 100% 89/89 [00:00<00:00, 112.98it/s, train_loss=0.65]\n", + "Epoch: 39 train: 0.63397 val: 1.01905 \n", + "(40 ): 100% 89/89 [00:00<00:00, 109.15it/s, train_loss=0.732]\n", + "Epoch: 40 train: 0.63401 val: 1.01783 \n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JxqWyDE4LxPb" + }, + "source": [ + "### Implicit Model Training" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "IBXdHOUXLdPE", + "outputId": "62bd667d-56d6-4969-e390-8cab4842353e" + }, + "source": [ + "!python run.py --example implicit" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", + " cpuset_checked))\n", + "( 1 ): 100% 46/46 [00:01<00:00, 28.55it/s, train_loss=0.382]\n", + "Epoch: 1 train: 0.41578 val: 0.39289 eval-auc: 0.55840 eval-patk: 0.00913 \n", + "( 2 ): 100% 46/46 [00:01<00:00, 28.86it/s, train_loss=0.323]\n", + "Epoch: 2 train: 0.34652 val: 0.34228 eval-auc: 0.61282 eval-patk: 0.01507 \n", + "( 3 ): 100% 46/46 [00:01<00:00, 30.01it/s, train_loss=0.273]\n", + "Epoch: 3 train: 0.27728 val: 0.31357 eval-auc: 0.65768 eval-patk: 0.02215 \n", + "( 4 ): 100% 46/46 [00:01<00:00, 29.36it/s, train_loss=0.226]\n", + "Epoch: 4 train: 0.23051 val: 0.29723 eval-auc: 0.69258 eval-patk: 0.02991 \n", + "( 5 ): 100% 46/46 [00:01<00:00, 29.57it/s, train_loss=0.198]\n", + "Epoch: 5 train: 0.20115 val: 0.28018 eval-auc: 0.71729 eval-patk: 0.03539 \n", + "( 6 ): 100% 46/46 [00:01<00:00, 28.66it/s, train_loss=0.152]\n", + "Epoch: 6 train: 0.17812 val: 0.26524 eval-auc: 0.73440 eval-patk: 0.03607 \n", + "( 7 ): 100% 46/46 [00:01<00:00, 30.65it/s, train_loss=0.15]\n", + "Epoch: 7 train: 0.16726 val: 0.25652 eval-auc: 0.74640 eval-patk: 0.03813 \n", + "( 8 ): 100% 46/46 [00:01<00:00, 29.89it/s, train_loss=0.172]\n", + "Epoch: 8 train: 0.15538 val: 0.24975 eval-auc: 0.75780 eval-patk: 0.03950 \n", + "( 9 ): 100% 46/46 [00:01<00:00, 29.88it/s, train_loss=0.133]\n", + "Epoch: 9 train: 0.14574 val: 0.24520 eval-auc: 0.76651 eval-patk: 0.04498 \n", + "(10 ): 100% 46/46 [00:01<00:00, 30.02it/s, train_loss=0.14]\n", + "Epoch: 10 train: 0.13953 val: 0.22739 eval-auc: 0.77529 eval-patk: 0.04749 \n", + "(11 ): 100% 46/46 [00:01<00:00, 29.70it/s, train_loss=0.151]\n", + "Epoch: 11 train: 0.13218 val: 0.22872 eval-auc: 0.78306 eval-patk: 0.04749 \n", + "(12 ): 100% 46/46 [00:01<00:00, 29.81it/s, train_loss=0.13]\n", + "Epoch: 12 train: 0.12857 val: 0.22756 eval-auc: 0.78880 eval-patk: 0.04840 \n", + "(13 ): 100% 46/46 [00:01<00:00, 30.26it/s, train_loss=0.13]\n", + "Epoch: 13 train: 0.12364 val: 0.21565 eval-auc: 0.79382 eval-patk: 0.05114 \n", + "(14 ): 100% 46/46 [00:01<00:00, 30.80it/s, train_loss=0.0979]\n", + "Epoch: 14 train: 0.11943 val: 0.21567 eval-auc: 0.79833 eval-patk: 0.05479 \n", + "(15 ): 100% 46/46 [00:01<00:00, 30.27it/s, train_loss=0.109]\n", + "Epoch: 15 train: 0.11619 val: 0.21074 eval-auc: 0.80249 eval-patk: 0.05548 \n", + "(16 ): 100% 46/46 [00:01<00:00, 29.81it/s, train_loss=0.129]\n", + "Epoch: 16 train: 0.11254 val: 0.21105 eval-auc: 0.80617 eval-patk: 0.05890 \n", + "(17 ): 100% 46/46 [00:01<00:00, 30.27it/s, train_loss=0.111]\n", + "Epoch: 17 train: 0.10796 val: 0.20284 eval-auc: 0.80958 eval-patk: 0.05890 \n", + "(18 ): 100% 46/46 [00:01<00:00, 30.48it/s, train_loss=0.1]\n", + "Epoch: 18 train: 0.10627 val: 0.19820 eval-auc: 0.81167 eval-patk: 0.06119 \n", + "(19 ): 100% 46/46 [00:01<00:00, 29.63it/s, train_loss=0.132]\n", + "Epoch: 19 train: 0.10392 val: 0.20573 eval-auc: 0.81511 eval-patk: 0.06370 \n", + "(20 ): 100% 46/46 [00:01<00:00, 29.22it/s, train_loss=0.106]\n", + "Epoch: 20 train: 0.10310 val: 0.20031 eval-auc: 0.81784 eval-patk: 0.06393 \n", + "(21 ): 100% 46/46 [00:01<00:00, 29.44it/s, train_loss=0.084]\n", + "Epoch: 21 train: 0.10323 val: 0.19672 eval-auc: 0.82062 eval-patk: 0.06530 \n", + "(22 ): 100% 46/46 [00:01<00:00, 28.61it/s, train_loss=0.123]\n", + "Epoch: 22 train: 0.10163 val: 0.19164 eval-auc: 0.82266 eval-patk: 0.06986 \n", + "(23 ): 100% 46/46 [00:01<00:00, 29.98it/s, train_loss=0.109]\n", + "Epoch: 23 train: 0.09932 val: 0.18622 eval-auc: 0.82489 eval-patk: 0.06849 \n", + "(24 ): 100% 46/46 [00:01<00:00, 30.33it/s, train_loss=0.125]\n", + "Epoch: 24 train: 0.09856 val: 0.18985 eval-auc: 0.82689 eval-patk: 0.06941 \n", + "(25 ): 100% 46/46 [00:01<00:00, 30.46it/s, train_loss=0.0867]\n", + "Epoch: 25 train: 0.09591 val: 0.18680 eval-auc: 0.82851 eval-patk: 0.07100 \n", + "(26 ): 100% 46/46 [00:01<00:00, 29.23it/s, train_loss=0.0945]\n", + "Epoch: 26 train: 0.09670 val: 0.18181 eval-auc: 0.83038 eval-patk: 0.07009 \n", + "(27 ): 100% 46/46 [00:01<00:00, 29.79it/s, train_loss=0.0699]\n", + "Epoch: 27 train: 0.09253 val: 0.18122 eval-auc: 0.83169 eval-patk: 0.06667 \n", + "(28 ): 100% 46/46 [00:01<00:00, 30.00it/s, train_loss=0.0759]\n", + "Epoch: 28 train: 0.09226 val: 0.18196 eval-auc: 0.83282 eval-patk: 0.06826 \n", + "(29 ): 100% 46/46 [00:01<00:00, 29.22it/s, train_loss=0.0822]\n", + "Epoch: 29 train: 0.09307 val: 0.18249 eval-auc: 0.83441 eval-patk: 0.07648 \n", + "(30 ): 100% 46/46 [00:01<00:00, 30.18it/s, train_loss=0.114]\n", + "Epoch: 30 train: 0.09162 val: 0.18411 eval-auc: 0.83504 eval-patk: 0.07648 \n", + "(31 ): 100% 46/46 [00:01<00:00, 29.39it/s, train_loss=0.086]\n", + "Epoch: 31 train: 0.08987 val: 0.17815 eval-auc: 0.83631 eval-patk: 0.07374 \n", + "(32 ): 100% 46/46 [00:01<00:00, 29.27it/s, train_loss=0.0911]\n", + "Epoch: 32 train: 0.08841 val: 0.18399 eval-auc: 0.83683 eval-patk: 0.07306 \n", + "(33 ): 100% 46/46 [00:01<00:00, 29.72it/s, train_loss=0.0876]\n", + "Epoch: 33 train: 0.09061 val: 0.17719 eval-auc: 0.83845 eval-patk: 0.07489 \n", + "(34 ): 100% 46/46 [00:01<00:00, 29.93it/s, train_loss=0.0647]\n", + "Epoch: 34 train: 0.08688 val: 0.18095 eval-auc: 0.83955 eval-patk: 0.06918 \n", + "(35 ): 100% 46/46 [00:01<00:00, 30.05it/s, train_loss=0.0928]\n", + "Epoch: 35 train: 0.08915 val: 0.17626 eval-auc: 0.84050 eval-patk: 0.07215 \n", + "(36 ): 100% 46/46 [00:01<00:00, 29.89it/s, train_loss=0.111]\n", + "Epoch: 36 train: 0.08683 val: 0.17530 eval-auc: 0.84146 eval-patk: 0.07420 \n", + "(37 ): 100% 46/46 [00:01<00:00, 29.70it/s, train_loss=0.0915]\n", + "Epoch: 37 train: 0.08663 val: 0.16717 eval-auc: 0.84286 eval-patk: 0.07215 \n", + "(38 ): 100% 46/46 [00:01<00:00, 29.93it/s, train_loss=0.0765]\n", + "Epoch: 38 train: 0.08452 val: 0.16749 eval-auc: 0.84417 eval-patk: 0.07763 \n", + "(39 ): 100% 46/46 [00:01<00:00, 30.19it/s, train_loss=0.0737]\n", + "Epoch: 39 train: 0.08514 val: 0.16763 eval-auc: 0.84427 eval-patk: 0.07443 \n", + "(40 ): 100% 46/46 [00:01<00:00, 29.84it/s, train_loss=0.0934]\n", + "Epoch: 40 train: 0.08454 val: 0.16994 eval-auc: 0.84553 eval-patk: 0.07283 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Y2yIPrmu98bL" + }, + "source": [ + "## Hybrid Model with PyTorch on ML-100k" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kImzuHXJ9_kV" + }, + "source": [ + "Testing out the features of Collie Recs library on MovieLens-100K. Training Factorization and Hybrid models with Pytorch Lightning." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "P43fS4H27gCt" + }, + "source": [ + "!pip install -q collie_recs\n", + "!pip install -q git+https://github.com/sparsh-ai/recochef.git" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "PD0n8kefAb67" + }, + "source": [ + "import os\n", + "import joblib\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from collie_recs.interactions import Interactions\n", + "from collie_recs.interactions import ApproximateNegativeSamplingInteractionsDataLoader\n", + "from collie_recs.cross_validation import stratified_split\n", + "from collie_recs.metrics import auc, evaluate_in_batches, mapk, mrr\n", + "from collie_recs.model import CollieTrainer, MatrixFactorizationModel, HybridPretrainedModel\n", + "from collie_recs.movielens import get_recommendation_visualizations\n", + "\n", + "import torch\n", + "from pytorch_lightning.utilities.seed import seed_everything\n", + "\n", + "from recochef.datasets.movielens import MovieLens\n", + "from recochef.preprocessing.encode import label_encode as le\n", + "\n", + "from IPython.display import HTML" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ClWobbREN4VU", + "outputId": "3f4c1125-a726-444e-d6f5-bc231df2a167" + }, + "source": [ + "# this handy PyTorch Lightning function fixes random seeds across all the libraries used here\n", + "seed_everything(22)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Global seed set to 22\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "22" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T0N-kmIrcDZr" + }, + "source": [ + "### Data Loading" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LEuHGsYgGv-e" + }, + "source": [ + "data_object = MovieLens()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "g3lLxQE1G120", + "outputId": "2c6b310f-b237-4f1f-a69e-bee2ac458172" + }, + "source": [ + "df = data_object.load_interactions()\n", + "df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "0 196 242 3.0 881250949\n", + "1 186 302 3.0 891717742\n", + "2 22 377 1.0 878887116\n", + "3 244 51 2.0 880606923\n", + "4 166 346 1.0 886397596" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3X8-bE9YcGBg" + }, + "source": [ + "### Preprocessing" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "tJvifZdrLsGK" + }, + "source": [ + "# drop duplicate user-item pair records, keeping recent ratings only\n", + "df.drop_duplicates(subset=['USERID','ITEMID'], keep='last', inplace=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "eq7GY0lEINgA" + }, + "source": [ + "# convert the explicit data to implicit by only keeping interactions with a rating ``>= 4``\n", + "df = df[df.RATING>=4].reset_index(drop=True)\n", + "df['RATING'] = 1" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Jo4FnRzSHPzs" + }, + "source": [ + "# label encode\n", + "df, umap = le(df, col='USERID')\n", + "df, imap = le(df, col='ITEMID')\n", + "\n", + "df = df.astype('int64')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "LA9BtiDrJ_wf", + "outputId": "9044fa8b-53c4-421f-cbfa-8ce6b374d666" + }, + "source": [ + "df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
0001884182806
1111891628467
2221879781125
3331876042340
4441879270459
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "0 0 0 1 884182806\n", + "1 1 1 1 891628467\n", + "2 2 2 1 879781125\n", + "3 3 3 1 876042340\n", + "4 4 4 1 879270459" + ] + }, + "metadata": {}, + "execution_count": 16 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "W3hUWib-MyjB", + "outputId": "148d3ca2-17fb-4622-db52-122f045d1563" + }, + "source": [ + "user_counts = df.groupby(by='USERID')['ITEMID'].count()\n", + "user_list = user_counts[user_counts>=3].index.tolist()\n", + "df = df[df.USERID.isin(user_list)]\n", + "\n", + "df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
0001884182806
1111891628467
2221879781125
3331876042340
4441879270459
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "0 0 0 1 884182806\n", + "1 1 1 1 891628467\n", + "2 2 2 1 879781125\n", + "3 3 3 1 876042340\n", + "4 4 4 1 879270459" + ] + }, + "metadata": {}, + "execution_count": 17 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "37Y7qEEJNiE4" + }, + "source": [ + "### Interactions\n", + "While we have chosen to represent the data as a ``pandas.DataFrame`` for easy viewing now, Collie uses a custom ``torch.utils.data.Dataset`` called ``Interactions``. This class stores a sparse representation of the data and offers some handy benefits, including: \n", + "\n", + "* The ability to index the data with a ``__getitem__`` method \n", + "* The ability to sample many negative items (we will get to this later!) \n", + "* Nice quality checks to ensure data is free of errors before model training \n", + "\n", + "Instantiating the object is simple! " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dLcqt57TI-6l", + "outputId": "b399e7e8-fec0-460a-8048-e7a1e9395106" + }, + "source": [ + "interactions = Interactions(\n", + " users=df['USERID'],\n", + " items=df['ITEMID'],\n", + " ratings=df['RATING'],\n", + " allow_missing_ids=True,\n", + ")\n", + "\n", + "interactions" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Checking for and removing duplicate user, item ID pairs...\n", + "Checking ``num_negative_samples`` is valid...\n", + "Maximum number of items a user has interacted with: 378\n", + "Generating positive items set...\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Interactions object with 55375 interactions between 942 users and 1447 items, returning 10 negative samples per interaction." + ] + }, + "metadata": {}, + "execution_count": 18 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4TaWM_OFNZzn" + }, + "source": [ + "### Data Splits \n", + "With an ``Interactions`` dataset, Collie supports two types of data splits. \n", + "\n", + "1. **Random split**: This code randomly assigns an interaction to a ``train``, ``validation``, or ``test`` dataset. While this is significantly faster to perform than a stratified split, it does not guarantee any balance, meaning a scenario where a user will have no interactions in the ``train`` dataset and all in the ``test`` dataset is possible. \n", + "2. **Stratified split**: While this code runs slower than a random split, this guarantees that each user will be represented in the ``train``, ``validation``, and ``test`` dataset. This is by far the most fair way to train and evaluate a recommendation model. \n", + "\n", + "Since this is a small dataset and we have time, we will go ahead and use ``stratified_split``. If you're short on time, a ``random_split`` can easily be swapped in, since both functions share the same API! " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "U_4IPy2aNLVE", + "outputId": "5db1d12d-5d17-46f5-c4d1-05cfe4d458e4" + }, + "source": [ + "train_interactions, val_interactions = stratified_split(interactions, test_p=0.1, seed=42)\n", + "train_interactions, val_interactions" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Generating positive items set...\n", + "Generating positive items set...\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(Interactions object with 49426 interactions between 942 users and 1447 items, returning 10 negative samples per interaction.,\n", + " Interactions object with 5949 interactions between 942 users and 1447 items, returning 10 negative samples per interaction.)" + ] + }, + "metadata": {}, + "execution_count": 19 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZO5rLYzH-imu" + }, + "source": [ + "### Model Architecture \n", + "With our data ready-to-go, we can now start training a recommendation model. While Collie has several model architectures built-in, the simplest by far is the ``MatrixFactorizationModel``, which use ``torch.nn.Embedding`` layers and a dot-product operation to perform matrix factorization via collaborative filtering." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZWi4VUjeghUA" + }, + "source": [ + "Digging through the code of [``collie_recs.model.MatrixFactorizationModel``](../collie_recs/model.py) shows the architecture is as simple as we might think. For simplicity, we will include relevant portions below so we know exactly what we are building: \n", + "\n", + "````python\n", + "def _setup_model(self, **kwargs) -> None:\n", + " self.user_biases = ZeroEmbedding(num_embeddings=self.hparams.num_users,\n", + " embedding_dim=1,\n", + " sparse=self.hparams.sparse)\n", + " self.item_biases = ZeroEmbedding(num_embeddings=self.hparams.num_items,\n", + " embedding_dim=1,\n", + " sparse=self.hparams.sparse)\n", + " self.user_embeddings = ScaledEmbedding(num_embeddings=self.hparams.num_users,\n", + " embedding_dim=self.hparams.embedding_dim,\n", + " sparse=self.hparams.sparse)\n", + " self.item_embeddings = ScaledEmbedding(num_embeddings=self.hparams.num_items,\n", + " embedding_dim=self.hparams.embedding_dim,\n", + " sparse=self.hparams.sparse)\n", + "\n", + " \n", + "def forward(self, users: torch.tensor, items: torch.tensor) -> torch.tensor:\n", + " user_embeddings = self.user_embeddings(users)\n", + " item_embeddings = self.item_embeddings(items)\n", + "\n", + " preds = (\n", + " torch.mul(user_embeddings, item_embeddings).sum(axis=1)\n", + " + self.user_biases(users).squeeze(1)\n", + " + self.item_biases(items).squeeze(1)\n", + " )\n", + "\n", + " if self.hparams.y_range is not None:\n", + " preds = (\n", + " torch.sigmoid(preds)\n", + " * (self.hparams.y_range[1] - self.hparams.y_range[0])\n", + " + self.hparams.y_range[0]\n", + " )\n", + "\n", + " return preds\n", + "````\n", + "\n", + "Let's go ahead and instantiate the model and start training! Note that even if you are running this model on a CPU instead of a GPU, this will still be relatively quick to fully train. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L7o3t9vNN-Lt" + }, + "source": [ + "Collie is built with PyTorch Lightning, so all the model classes and the ``CollieTrainer`` class accept all the training options available in PyTorch Lightning. Here, we're going to set the embedding dimension and learning rate differently, and go with the defaults for everything else" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LNfxzlruN1xx" + }, + "source": [ + "model = MatrixFactorizationModel(\n", + " train=train_interactions,\n", + " val=val_interactions,\n", + " embedding_dim=10,\n", + " lr=1e-2,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "TKxeyMsMN1vg" + }, + "source": [ + "trainer = CollieTrainer(model, max_epochs=10, deterministic=True)\n", + "\n", + "trainer.fit(model)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dx8EoEhC_Cjh" + }, + "source": [ + "```text\n", + "GPU available: False, used: False\n", + "TPU available: False, using: 0 TPU cores\n", + "IPU available: False, using: 0 IPUs\n", + "\n", + " | Name | Type | Params\n", + "----------------------------------------------------\n", + "0 | user_biases | ZeroEmbedding | 942 \n", + "1 | item_biases | ZeroEmbedding | 1.4 K \n", + "2 | user_embeddings | ScaledEmbedding | 9.4 K \n", + "3 | item_embeddings | ScaledEmbedding | 14.5 K\n", + "4 | dropout | Dropout | 0 \n", + "----------------------------------------------------\n", + "26.3 K Trainable params\n", + "0 Non-trainable params\n", + "26.3 K Total params\n", + "0.105 Total estimated model params size (MB)\n", + "Validation sanity check: 0%\n", + "0/2 [00:00User 895:\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Willy Wonka and the Chocolate Factory (1971)Mighty Aphrodite (1995)Conspiracy Theory (1997)Sense and Sensibility (1995)Liar Liar (1997)In & Out (1997)Return of the Jedi (1983)Ransom (1996)Emma (1996)Toy Story (1995)
Some loved films:
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Princess Bride, The (1987)Graduate, The (1967)Cold Comfort Farm (1995)Apartment, The (1960)Jerry Maguire (1996)Sleeper (1973)Independence Day (ID4) (1996)Desperado (1995)Three Colors: Red (1994)Lawnmower Man, The (1992)
Recommended films:
-----

User 895 has rated 12 films with a 4 or 5

User 895 has rated 8 films with a 1, 2, or 3

% of these films rated 5 or 4 appearing in the first 10 recommendations:0.0%

% of these films rated 1, 2, or 3 appearing in the first 10 recommendations: 0.0%

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DbZ9ufGvghUE" + }, + "source": [ + "### Save and Load a Standard Model " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WqJbXHXgghUG" + }, + "source": [ + "# we can save the model with...\n", + "os.makedirs('models', exist_ok=True)\n", + "model.save_model('models/matrix_factorization_model.pth')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Dz_8miLPghUG", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "1f2f8c07-be29-4fb9-ca8c-168ac1515f16" + }, + "source": [ + "# ... and if we wanted to load that model back in, we can do that easily...\n", + "model_loaded_in = MatrixFactorizationModel(load_model_path='models/matrix_factorization_model.pth')\n", + "\n", + "model_loaded_in" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "MatrixFactorizationModel(\n", + " (user_biases): ZeroEmbedding(942, 1)\n", + " (item_biases): ZeroEmbedding(1447, 1)\n", + " (user_embeddings): ScaledEmbedding(942, 10)\n", + " (item_embeddings): ScaledEmbedding(1447, 10)\n", + " (dropout): Dropout(p=0.0, inplace=False)\n", + ")" + ] + }, + "metadata": {}, + "execution_count": 25 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cQpWlCuughUH" + }, + "source": [ + "Now that we've built our first model and gotten some baseline metrics, we now will be looking at some more advanced features in Collie's ``MatrixFactorizationModel``. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Q3Ne8ETsgzLe" + }, + "source": [ + "### Faster Data Loading Through Approximate Negative Sampling " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8nBu6PZhgzLe" + }, + "source": [ + "With sufficiently large enough data, verifying that each negative sample is one a user has *not* interacted with becomes expensive. With many items, this can soon become a bottleneck in the training process. \n", + "\n", + "Yet, when we have many items, the chances a user has interacted with most is increasingly rare. Say we have ``1,000,000`` items and we want to sample ``10`` negative items for a user that has positively interacted with ``200`` items. The chance that we accidentally select a positive item in a random sample of ``10`` items is just ``0.2%``. At that point, it might be worth it to forgo the expensive check to assert our negative sample is true, and instead just randomly sample negative items with the hope that most of the time, they will happen to be negative. \n", + "\n", + "This is the theory behind the ``ApproximateNegativeSamplingInteractionsDataLoader``, an alternate DataLoader built into Collie. Let's train a model with this below, noting how similar this procedure looks to that in the previous tutorial. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "MgSijz04gzLf" + }, + "source": [ + "train_loader = ApproximateNegativeSamplingInteractionsDataLoader(train_interactions, batch_size=1024, shuffle=True)\n", + "val_loader = ApproximateNegativeSamplingInteractionsDataLoader(val_interactions, batch_size=1024, shuffle=False)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "n9wQLd9-gzLf" + }, + "source": [ + "model = MatrixFactorizationModel(\n", + " train=train_loader,\n", + " val=val_loader,\n", + " embedding_dim=10,\n", + " lr=1e-2,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "AbYirCSNgzLg" + }, + "source": [ + "trainer = CollieTrainer(model, max_epochs=10, deterministic=True)\n", + "\n", + "trainer.fit(model)\n", + "model.eval()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uvykQiW2_Oof" + }, + "source": [ + "```text\n", + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "----------------------------------------------------\n", + "0 | user_biases | ZeroEmbedding | 941 \n", + "1 | item_biases | ZeroEmbedding | 1.4 K \n", + "2 | user_embeddings | ScaledEmbedding | 9.4 K \n", + "3 | item_embeddings | ScaledEmbedding | 14.5 K\n", + "4 | dropout | Dropout | 0 \n", + "----------------------------------------------------\n", + "26.3 K Trainable params\n", + "0 Non-trainable params\n", + "26.3 K Total params\n", + "0.105 Total estimated model params size (MB)\n", + "Detected GPU. Setting ``gpus`` to 1.\n", + "Global seed set to 22\n", + "\n", + "MatrixFactorizationModel(\n", + " (user_biases): ZeroEmbedding(941, 1)\n", + " (item_biases): ZeroEmbedding(1447, 1)\n", + " (user_embeddings): ScaledEmbedding(941, 10)\n", + " (item_embeddings): ScaledEmbedding(1447, 10)\n", + " (dropout): Dropout(p=0.0, inplace=False)\n", + ")\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117, + "referenced_widgets": [ + "d7c34c7be74246acb1a7da702b490029", + "8ca15bf2e3be4ba88b7dbbf4ed26820d", + "3e63ae601740455e81c35d4fbab2db6e", + "b289ad3cdd3f4a25a9e92ed22c37aeed", + "3d7d55e46c7d4e2caa6680229fe73b4e", + "e841d95562314124b242c2e4225afbdb", + "7b9d13b53df04e2082d4e236f50b807d", + "9a4137c369cc4f26817ed3ab568a5ef7" + ] + }, + "id": "xLKDSq-hgzLg", + "outputId": "44183d74-a567-418d-a85e-946bf1443713" + }, + "source": [ + "mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n", + "\n", + "print(f'MAP@10 Score: {mapk_score}')\n", + "print(f'MRR Score: {mrr_score}')\n", + "print(f'AUC Score: {auc_score}')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d7c34c7be74246acb1a7da702b490029", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "MAP@10 Score: 0.027979833367276323\n", + "MRR Score: 0.1703751336709069\n", + "AUC Score: 0.8517987786322347\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qipOCQzxgzLh" + }, + "source": [ + "We're seeing a small hit on performance and only a marginal improvement in training time compared to the standard ``MatrixFactorizationModel`` model because MovieLens 100K has so few items. ``ApproximateNegativeSamplingInteractionsDataLoader`` is especially recommended for when we have more items in our data and training times need to be optimized. \n", + "\n", + "For more details on this and other DataLoaders in Collie (including those for out-of-memory datasets), check out the [docs](https://collie.readthedocs.io/en/latest/index.html)! " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iGgM-FaegzLi" + }, + "source": [ + "### Multiple Optimizers " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4xcFspsbgzLi" + }, + "source": [ + "Training recommendation models at ShopRunner, we have encountered something we call \"the curse of popularity.\" \n", + "\n", + "This is best thought of in the viewpoint of a model optimizer - say we have a user, a positive item, and several negative items that we hope have recommendation scores that score lower than the positive item. As an optimizer, you can either optimize every single embedding dimension (hundreds of parameters) to achieve this, or instead choose to score a quick win by optimizing the bias terms for the items (just add a positive constant to the positive item and a negative constant to each negative item). \n", + "\n", + "While we clearly want to have varied embedding layers that reflect each user and item's taste profiles, some models learn to settle for popularity as a recommendation score proxy by over-optimizing the bias terms, essentially just returning the same set of recommendations for every user. Worst of all, since popular items are... well, popular, **the loss of this model will actually be decent, solidifying the model getting stuck in a local loss minima**. \n", + "\n", + "To counteract this, Collie supports multiple optimizers in a ``MatrixFactorizationModel``. With this, we can have a faster optimizer work to optimize the embedding layers for users and items, and a slower optimizer work to optimize the bias terms. With this, we impel the model to do the work actually coming up with varied, personalized recommendations for users while still taking into account the necessity of the bias (popularity) terms on recommendations. \n", + "\n", + "At ShopRunner, we have seen significantly better metrics and results from this type of model. With Collie, this is simple to do, as shown below. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GxUMsr61gzLj" + }, + "source": [ + "model = MatrixFactorizationModel(\n", + " train=train_interactions,\n", + " val=val_interactions,\n", + " embedding_dim=10,\n", + " lr=1e-2,\n", + " bias_lr=1e-1,\n", + " optimizer='adam',\n", + " bias_optimizer='sgd',\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 557, + "referenced_widgets": [ + "49f97e2e5f55454bab313b1570b4b9c4", + "b3c7b293fe8b44e0920b060f00382401", + "80b4510b6a7e4d77879516c59662180c", + "f1e818c6ef934704ae0ef1f80eaa7b5b", + "695c6402e0a1475eb2965bee589c2bf4", + "06f180111dfe4f57a12723bccb253f98", + "316eaa17f9cd4558b8c7f6951b98d6f0", + "75fc6c2f339b4e47bb22790d37e7ebab", + "8d714e45f0d64e85b637961f4a25f3d0", + "b5a4c2dd499c4d5c8d3ab2e890b44642", + "d1dd8920ddfc4c97beffe89ece6b982e", + "fc7d1f4bea4145af93d9059177a477fd", + "ef66bb3e20eb4d25a6dc3af0fccf2e2a", + "9275275798974477a5f5858880d1b429", + "2bcffbf7f05b41688936d51d6c3ffcac", + "28198fbab991465583818c47d8ddbb6d", + "83658275a6c346d7822fd1368345e09a", + "3389371180f64e088e7c6549a25b3ee1", + "54d47349fff54cd0bca2e4903b06e5d8", + "ec37483292aa4715a97c6f0de133cb24", + "78845118227b40adac2503e57e1f38a7", + "ce20fc80baf0494f976d7ebb54303833", + "b0606438031a4d0fbe1b3b1bc4d49e9e", + "38cb1183eb294363af2d2762d8a49a13", + "b9a2d6afed7641d995fba91c5a94f8ec", + "6f42c45ff32f46deb83569801ab976c8", + "58d6593d8810421bb8bd39099e82feb9", + "7cdc87e9c27f48e2b081bdbf2d6228e8", + "45b86b0dc3394da4b2f87191d23dfed7", + "585898b88c044e2e8d6fd0c46c3b575d", + "a0259f8bb2264993a6d27acd097b6c05", + "5b0ed212d376472987768ea06e314f9d", + "a59ea6641fca44bbaa553a1624deab51", + "671b80df47dc444d817c51101c7adf49", + "d56ba50fc09c4502a1055ff9a3199062", + "1bd1fa4ccb564778be34c5fe9036e731", + "1b3ff709422c4864acff0ca42ec89e07", + "5b6f8c0d04be4ec19c55259caf622c02", + "b8903c665b7c4a3695101aa7d6e8f929", + "518b4b044b4d411fb43dac5fc993c02c", + "36d600f8e9be46a6b05ec580025aac20", + "f14194bdbf724d20bdd7d5b572374608", + "b62d3801dd3d4f45b53de6d6c25e26bd", + "066ce24dcd2e46acb54f1fd437963b85", + "bd80f8f37cb247c69f969d96b146ea8c", + "4100968fe609430f8cd1f73c48a356f2", + "9e6c6c48df824c548aa0425888adac9b", + "27866404913f4820bb46d1641f63dc1f", + "d75b9faa61b242279706a77458e55276", + "ba792412f171494d937051219e01607d", + "f43bba18ba134b269006e100f8de55d1", + "63c6fb5bfa3147a89962641d1a7d215b", + "6d7a19d43c4846d99d3d470fd5d7d66e", + "90af7f79f905435f8c81ca21e59cab59", + "fad0632c95f74aca92b6e72751da0632", + "7eb49fb403564170b47e0c6f867f81e1", + "57322a343ddd4ad2bc408a840eca0e98", + "18b3142ac5c24401941cd4f79bac783b", + "6d730313b939447bbd396526cdee3fad", + "73cccc23b004406e8521b0cc26401c9c", + "7520241c3b2c4591bc0532d20dafccda", + "5dd8ebbab9ce418cae4eaf9f568ab5c9", + "0afd2fe607b04590813dc297c3bfc4da", + "c14405b6297a435eaab8032dbbb9966f", + "0351e40f8c0f46708aeea7233565800a", + "d7e152362cd54b4380a6cd3e506e7b86", + "73dff796a59d444a96016b2578f277d7", + "89a5fbcb3a4e4732a72c8cee79a346b6", + "bcde36e5444444568f7a6fa34d85ccaa", + "53e3249bbef446208f54a6216420062f", + "235ef9871d5443a9ab3b31f1c231f53d", + "b329742fdb3e44e2974d3a31700072a3", + "93fbdf1f9df1464888cdecc6285ef575", + "7ef0973e658740528843af596067fe8e", + "dad5c7f76e20437ca651d4d39ac68660", + "3a61c8ddd8b74af78c6ceffb7001da02", + "fec250de7d6f45688eb48b51d837b53a", + "54d415a20e204d53bbe9baddd4598e2f", + "136520e97ebe4b04b9b46a44e46297fc", + "c4d1f1810116465b9dcb9282eed0f113", + "2c8df37c614447d5b5064dbbaa837081", + "dc9b62bd3eb94dc8afd9c21491faff0b", + "2416bc3ad43149ff9e8d10572693f115", + "93054dc57a344b9daae9e6e36e14b594", + "4d9cf1e48cb74afb91410b46b2d601b5", + "897727bf519648e0bc3f204771ded957", + "795429863893411cab25dfe5771a2e4a", + "93a527dd2045487393fb8280ff3945b1", + "3d3ca53ac0cb41d1bbb58aa754a79619", + "600e766c4f4c4136875db902954f9ac7", + "2cf5bd88c0c74729a89a70f6ba9dd02e", + "a247748059184bbaac041b8fffd99e3f", + "010c2bf6e2af470d950542394f8ceef5", + "5bb0a358fcd64106acdac950f82d12a7", + "16202361258644f6ab7b168990f73136", + "c2fffba8678b48f9957b9a581aae9e2e" + ] + }, + "id": "dQ_tTRfOgzLj", + "outputId": "4d3af437-c62d-4b9b-ddb5-eaf3731556da" + }, + "source": [ + "trainer = CollieTrainer(model, max_epochs=10, deterministic=True)\n", + "\n", + "trainer.fit(model)\n", + "model.eval()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "----------------------------------------------------\n", + "0 | user_biases | ZeroEmbedding | 941 \n", + "1 | item_biases | ZeroEmbedding | 1.4 K \n", + "2 | user_embeddings | ScaledEmbedding | 9.4 K \n", + "3 | item_embeddings | ScaledEmbedding | 14.5 K\n", + "4 | dropout | Dropout | 0 \n", + "----------------------------------------------------\n", + "26.3 K Trainable params\n", + "0 Non-trainable params\n", + "26.3 K Total params\n", + "0.105 Total estimated model params size (MB)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Detected GPU. Setting ``gpus`` to 1.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "49f97e2e5f55454bab313b1570b4b9c4", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Global seed set to 22\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\r" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8d714e45f0d64e85b637961f4a25f3d0", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "83658275a6c346d7822fd1368345e09a", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b9a2d6afed7641d995fba91c5a94f8ec", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a59ea6641fca44bbaa553a1624deab51", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n", + "Epoch 3: reducing learning rate of group 0 to 1.0000e-02.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "36d600f8e9be46a6b05ec580025aac20", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d75b9faa61b242279706a77458e55276", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "57322a343ddd4ad2bc408a840eca0e98", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0351e40f8c0f46708aeea7233565800a", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "93fbdf1f9df1464888cdecc6285ef575", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2c8df37c614447d5b5064dbbaa837081", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3d3ca53ac0cb41d1bbb58aa754a79619", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "MatrixFactorizationModel(\n", + " (user_biases): ZeroEmbedding(941, 1)\n", + " (item_biases): ZeroEmbedding(1447, 1)\n", + " (user_embeddings): ScaledEmbedding(941, 10)\n", + " (item_embeddings): ScaledEmbedding(1447, 10)\n", + " (dropout): Dropout(p=0.0, inplace=False)\n", + ")" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 55 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117, + "referenced_widgets": [ + "5c2218b13d6345ff91c4d8980b2b2ca0", + "309d24f05b1b4bc5b7d0148795fd5cf4", + "ccfb5dc3eef14f1f8324bc48b3dc0e7f", + "a84e5372c5c24417a08487dd3efcebe8", + "c28c07424b7f45088e73ac25f2bb712a", + "5d01bda6f6354f25bb6dcb15ef4596a0", + "758cba83e9414f60bf87799a71b3cd7f", + "48d71d4847be4a25a237568371ae4493" + ] + }, + "id": "ENZSOd1DgzLk", + "outputId": "7b1844fb-b81a-43cb-8a98-5206b87b4506" + }, + "source": [ + "mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n", + "\n", + "print(f'MAP@10 Score: {mapk_score}')\n", + "print(f'MRR Score: {mrr_score}')\n", + "print(f'AUC Score: {auc_score}')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5c2218b13d6345ff91c4d8980b2b2ca0", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "MAP@10 Score: 0.03243186201880122\n", + "MRR Score: 0.19819369246580287\n", + "AUC Score: 0.8617710409716284\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "blXcVpg3gzLk" + }, + "source": [ + "Again, we're not seeing as much performance increase here compared to the standard model because MovieLens 100K has so few items. For a more dramatic difference, try training this model on a larger dataset, such as MovieLens 10M, adjusting the architecture-specific hyperparameters, or train longer. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1Dt0IsWJgzLk" + }, + "source": [ + "### Item-Item Similarity " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sqQJpyKVgzLl" + }, + "source": [ + "While we've trained every model thus far to work for member-item recommendations (given a *member*, recommend *items* - think of this best as \"Personalized recommendations for you\"), we also have access to item-item recommendations for free (given a seed *item*, recommend similar *items* - think of this more like \"People who interacted with this item also interacted with...\"). \n", + "\n", + "With Collie, accessing this is simple! " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 343 + }, + "id": "RRMouFweUOhw", + "outputId": "02e6281a-f64e-43c4-a151-84601d91ab50" + }, + "source": [ + "df_item = data_object.load_items()\n", + "df_item.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
01Toy Story (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Toy%20Story%2...0001110000000000000
12GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
23Four Rooms (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Four%20Rooms%...0000000000000000100
34Get Shorty (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Get%20Shorty%...0100010010000000000
45Copycat (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Copycat%20(1995)0000001010000000100
\n", + "
" + ], + "text/plain": [ + " ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n", + "0 1 Toy Story (1995) 01-Jan-1995 ... 0 0 0\n", + "1 2 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n", + "2 3 Four Rooms (1995) 01-Jan-1995 ... 1 0 0\n", + "3 4 Get Shorty (1995) 01-Jan-1995 ... 0 0 0\n", + "4 5 Copycat (1995) 01-Jan-1995 ... 1 0 0\n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 68 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Ycl8PcLIgzLl", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 343 + }, + "outputId": "27f2b6aa-9b6f-4fe4-9d8f-3637cadd4bbe" + }, + "source": [ + "df_item = le(df_item, col='ITEMID', maps=imap)\n", + "df_item.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
09.0Toy Story (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Toy%20Story%2...0001110000000000000
1160.0GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
2579.0Four Rooms (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Four%20Rooms%...0000000000000000100
325.0Get Shorty (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Get%20Shorty%...0100010010000000000
4436.0Copycat (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?Copycat%20(1995)0000001010000000100
\n", + "
" + ], + "text/plain": [ + " ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n", + "0 9.0 Toy Story (1995) 01-Jan-1995 ... 0 0 0\n", + "1 160.0 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n", + "2 579.0 Four Rooms (1995) 01-Jan-1995 ... 1 0 0\n", + "3 25.0 Get Shorty (1995) 01-Jan-1995 ... 0 0 0\n", + "4 436.0 Copycat (1995) 01-Jan-1995 ... 1 0 0\n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 69 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "TvtbOrbtgzLl", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117 + }, + "outputId": "67ef757a-91e9-4b32-c2ac-5f43c4742634" + }, + "source": [ + "df_item.loc[df_item['TITLE'] == 'GoldenEye (1995)']" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
1160.0GoldenEye (1995)01-Jan-1995NaNhttp://us.imdb.com/M/title-exact?GoldenEye%20(...0110000000000000100
\n", + "
" + ], + "text/plain": [ + " ITEMID TITLE RELEASE ... THRILLER WAR WESTERN\n", + "1 160.0 GoldenEye (1995) 01-Jan-1995 ... 1 0 0\n", + "\n", + "[1 rows x 24 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 70 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Uom6EVG8gzLm", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "a4f6c8a4-c472-48c2-f240-14763b5d1d8e" + }, + "source": [ + "# let's start by finding movies similar to GoldenEye (1995)\n", + "item_similarities = model.item_item_similarity(item_id=160)\n", + "\n", + "item_similarities" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "160 1.000000\n", + "123 0.842003\n", + "948 0.828162\n", + "398 0.827030\n", + "197 0.826931\n", + " ... \n", + "26 -0.654127\n", + "88 -0.680429\n", + "165 -0.697536\n", + "499 -0.729313\n", + "312 -0.780792\n", + "Length: 1447, dtype: float64" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 71 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "MW741iOcgzLm", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 428 + }, + "outputId": "ad4b1617-bc85-4698-ebf6-65b70161e7ce" + }, + "source": [ + "df_item.iloc[item_similarities.index][:5]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ITEMIDTITLERELEASEVIDRELEASEURLUNKNOWNACTIONADVENTUREANIMATIONCHILDRENCOMEDYCRIMEDOCUMENTARYDRAMAFANTASYFILMNOIRHORRORMUSICALMYSTERYROMANCESCIFITHRILLERWARWESTERN
162182.0Return of the Pink Panther, The (1974)01-Jan-1974NaNhttp://us.imdb.com/M/title-exact?Return%20of%2...0000010000000000000
125229.0Spitfire Grill, The (1996)06-Sep-1996NaNhttp://us.imdb.com/M/title-exact?Spitfire%20Gr...0000000010000000000
975933.0Solo (1996)23-Aug-1996NaNhttp://us.imdb.com/M/title-exact?Solo%20(1996)0100000000000001100
401175.0Ghost (1990)01-Jan-1990NaNhttp://us.imdb.com/M/title-exact?Ghost%20(1990)0000010000000010100
19972.0Shining, The (1980)01-Jan-1980NaNhttp://us.imdb.com/M/title-exact?Shining,%20Th...0000000000010000000
\n", + "
" + ], + "text/plain": [ + " ITEMID TITLE ... WAR WESTERN\n", + "162 182.0 Return of the Pink Panther, The (1974) ... 0 0\n", + "125 229.0 Spitfire Grill, The (1996) ... 0 0\n", + "975 933.0 Solo (1996) ... 0 0\n", + "401 175.0 Ghost (1990) ... 0 0\n", + "199 72.0 Shining, The (1980) ... 0 0\n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 72 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_py39oQ8gzLm" + }, + "source": [ + "Unfortunately, not seen these movies. Can't say if these are relevant.\n", + "\n", + "``item_item_similarity`` method is available in all Collie models, not just ``MatrixFactorizationModel``! \n", + "\n", + "Next, we will incorporate item metadata into recommendations for even better results." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8hkoWyfVg9AK" + }, + "source": [ + "### Partial Credit Loss\n", + "Most of the time, we don't *only* have user-item interactions, but also side-data about our items that we are recommending. These next two notebooks will focus on incorporating this into the model training process. \n", + "\n", + "In this notebook, we're going to add a new component to our loss function - \"partial credit\". Specifically, we're going to use the genre information to give our model \"partial credit\" for predicting that a user would like a movie that they haven't interacted with, but is in the same genre as one that they liked. The goal is to help our model learn faster from these similarities. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4iFhjr7eg9AK" + }, + "source": [ + "### Read in Data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bK4bGSUEWe9F" + }, + "source": [ + "To do the partial credit calculation, we need this data in a slightly different form. Instead of the one-hot-encoded version above, we're going to make a ``1 x n_items`` tensor with a number representing the first genre associated with the film, for simplicity. Note that with Collie, we could instead make a metadata tensor for each genre" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "mi7IZRHbWwsc", + "outputId": "20906809-3682-4bee-eee3-1d98fe9b03ca" + }, + "source": [ + "df_item.columns[5:]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Index(['UNKNOWN', 'ACTION', 'ADVENTURE', 'ANIMATION', 'CHILDREN', 'COMEDY',\n", + " 'CRIME', 'DOCUMENTARY', 'DRAMA', 'FANTASY', 'FILMNOIR', 'HORROR',\n", + " 'MUSICAL', 'MYSTERY', 'ROMANCE', 'SCIFI', 'THRILLER', 'WAR', 'WESTERN'],\n", + " dtype='object')" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 76 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "bWBxAUUXYvZ3" + }, + "source": [ + "metadata_df = df_item[df_item.columns[5:]]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "LSy_-Jsxg9AL", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "a1d9aa32-d10c-454a-dc7b-9cfcda410071" + }, + "source": [ + "genres = (\n", + " torch.tensor(metadata_df.values)\n", + " .topk(1)\n", + " .indices\n", + " .view(-1)\n", + ")\n", + "\n", + "genres" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor([ 5, 1, 16, ..., 14, 5, 8])" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 77 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LhaQLQQig9AM" + }, + "source": [ + "### Train a model with our new loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5NrYwTFVXCco" + }, + "source": [ + "now, we will pass in ``metadata_for_loss`` and ``metadata_for_loss_weights`` into the model ``metadata_for_loss`` should have a tensor containing the integer representations for metadata we created above for every item ID in our dataset ``metadata_for_loss_weights`` should have the weights for each of the keys in ``metadata_for_loss``" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Sysr04kSg9AN" + }, + "source": [ + "model = MatrixFactorizationModel(\n", + " train=train_interactions,\n", + " val=val_interactions,\n", + " embedding_dim=10,\n", + " lr=1e-2,\n", + " metadata_for_loss={'genre': genres},\n", + " metadata_for_loss_weights={'genre': 0.4},\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 472, + "referenced_widgets": [ + "7f9d29d6c1f04d4b9adcda6292a698fb", + "4d95aa15fb8e45168224c12f033db89e", + "567edd5b84854cc3ba9d8b34702716ce", + "3ccc6ebee63d473aba28adc868b5b5c0", + "5db3013f6fca47428b7f05d89d611441", + "650154b034e843648fdd31198e6b5c8b", + "b85c146244634db5a8ec5bb8c37a6a3f", + "71e7704f60a44170988aed05dc8a4788", + "71fb17475171409ab7b30f0915d3c3f9", + "ec82867f02de401faf33bdeb69a1bf2e", + "4ffc6e6cdbbb47239b7da5a07f16b5fb", + "cce5557f779742e3932bb8ef2016a17c", + "668011e6db324325974aa6a6b1fdc782", + "ebcdd1df171847a38f7655716f2d6061", + "9d11e361b37a4849a83531e249455615", + "30fefc2060004b2bb028ed54ea9383b6", + "85b96bc221984d08815cf3c09fde7c9f", + "1abf2c736a91486eb8621d1cd679f9b2", + "89b238d17aad4ef389c20de16ed1e55c", + "70d5a3c3cf41478b83813ec40298cd22", + "bdf376ea2f8e4b9e9d495a1fe44b2b59", + "c6213cfdba3d4228aad387361e062fc7", + "a28aa408fe444c179080e2ef9433a877", + "6e3313408a184dcca85da1f7514d8bfe", + "b6f3004c83574b379bc3520a410b5df5", + "57e0d47b70b744898e4e548c28d27095", + "d5f9ad23ba314382a2ffa29ecc311b5c", + "75bcb22fb4514a288fdd43cdb2084b11", + "d7bf1b026d10490fa70221af127986e5", + "bef2ed39c203462a80a1f217adfc301b", + "1038bcbdb10a4da08c7b2ffbc08c7e81", + "2a435432b9914e3faf2d49a0645a9fd5", + "127ad96af11c41519ded336bbc246c42", + "b5fe20560cc64837af9e13936fd702d3", + "2260ec6d57724ecca381bf65a151e97f", + "add9b2d1bfeb4b3baaa97d39ca4eb8e3", + "2baf39d33c2a4f28bfd557ba7b9f8c78", + "bdf0ea6c2c8042d68c50cd5e19b1c15b", + "143ecbd32aa14efeb1d2eae6873609ce", + "b4eee80b60f748b4886ede6073b37a7c", + "1997f9f7a4d04e5ebec4b8907e32b797", + "21380c3a63df4a7592b0e580535603df", + "0bd610aa3a55452dae1e3fbc721cb87a", + "07e2308d6aff45c5bd0ad77e246e9981", + "2354778e0ce6427385e2e107ea30332b", + "717568c66bc444ddb60b415c5169588c", + "13a5b3039ac04a2dbd6049992f822c89", + "96773fc1d61d4b9faf8f8e7f0d0c3786", + "2bb338780a1a40318d91fc689692c9ae", + "869cd72009244cd997fc6b8bca19cd53", + "8f341ddecce34a2e9967972e901123b6", + "4f53235cc013444c8a71141df07ccc4b", + "d76f878252a54a79bd98b1e093285964", + "69b2d3fe2f5d4f73ba34142ccea3dbf8", + "818427165e3c40d6b9c60f5fc26ea0db", + "a62b8b75b5bb4b1a9a35a98b9f87edc0", + "b066cece2abe4a74b1aed6c238cd82b3", + "5a5b267987ed4ad0a143744fe30f27b1", + "2666a159d7f1406887a6bff3efdd1dd6", + "7b3b88973bc74728a3ea7eb04a122371", + "7e684d1ba34a49e1b108909220e0a487", + "a96a161717124b54ba9473f5d4476177", + "2137633428de4deb8056902144e7551d", + "2454cc03bb5342eca1ab02501d3ac39e", + "2da2765da24841709884a7cca16f1107", + "9d9f37ef8c9844a9ad5f4022ce904af3", + "b7febc3da8714283b798aafaab3a8d8f", + "f44ea695c0b2485faff83a6bbe12407e", + "1e1888b6dc3e4c47aadf843a62f03233", + "d75a669ab0a147ddb4185d6950c8267e", + "a1eab9ad9a6147c2a3a13ed1d837f94b", + "dd40e90d90a34827bb07d9d86c4234ce", + "2391e70d0de74d71ab5372f39a4fa49b", + "05fa6eea049e4b41a40604eed4c798d0", + "e271494364c24f4e901c42b64e464056", + "69b72f3ef4ae40a0a7b060be0849a354", + "3e756777c21b4c56a06b537ca232ffc3", + "0d4e9b5e9a3241dd920146b8be70d1ce", + "d6e7bf43dc2346c0a2278d6b19078c90", + "852593a171a048bc81b0cd3466366bd0", + "13c2754aca2a42c2ac06d3640062b6d4", + "1ac7aa00d4b24eccaf9b0cb622a65b05", + "50691a81b511451aa9f78cbd0803a652", + "43a743d7e9f14787a606cea5d0176931", + "8561cb62512f47d9929c3abd557ef739", + "52ac26d1722c40b68b18e0d92c6ac994", + "cbf2e61a7dea4b91a8f86774bcfc8ff7", + "8e58b5a80dca4cca8b03635c8d529072", + "9adf92080dd2458a942dbd76a285ce48", + "9a6a243411e84f049dfaeb99cccbc562", + "5458638e910744509182e8c4ace883ec", + "e692df9e6fb949c6a345dbaf9235d195", + "7409891326ad4a259842211e306e980a", + "0f57dd28b6954df693a160125075b506", + "ae151ed064b547b6a8bca5181668f491", + "5e8575da45094823bce0d16b9fe01f09" + ] + }, + "id": "ZAk1C815g9AN", + "outputId": "0c5600e3-d81d-4f6e-d621-c7882a767fb1" + }, + "source": [ + "trainer = CollieTrainer(model=model, max_epochs=10, deterministic=True)\n", + "\n", + "trainer.fit(model)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "----------------------------------------------------\n", + "0 | user_biases | ZeroEmbedding | 941 \n", + "1 | item_biases | ZeroEmbedding | 1.4 K \n", + "2 | user_embeddings | ScaledEmbedding | 9.4 K \n", + "3 | item_embeddings | ScaledEmbedding | 14.5 K\n", + "4 | dropout | Dropout | 0 \n", + "----------------------------------------------------\n", + "26.3 K Trainable params\n", + "0 Non-trainable params\n", + "26.3 K Total params\n", + "0.105 Total estimated model params size (MB)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Detected GPU. Setting ``gpus`` to 1.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7f9d29d6c1f04d4b9adcda6292a698fb", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Global seed set to 22\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\r" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "71fb17475171409ab7b30f0915d3c3f9", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "85b96bc221984d08815cf3c09fde7c9f", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b6f3004c83574b379bc3520a410b5df5", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "127ad96af11c41519ded336bbc246c42", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n", + "Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1997f9f7a4d04e5ebec4b8907e32b797", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2bb338780a1a40318d91fc689692c9ae", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b066cece2abe4a74b1aed6c238cd82b3", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 6: reducing learning rate of group 0 to 1.0000e-04.\n", + "Epoch 6: reducing learning rate of group 0 to 1.0000e-04.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2da2765da24841709884a7cca16f1107", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2391e70d0de74d71ab5372f39a4fa49b", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "13c2754aca2a42c2ac06d3640062b6d4", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9adf92080dd2458a942dbd76a285ce48", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NQdGTCPvg9AN" + }, + "source": [ + "### Evaluate the Model " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ISQzTUnVg9AO" + }, + "source": [ + "Again, we'll evaluate the model and look at some particular users' recommendations to get a sense of what these recommendations look like using a partial credit loss function during model training. " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117, + "referenced_widgets": [ + "b1c126acba99422c9aad4bc9b1b0293f", + "3e7789b77e3342acabd132d924906d5e", + "a9db4a7b06d34e2eaee4933580e8e28e", + "2e28af584886415c869f079215546e5e", + "e078e83f8216456997e974acfc7f4816", + "12b2182ba29547a8845685618988224e", + "6d1261287d6042b486877c29f1056305", + "6715a50cf1074e98b4e8e8f9aca4932e" + ] + }, + "id": "6STWH4Ozg9AO", + "outputId": "861755ec-75be-40fe-975b-17accf21477f" + }, + "source": [ + "mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n", + "\n", + "print(f'MAP@10 Score: {mapk_score}')\n", + "print(f'MRR Score: {mrr_score}')\n", + "print(f'AUC Score: {auc_score}')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b1c126acba99422c9aad4bc9b1b0293f", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "MAP@10 Score: 0.02882727154889818\n", + "MRR Score: 0.1829242957435939\n", + "AUC Score: 0.8585049499223719\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bk75mVQWg9AP" + }, + "source": [ + "Broken record alert: we're not seeing as much performance increase here compared to the standard model because MovieLens 100K has so few items. For a more dramatic difference, try training this model on a larger dataset, such as MovieLens 10M, adjusting the architecture-specific hyperparameters, or train longer. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9X25yfucbbKx" + }, + "source": [ + "### Inference" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dB6eeXWfg9AP", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "805ffd4f-38fd-4f4b-d9d1-b9ddfbdda60c" + }, + "source": [ + "user_id = np.random.randint(10, train_interactions.num_users)\n", + "\n", + "display(\n", + " HTML(\n", + " get_recommendation_visualizations(\n", + " model=model,\n", + " user_id=user_id,\n", + " filter_films=True,\n", + " shuffle=True,\n", + " detailed=True,\n", + " )\n", + " )\n", + ")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "

User 895:

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Willy Wonka and the Chocolate Factory (1971)Mighty Aphrodite (1995)Conspiracy Theory (1997)Sense and Sensibility (1995)Liar Liar (1997)In & Out (1997)Return of the Jedi (1983)Ransom (1996)Emma (1996)Toy Story (1995)
Some loved films:
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Cold Comfort Farm (1995)Apartment, The (1960)Blown Away (1994)Star Wars (1977)Star Trek: First Contact (1996)Sex, Lies, and Videotape (1989)Big Squeeze, The (1996)Client, The (1994)Jerry Maguire (1996)Ghost and the Darkness, The (1996)
Recommended films:
-----

User 895 has rated 12 films with a 4 or 5

User 895 has rated 8 films with a 1, 2, or 3

% of these films rated 5 or 4 appearing in the first 10 recommendations:10.0%

% of these films rated 1, 2, or 3 appearing in the first 10 recommendations: 10.0%

" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZoIh0oHfg9AQ" + }, + "source": [ + "Partial credit loss is useful when we want an easy way to boost performance of any implicit model architecture, hybrid or not. When tuned properly, partial credit loss more fairly penalizes the model for more egregious mistakes and relaxes the loss applied when items are more similar. \n", + "\n", + "Of course, the loss function isn't the only place we can incorporate this metadata - we can also directly use this in the model (and even use a hybrid model combined with partial credit loss). Next, we will train a hybrid Collie model! " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Laxa0vh1hE3o" + }, + "source": [ + "### Train a ``MatrixFactorizationModel`` " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Fj3tJg-1hE3o" + }, + "source": [ + "The first step towards training a Collie Hybrid model is to train a regular ``MatrixFactorizationModel`` to generate rich user and item embeddings. We'll use these embeddings in a ``HybridPretrainedModel`` a bit later. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "m75xWkQLhE3o" + }, + "source": [ + "model = MatrixFactorizationModel(\n", + " train=train_interactions,\n", + " val=val_interactions,\n", + " embedding_dim=30,\n", + " lr=1e-2,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 438, + "referenced_widgets": [ + "62986af6e0394e969bb8942274ba6784", + "4e51bc72f9fe41ca82096d01e26a5c23", + "a3b4a6f432f7406b97c59bdd31eec848", + "d6b7dc0b58874a93927b9182fe92ae63", + "2528199682404110adb35e51bf1c39ff", + "16dc9e31d8bb483faa575fd6db3915d7", + "d9b41c7acb1741d98f5edf4e942e3b77", + "7aabb759b56a4fc295687e0a24b6cb62", + "b1ccac573c5c4c79b4357c14f99a08b6", + "1c45f04d9cdc4876969a8a85d9087f18", + "84065182993c4cdcbc3e5a37c769447b", + "13ef2e644cae4a48ad8ab142e8b6435e", + "191a266c1f034c9fa6e7b4138805d60c", + "b8f64ba5faab4995a8b6f676b5a8e5d5", + "22cb3e6197d34688914565f07d26ecef", + "cc2768fdc75942c4a894bcf6006ad11f", + "952b7b171ddb4f4eb96cc91f1257ca9a", + "660daa560af241f2911156e2b7187c7e", + "847fd69ae9924b03929b1a0e59e80f87", + "e1cfc5099d1b4cc7bedcc7a53da34453", + "9b62c6c326784d799f790398e7fa22e9", + "65115b3c64504cc09ed3459b778995bf", + "069b286ddde94f77a425f288e6e14d09", + "f051d817821744d78a30577f69178e1e", + "c0632ad487cf400e906da9a52efa239a", + "12183c2f615f435a9cd920100f2820d4", + "839eb1025574493ca1cf19f5a2f2010e", + "ca5087b11f79495e8d32cfe8cca64f7b", + "949ae8d78ed644cebf9b8fd86a6be7fc", + "38f2d3b8e5ad41d1af054ef4c2ea47be", + "195cc632420d4e85aadf55741dafe166", + "e6ee962e992e44cca46b92e93b9f2c37", + "b06e9ed8cacb428ca888b8c7e0558e54", + "77d14fa2a785407181e4a9da9d40a3d0", + "fd4cec7630304a159031e309fdbd715c", + "cffb8b1a826b429488fc7d6af390df80", + "c58a9370d7484bf8bf50c8e1fef3da12", + "8d8c48b368c943aca716058d8a6c7485", + "4ff6be2f8fe8422fa6ed67aa4e4bb2cc", + "741722e693c344cca086aacee15eb874", + "d227738e76a3420ba0a79a6684de0dcf", + "0057a0667a434d1e8ac207b121049b2c", + "c6316fd570b741628ea906f553c52679", + "845c79c672984d0bab35df090dc632e5", + "52d654208b6b4001a56c287a822354d3", + "c47d7500fff14be39a2184e2d30b3795", + "f6273966027146e9b345c02f935e65be", + "72d633d88bd6413cad4cd3d2717dc7ca", + "b423c474bb5d4ba98c14502c6b02d95d", + "75241cec097b4fba877099eede78e3e7", + "2cef863fdeb0413a8a3ce9bc68d1f985", + "be5f89ff1dbc41b2a8263817d4a59f1f", + "9459414718e44c6a9b88760f1fb1b46a", + "1c8e2a3fde7942e697e111b07aefe26b", + "637168904e464ac9a6782ada14784eb1", + "a069be3b5a6d46e083524d14053bcb9f", + "71907183c5584961bfd232d68e685d3d", + "9e35548cd1714a0c9bf3ccc52d268fb1", + "5a131f872f8d4fcb96b713b60e56656b", + "7e72190f3fd44d1489badb8d28b6bb29", + "82dc472697914b1aaf5866e1573c662c", + "dd1af81809254587aa2a3d0deacff0a0", + "b61d4544c6a54060a2dc120eee5362f3", + "3b096679a00843148f3b874b9c70e901", + "2e2eb1edad7748ee92249d555840d612", + "8b2752894a8b4a69af0ad6be380c8c23", + "09a7ea71de884bcc94aa330697ab728b", + "29c1f761f5634be2bf0ff25736fd7d61", + "be4983b296534b4ba8991262f81d6c0a", + "058111735ae7416fab7f7adb8c981d14", + "1e7ed32472134c508325266efe272c40", + "d46475c0add24876b714a409bc9933b2", + "51757d5291844c6981c0258a245ad5be", + "36090e9a0cc94f56a377ff368eca2b64", + "cb77a4c364704cdca658427246c1a3fb", + "921e0906b18841d4b68b0b72dc911121", + "86439582332e43cea3d1eea6062510f2", + "edb0b8167858489c8f94c95337647d9f", + "92b45c3467f44cdfb4b12fc7d9f25f5a", + "1b13814df36f42969c8795c40b351ccf", + "d1a11fbd7c3a4dadbdee14b744937c30", + "248753435c404995812b93422ca8c062", + "2b3ae44914934e31ab2229c3b7d1b9a7", + "08c6b30a70024da28e546ff11dce4f94", + "ce736cb44cec467e8d6db28528e30780", + "d0ea236e247c46c78d6ee489433b76cf", + "33e822baba3b4f0bbec237b48deba7f4", + "fb8e91ab0228428487907e96827a91b9", + "c7fec0dc1e8648ca869c65caefa05751", + "acaa2bdd5eef41a49a99c02f384a8173", + "2a62b154d02740988eaf6df9ce39cd93", + "369e4fbc8a1b4c6d91f502a7fe5b18f4", + "f2ff83d1d99d40c7bb9d563ad623e164", + "e359213b91d841e3a28cdc293668a104", + "86a3bee8f0604c9a922c94e4161fc24b", + "f260359ad48e4f4c99e7aec1fc5cd09f" + ] + }, + "id": "bjh0jE0yhE3p", + "outputId": "34e09142-f66f-4d32-9052-5e41d3e7d166" + }, + "source": [ + "trainer = CollieTrainer(model=model, max_epochs=10, deterministic=True)\n", + "\n", + "trainer.fit(model)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "----------------------------------------------------\n", + "0 | user_biases | ZeroEmbedding | 941 \n", + "1 | item_biases | ZeroEmbedding | 1.4 K \n", + "2 | user_embeddings | ScaledEmbedding | 28.2 K\n", + "3 | item_embeddings | ScaledEmbedding | 43.4 K\n", + "4 | dropout | Dropout | 0 \n", + "----------------------------------------------------\n", + "74.0 K Trainable params\n", + "0 Non-trainable params\n", + "74.0 K Total params\n", + "0.296 Total estimated model params size (MB)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Detected GPU. Setting ``gpus`` to 1.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "62986af6e0394e969bb8942274ba6784", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Global seed set to 22\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\r" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b1ccac573c5c4c79b4357c14f99a08b6", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "952b7b171ddb4f4eb96cc91f1257ca9a", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c0632ad487cf400e906da9a52efa239a", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b06e9ed8cacb428ca888b8c7e0558e54", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n", + "Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d227738e76a3420ba0a79a6684de0dcf", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b423c474bb5d4ba98c14502c6b02d95d", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "71907183c5584961bfd232d68e685d3d", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2e2eb1edad7748ee92249d555840d612", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "51757d5291844c6981c0258a245ad5be", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d1a11fbd7c3a4dadbdee14b744937c30", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c7fec0dc1e8648ca869c65caefa05751", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117, + "referenced_widgets": [ + "9b69ca9af02e4dcead166064746dfcb2", + "1748fb86e5824e739a39350e4777a4eb", + "27157458047e4a769ae205d23b8e6221", + "2c7ac2735ed649cfafc52ca48eb38d1a", + "ed61762550634ce6a906b6da4550f4c7", + "b1ab812f3fae427db2b8cef8fe572f83", + "08103e370a6047068d6e7d54653c1f3a", + "1d603bf9ea684001b97a63be7f446e3b" + ] + }, + "id": "6tvE66cfhE3p", + "outputId": "1424a753-c468-48c2-dfc9-3b2118195955" + }, + "source": [ + "mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)\n", + "\n", + "print(f'Standard MAP@10 Score: {mapk_score}')\n", + "print(f'Standard MRR Score: {mrr_score}')\n", + "print(f'Standard AUC Score: {auc_score}')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9b69ca9af02e4dcead166064746dfcb2", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "Standard MAP@10 Score: 0.024415062120220127\n", + "Standard MRR Score: 0.1551878337645617\n", + "Standard AUC Score: 0.8575152364604943\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CRq29RVfhE3q" + }, + "source": [ + "### Train a ``HybridPretrainedModel`` " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lFh1LEcChE3q" + }, + "source": [ + "With our trained ``model`` above, we can now use these embeddings and additional side data directly in a hybrid model. The architecture essentially takes our user embedding, item embedding, and item metadata for each user-item interaction, concatenates them, and sends it through a simple feedforward network to output a recommendation score. \n", + "\n", + "We can initially freeze the user and item embeddings from our previously-trained ``model``, train for a few epochs only optimizing our newly-added linear layers, and then train a model with everything unfrozen at a lower learning rate. We will show this process below. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "RPgUTdR1hE3r" + }, + "source": [ + "# we will apply a linear layer to the metadata with ``metadata_layers_dims`` and\n", + "# a linear layer to the combined embeddings and metadata data with ``combined_layers_dims``\n", + "hybrid_model = HybridPretrainedModel(\n", + " train=train_interactions,\n", + " val=val_interactions,\n", + " item_metadata=metadata_df,\n", + " trained_model=model,\n", + " metadata_layers_dims=[8],\n", + " combined_layers_dims=[16],\n", + " lr=1e-2,\n", + " freeze_embeddings=True,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 438, + "referenced_widgets": [ + "1b0f046d63ec46909920000d416a956b", + "2010b6b4500e4b139e93077008383a3c", + "3a31cc9116d7465bbeea6471c6e93968", + "560c2c1f4ca94b73a47aeb344ee81ab1", + "cf1581e1b6614caa92b7845465c9de39", + "346d17d614ee426fbe2be794a34737f3", + "52271d9aa0d64d0dacb8d2d0d827adc0", + "eb62d6f5333a4cad90fe00ecab4f4de3", + "73d516b143a94e0a9ff2115a975dfe0b", + "cabcae1934154c03a2ddbfa47b2b4501", + "726b226d57474815ac786d93f4039902", + "1a2825490a2043aca84c13d316d11ed2", + "293790e04a5a44bb9f14edf7ac0ab8cc", + "5ccacb85b1214ee1a0e5665364c235b7", + "fb3d481dfea04ea3b6d1f8d57682826c", + "044c05d2a4824890a595a2f196d64459", + "ad5390af70b14da0aa0d3a0b5d20a90a", + "0f93d0dc68984a279b6382fbedda6630", + "e20d6099034e4567a726f6a7b038395c", + "76f0fc24f91d40e4869be6e1b5b4b0d3", + "a0666c429914487faf1fac59402e7364", + "d6c7474eb9114cde9ff9adde036f3bc2", + "83e1559c03c543c88418b270c16c88f6", + "01c61537bfe842649a60c70fde9a51a3", + "32b3c6d930f94d56b9f6e652303cc1ca", + "27b7e22b72b94b9da9d22190c338ac84", + "e39a8caba8854f5696d3ff03277bfaa0", + "ad367116739e4f4c8717c5b5b741b2ee", + "916dfdca43f74c058da5b45dff1eb621", + "564e25c0cc9d4f8bb9a00e8f480eaf7b", + "0cfb4e7dc348480d9cae85167df7a73e", + "24f32e83468d4a4eb03b91130a526df5", + "cf3dcff653a3443184d0942defe61230", + "7f75f89935e84173bf5d914bc8f50150", + "8c86d4c10827436b90b7f62885c57ef6", + "3971fc6570fc4f3ca85df75030b1a984", + "b5be05a8a7c045c1bce4eda3852a9256", + "5802083d9af4473684eadd22aa17d5c0", + "00a30c0b9eac4a38b0bd3be2cc05476b", + "dc0817dca939450caa82a751377451d4", + "aa23f91202f7432ab937a60fd0cc69c2", + "fa12cd0edffc4459aadb255b2f85309c", + "cfa25ee4254c479498e117189651d8c4", + "7e6c87609e254706b7e6ee6ac7bf78b4", + "a50062a20b2a475fb760c5ddf6266197", + "82c055fff9914a1d8a6c838062046b99", + "c16130a0a9624f848af1b5553264b78d", + "1f2cb0a8875946439822315ad3a7e1e5", + "22fbff68d90c43d6960fea7ddd420810", + "6f02c441db1a45488e41b29c656de74e", + "1408cb730c964bd39fd32f56916b9701", + "095d3de7d1e1464eb81d927e6edc3b0c", + "e5379796148347bc8f6ab506e1f5b1cd", + "bb09a545620c4dd1af632339d6fdd04a", + "08ee77ef97744d6d92a86674b9247b25", + "ef2fb16bb26f4a489fe456a33a9aaabf", + "c4588fb79e384aeabc04a175fe8d6548", + "d519f414b012498bafdd6ebd8503bbf0", + "a571a202b33d4a0dae7428d244d64cf6", + "f8841a4feab24ea1ace015833bbf0891", + "e0eb2bb0242e44aba710350121762fad", + "2cbdcb710a39477494596ee8e941152a", + "7ef288048daf401699f47fcf24ae2b2a", + "7927e1af2f644102b87c7c13ad22546a", + "c560a29789ca48129d338f88feddfeee", + "a2dc9ff9d3b6413489e37645e5a81969", + "e23fffe8236f4f9ab8b76858b371ed8c", + "7a1cdaf584e44e3392d963cd39f65cc0", + "0303b557637e45078eb791c8e6091f61", + "7025cdb08e7646169e13713fc442b5e3", + "c3216b05839d47d781220cc4dc762e46", + "41b780dbe1394396a24bb180a7945c3d", + "c58dc7499eef43f39a406d873aa9bf2f", + "3a7fe9e14ff94e14a3ea9be6a3685299", + "65d8dc2103ba4794a94ddbf4e1ef9825", + "16a6b8dc6f36458da47ce830fac15d26", + "5be8f16d753c47a6a68248c561a9605e", + "c85c795ddd734662b1277c3b4dd4bb6f", + "37aacb2f901c46c4bc7b9c580890a3c6", + "f837d58642d247cf9ffb657002d83ae6", + "109fb55f40334538966bbdb2961cfdf9", + "1a1fa94f37524d4cb93f5a2bfde53b12", + "7978b82dcd874c2782f4821d859ef918", + "e4f3bb71ec4645649a3e81bfd8b3e310", + "0374e23a55c249bb955a605926214583", + "14acab5f63754fd0970ca0a6e204d47e", + "720887ecd42045539a4f03a57b8a2d5e", + "94f0ec5f8e5e43de8397102e899dde43", + "fcbf9f9aecf54c1f97c3e2e3915c36c2", + "2b71910b6bf74406a468ef78e914db9f", + "7bfe5a6f9f2f4787acb15eedd6e23341", + "365f4e59afcb443f8c3cad99137dcd40", + "d9068f85980247bc89628635dd221f3c", + "c98453288d7e416c8ae6067db3220b5f", + "6657854d04e446339437228998a63325", + "27a1d1d028474aab8ee3cdc1c00166ad" + ] + }, + "id": "vyyUg5ilhE3r", + "outputId": "77ffdf06-0964-4a7d-f491-a83d53d3e210" + }, + "source": [ + "hybrid_trainer = CollieTrainer(model=hybrid_model, max_epochs=10, deterministic=True)\n", + "\n", + "hybrid_trainer.fit(hybrid_model)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "--------------------------------------------------------------\n", + "0 | _trained_model | MatrixFactorizationModel | 74.0 K\n", + "1 | embeddings | Sequential | 71.6 K\n", + "2 | dropout | Dropout | 0 \n", + "3 | metadata_layer_0 | Linear | 160 \n", + "4 | combined_layer_0 | Linear | 1.1 K \n", + "5 | combined_layer_1 | Linear | 17 \n", + "--------------------------------------------------------------\n", + "75.3 K Trainable params\n", + "71.6 K Non-trainable params\n", + "146 K Total params\n", + "0.588 Total estimated model params size (MB)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Detected GPU. Setting ``gpus`` to 1.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1b0f046d63ec46909920000d416a956b", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Global seed set to 22\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\r" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "73d516b143a94e0a9ff2115a975dfe0b", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ad5390af70b14da0aa0d3a0b5d20a90a", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "32b3c6d930f94d56b9f6e652303cc1ca", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cf3dcff653a3443184d0942defe61230", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "aa23f91202f7432ab937a60fd0cc69c2", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "22fbff68d90c43d6960fea7ddd420810", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c4588fb79e384aeabc04a175fe8d6548", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c560a29789ca48129d338f88feddfeee", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c58dc7499eef43f39a406d873aa9bf2f", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "109fb55f40334538966bbdb2961cfdf9", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fcbf9f9aecf54c1f97c3e2e3915c36c2", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 10: reducing learning rate of group 0 to 1.0000e-03.\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117, + "referenced_widgets": [ + "f3842767c3b9491ca0e39c7d83c3bba7", + "42e789d0121c43df87d03eeb4e58d3fe", + "bdebaa528fe345f58ed0e85d0015187b", + "08455bd7b312416a8828902e10a0a6de", + "4a1d091431614ff2a9a94baa79f4ebdd", + "78f224d19e054aa3a6787b9c3aa536eb", + "b6cff5f6b6004d6999a277ef1ed64fe1", + "9c1fd029256144a68b8c996e201c3e08" + ] + }, + "id": "I8eEYwcfhE3s", + "outputId": "9babb089-060c-4636-f3dd-ebbf91d939e6" + }, + "source": [ + "mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, hybrid_model)\n", + "\n", + "print(f'Hybrid MAP@10 Score: {mapk_score}')\n", + "print(f'Hybrid MRR Score: {mrr_score}')\n", + "print(f'Hybrid AUC Score: {auc_score}')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f3842767c3b9491ca0e39c7d83c3bba7", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "Hybrid MAP@10 Score: 0.02650305521043056\n", + "Hybrid MRR Score: 0.15837650977843062\n", + "Hybrid AUC Score: 0.780685132170672\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "EEw83cTUhE3s" + }, + "source": [ + "hybrid_model_unfrozen = HybridPretrainedModel(\n", + " train=train_interactions,\n", + " val=val_interactions,\n", + " item_metadata=metadata_df,\n", + " trained_model=model,\n", + " metadata_layers_dims=[8],\n", + " combined_layers_dims=[16],\n", + " lr=1e-4,\n", + " freeze_embeddings=False,\n", + ")\n", + "\n", + "hybrid_model.unfreeze_embeddings()\n", + "hybrid_model_unfrozen.load_from_hybrid_model(hybrid_model)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 472, + "referenced_widgets": [ + "b776627648254a919b1a18b4025fbcb3", + "2b4a275a432241ec91b5fe2a67fff267", + "b9fed3cbac434c5e8e8eccc37fdc7909", + "518844d0b3cc4deabe8b0ebb657eac8f", + "20a1726ffbf9408b8a00ccd5c44b81a8", + "f9e3ccff7f41406eb5ec4f9658b24718", + "d354c4d6b73d412f87a41f8cbb7bb9eb", + "01f11efae5cc4bc48726f6554c346655", + "bede57139cc14b599167088250548b43", + "512af7cae0c84552b736431802ed4402", + "2a5f13e1cce34787ba032800b8a59ef6", + "2dd137e524324334b5f7c56e7d4d8877", + "e7f0ad3f8d204c1a9c611a897655abbe", + "dbaaabf67d1640d5b988ca6e095577c2", + "9814e7c9d49245e7802828d385800a6d", + "7efc774340274141a6a6d4c3e5bfcf12", + "fe7f069fe1f94fe3a98c44c3cf5c3ed1", + "82de05cc4d4f4b32bf7127007ba17bef", + "3e5ca461bd1f4d6ba6e3f5d2a5c708b4", + "4d565f9c62d84ba487c79a1e2053770d", + "7beb2d7bb4b8421fa6a3fdc5a0a578b1", + "b56b52a8d05c4c2496e18f32df63ee5b", + "55a7ecf02b1f492e82b95e5460e04f22", + "7d93dfd6d22843108d76c8a2416bee21", + "3b0d7093adde4f64bfddffdb4444e1f2", + "aa9c6996aa3c40d18f0b3f0b8cd1705d", + "7dc6a6399f604fef8dda1b5a1d7b2920", + "c1ab4cda390d4a99b92162421e86718a", + "c894ed9774704d88bff3e9d1ad542900", + "4968d6ba2303488c9256042f3a7f8206", + "27da441d4d55492ebc526ca00dd7d01e", + "ad048e4075c847f1b911080e51548cdf", + "84cd0ec866694431a1bc3f6eb7686107", + "8b4924ef271d4097abd6e57303794327", + "200eca6c62424921ab682d2ab8a0785d", + "80117a933a694fd9914f88917466a00f", + "47c26c18cccf44f4b6a5caf3ccdd4e83", + "4fdf8e0a36244ff1bdf34e9060e3f035", + "7478d08325084ef28fa9f1c5a6ab18f6", + "1b0c6cff674c4930b18748d9fa4f9090", + "73232374fc7f4880a87dec552959d3a4", + "03dad58a3f004920aff305a2357ad121", + "fe02f58536de4d49b132a067fc065671", + "7805d39a07414d199a012bad80e90acb", + "094a99863ecc425dae15237f990ffb3e", + "b91817bbe07449b2a9a8d1b6be2ce378", + "82c6bf962cd04ee8bf02d24b03952b28", + "3e392b5c457a4ef2b7c883982f60c36b", + "5ebee37d75004546b316d10399b69431", + "b7fbdb82ca614754a12635170518e0cb", + "6cb3f43666734c869fcc960645e129e9", + "ad4885e1e6ab41f48ac113c30aa13b39", + "87093d6b3c924e6087e9d2b78ba0c6eb", + "0a4702427f524e768b8d1b2379e65499", + "3c9b57c8d7e24cb181e50d3b5e9a89d4", + "1f6a03090e2f4bd5a08b973a2e31a48e", + "43ff46cfff674d37a04bf926feca9048", + "7fb80bf0ab9549989de36323648126cd", + "34c2e6b19152468e8e8bbdcbf1e7d87e", + "521b635587644d588e28f0efba61aea2", + "6408b9869970482d949d6a794700716b", + "61a742fb4fde49fb9e4026e330cf2159", + "05bdc0f323064d359a32a0b8d345dd78", + "424e11f8cff442ee8a7643e56ffb36af", + "2ac6cf2e1f304f06bdc354d04507fecd", + "44ac79c4dfec4caa9bd2e4f987aeca9d", + "ebe6a36807834fb38ff46654e27075c1", + "ac257e025a9545d5b924d2946b422735", + "4606964ad6b5447db1d7498178cb5a78", + "e823a52deb434d5f81deb90ae34adc5e", + "9dc20f131e8642ddae5a90537309d835", + "44eea8a841af4ddb92d93bfe06c4c6fc", + "5a147c6e4b59428ebd7e2e3412cc52fc", + "49bf717484f84a25a7ac27b01b2606a3", + "c91bb1aa56074b398362c4326c9b9b13", + "1aad34c1efeb4af6a9ec6809a12a3569", + "7214617bec6645ba891a2071f6bc6442", + "ab827148c2884c728fe50dd09ce55912", + "d48633090c514ddd9e9fc2baa4bf347d", + "6315ef0a9eb14469a4de8063b9e745ca", + "afe817a0babe466cbbf8e4b802ef3360", + "60f26b29709e462381c221393c45e76b", + "cd7d3ce3534447b98fafe4d5aab0194b", + "f44f6872eb08434ea56d62708770cd25", + "cb38f538344b43928074a1184cd05997", + "286f2daf07f745d891110f78c116823c", + "c456a6bba5804e8e911a89abf34e5670", + "b4b847b559944ce2ba7a4ce34c472120", + "975f803a94994f91891c3fb7f187a135", + "47d4f8168a3049b09471968aca76fa08", + "0af18a7ade304878bdcb8dbca2ef1074", + "a521fbb49c5946b1afca37cbc4052b52", + "dac2cbf4f3f2477ab18adce6db8e77a8", + "5e9949db3b97478a905b23c0a437dd45", + "544a63b30c714580be37d69eb8669328", + "33fbfb7b8c2d4a5bb0e979c626862b13" + ] + }, + "id": "yiA-EylqhE3t", + "outputId": "aefcb665-c88d-43ab-eb7a-322cbc58a262" + }, + "source": [ + "hybrid_trainer_unfrozen = CollieTrainer(model=hybrid_model_unfrozen, max_epochs=10, deterministic=True)\n", + "\n", + "hybrid_trainer_unfrozen.fit(hybrid_model_unfrozen)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "GPU available: True, used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "--------------------------------------------------------------\n", + "0 | _trained_model | MatrixFactorizationModel | 74.0 K\n", + "1 | embeddings | Sequential | 71.6 K\n", + "2 | dropout | Dropout | 0 \n", + "3 | metadata_layer_0 | Linear | 160 \n", + "4 | combined_layer_0 | Linear | 1.1 K \n", + "5 | combined_layer_1 | Linear | 17 \n", + "--------------------------------------------------------------\n", + "75.3 K Trainable params\n", + "71.6 K Non-trainable params\n", + "146 K Total params\n", + "0.588 Total estimated model params size (MB)\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Detected GPU. Setting ``gpus`` to 1.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b776627648254a919b1a18b4025fbcb3", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Global seed set to 22\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\r" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bede57139cc14b599167088250548b43", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fe7f069fe1f94fe3a98c44c3cf5c3ed1", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3b0d7093adde4f64bfddffdb4444e1f2", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "84cd0ec866694431a1bc3f6eb7686107", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 3: reducing learning rate of group 0 to 1.0000e-03.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "73232374fc7f4880a87dec552959d3a4", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5ebee37d75004546b316d10399b69431", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "43ff46cfff674d37a04bf926feca9048", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2ac6cf2e1f304f06bdc354d04507fecd", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 7: reducing learning rate of group 0 to 1.0000e-04.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5a147c6e4b59428ebd7e2e3412cc52fc", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "afe817a0babe466cbbf8e4b802ef3360", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Epoch 9: reducing learning rate of group 0 to 1.0000e-05.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "975f803a94994f91891c3fb7f187a135", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2Txbqf3fbqvD" + }, + "source": [ + "### Evaluate the Model" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 117, + "referenced_widgets": [ + "0d9f2c4528634f63a26527a755b33773", + "6d196fda01ee4b63a7731b169c2f36b7", + "20d9caea7c744c09a8efd05793d2c6db", + "1760eeca2a084b1c8e57a9989139cdf6", + "8ccaa38836fd4fa588723ba2516f635b", + "113a8ae2c462494abf3ca97f65e51d06", + "c0282a49c81f4dce83322f9734eb0efd", + "9848dcb0b0e048c88f9c6243dc5ede33" + ] + }, + "id": "sof4rqMbhE3u", + "outputId": "a344f1bb-0627-4e46-968e-f89bc81a5224" + }, + "source": [ + "mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc],\n", + " val_interactions,\n", + " hybrid_model_unfrozen)\n", + "\n", + "print(f'Hybrid Unfrozen MAP@10 Score: {mapk_score}')\n", + "print(f'Hybrid Unfrozen MRR Score: {mrr_score}')\n", + "print(f'Hybrid Unfrozen AUC Score: {auc_score}')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0d9f2c4528634f63a26527a755b33773", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=28.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "Hybrid Unfrozen MAP@10 Score: 0.02789580198163252\n", + "Hybrid Unfrozen MRR Score: 0.17139103232628614\n", + "Hybrid Unfrozen AUC Score: 0.8118089364191508\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0FzTQc6WbtJA" + }, + "source": [ + "### Inference" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "EwM1pkf_hE3v", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "650e4d07-8566-484c-c7b7-543e7c7428a9" + }, + "source": [ + "user_id = np.random.randint(10, train_interactions.num_users)\n", + "\n", + "display(\n", + " HTML(\n", + " get_recommendation_visualizations(\n", + " model=hybrid_model_unfrozen,\n", + " user_id=user_id,\n", + " filter_films=True,\n", + " shuffle=True,\n", + " detailed=True,\n", + " )\n", + " )\n", + ")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "

User 895:

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Willy Wonka and the Chocolate Factory (1971)Mighty Aphrodite (1995)Conspiracy Theory (1997)Sense and Sensibility (1995)Liar Liar (1997)In & Out (1997)Return of the Jedi (1983)Ransom (1996)Emma (1996)Toy Story (1995)
Some loved films:
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Jerry Maguire (1996)Bad Boys (1995)Blown Away (1994)Santa Clause, The (1994)Tin Cup (1996)Graduate, The (1967)Cold Comfort Farm (1995)Princess Bride, The (1987)Private Benjamin (1980)True Romance (1993)
Recommended films:
-----

User 895 has rated 12 films with a 4 or 5

User 895 has rated 8 films with a 1, 2, or 3

% of these films rated 5 or 4 appearing in the first 10 recommendations:0.0%

% of these films rated 1, 2, or 3 appearing in the first 10 recommendations: 10.0%

" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fNwj-u-AhE3w" + }, + "source": [ + "The metrics and results look great, and we should only see a larger difference compared to a standard model as our data becomes more nuanced and complex (such as with MovieLens 10M data). \n", + "\n", + "If we're happy with this model, we can go ahead and save it for later! " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xYmFQZEhhE3w" + }, + "source": [ + "### Save and Load a Hybrid Model " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "2ZDlfmAVhE3w" + }, + "source": [ + "# we can save the model with...\n", + "os.makedirs('models', exist_ok=True)\n", + "hybrid_model_unfrozen.save_model('models/hybrid_model_unfrozen')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qW3kPpenhE3x", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "da7c6d72-d7ca-4913-98fc-e85691a771f4" + }, + "source": [ + "# ... and if we wanted to load that model back in, we can do that easily...\n", + "hybrid_model_loaded_in = HybridPretrainedModel(load_model_path='models/hybrid_model_unfrozen')\n", + "\n", + "\n", + "hybrid_model_loaded_in" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "HybridPretrainedModel(\n", + " (embeddings): Sequential(\n", + " (0): ScaledEmbedding(941, 30)\n", + " (1): ScaledEmbedding(1447, 30)\n", + " )\n", + " (dropout): Dropout(p=0.0, inplace=False)\n", + " (metadata_layer_0): Linear(in_features=19, out_features=8, bias=True)\n", + " (combined_layer_0): Linear(in_features=68, out_features=16, bias=True)\n", + " (combined_layer_1): Linear(in_features=16, out_features=1, bias=True)\n", + ")" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 98 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qKn2XvzuSaqU" + }, + "source": [ + "## Yet another Movie Recommender from scratch\n", + "> Building and training Item-popularity and MLP model on movielens dataset in pure pytorch." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GUoQMgjCPmkp" + }, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Q6wvep55K6of" + }, + "source": [ + "import math\n", + "import torch\n", + "import heapq\n", + "import pickle\n", + "import argparse\n", + "import numpy as np\n", + "import pandas as pd\n", + "from torch import nn\n", + "import seaborn as sns\n", + "from time import time\n", + "import scipy.sparse as sp\n", + "import matplotlib.pyplot as plt\n", + "import torch.nn.functional as F\n", + "from torch.autograd import Variable\n", + "from torch.utils.data import Dataset, DataLoader" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "jz4ocKNnLN4P" + }, + "source": [ + "np.random.seed(7)\n", + "torch.manual_seed(0)\n", + "\n", + "_model = None\n", + "_testRatings = None\n", + "_testNegatives = None\n", + "_topk = None\n", + "\n", + "use_cuda = torch.cuda.is_available()\n", + "device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "srnZMdMoPh9V" + }, + "source": [ + "### Data Loading" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J52BdmTvKvUv" + }, + "source": [ + "!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.train.rating\n", + "!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.test.rating\n", + "!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/u.data" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "csjr7o5wPd7n" + }, + "source": [ + "### Eval Methods" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "et-6h-pkLLMk" + }, + "source": [ + "def evaluate_model(model, full_dataset: MovieLensDataset, topK: int):\n", + " \"\"\"\n", + " Evaluate the performance (Hit_Ratio, NDCG) of top-K recommendation\n", + " Return: score of each test rating.\n", + " \"\"\"\n", + " global _model\n", + " global _testRatings\n", + " global _testNegatives\n", + " global _topk\n", + " _model = model\n", + " _testRatings = full_dataset.testRatings\n", + " _testNegatives = full_dataset.testNegatives\n", + " _topk = topK\n", + "\n", + " hits, ndcgs = [], []\n", + " for idx in range(len(_testRatings)):\n", + " (hr, ndcg) = eval_one_rating(idx, full_dataset)\n", + " hits.append(hr)\n", + " ndcgs.append(ndcg)\n", + " return (hits, ndcgs)\n", + "\n", + "\n", + "def eval_one_rating(idx, full_dataset: MovieLensDataset):\n", + " rating = _testRatings[idx]\n", + " items = _testNegatives[idx]\n", + " u = rating[0]\n", + "\n", + " gtItem = rating[1]\n", + " items.append(gtItem)\n", + " # Get prediction scores\n", + " map_item_score = {}\n", + " users = np.full(len(items), u, dtype='int32')\n", + "\n", + " feed_dict = {\n", + " 'user_id': users,\n", + " 'item_id': np.array(items),\n", + " }\n", + " predictions = _model.predict(feed_dict)\n", + " for i in range(len(items)):\n", + " item = items[i]\n", + " map_item_score[item] = predictions[i]\n", + "\n", + " # Evaluate top rank list\n", + " ranklist = heapq.nlargest(_topk, map_item_score, key=map_item_score.get)\n", + " hr = getHitRatio(ranklist, gtItem)\n", + " ndcg = getNDCG(ranklist, gtItem)\n", + " return (hr, ndcg)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SvpXLWBpPYKk" + }, + "source": [ + "### Eval Metrics" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "vvU46669Mnmz" + }, + "source": [ + "def getHitRatio(ranklist, gtItem):\n", + " for item in ranklist:\n", + " if item == gtItem:\n", + " return 1\n", + " return 0\n", + "\n", + "\n", + "def getNDCG(ranklist, gtItem):\n", + " for i in range(len(ranklist)):\n", + " item = ranklist[i]\n", + " if item == gtItem:\n", + " return math.log(2) / math.log(i+2)\n", + " return 0" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Rv_-b2rnPQXI" + }, + "source": [ + "### Pytorch Dataset" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "grg5RywRK1H8" + }, + "source": [ + "class MovieLensDataset(Dataset):\n", + " 'Characterizes the dataset for PyTorch, and feeds the (user,item) pairs for training'\n", + "\n", + " def __init__(self, file_name, num_negatives_train=5, num_negatives_test=100):\n", + " 'Load the datasets from disk, and store them in appropriate structures'\n", + "\n", + " self.trainMatrix = self.load_rating_file_as_matrix(\n", + " file_name + \".train.rating\")\n", + " self.num_users, self.num_items = self.trainMatrix.shape\n", + " # make training set with negative sampling\n", + " self.user_input, self.item_input, self.ratings = self.get_train_instances(\n", + " self.trainMatrix, num_negatives_train)\n", + " # make testing set with negative sampling\n", + " self.testRatings = self.load_rating_file_as_list(\n", + " file_name + \".test.rating\")\n", + " self.testNegatives = self.create_negative_file(\n", + " num_samples=num_negatives_test)\n", + " assert len(self.testRatings) == len(self.testNegatives)\n", + "\n", + " def __len__(self):\n", + " 'Denotes the total number of rating in test set'\n", + " return len(self.user_input)\n", + "\n", + " def __getitem__(self, index):\n", + " 'Generates one sample of data'\n", + "\n", + " # get the train data\n", + " user_id = self.user_input[index]\n", + " item_id = self.item_input[index]\n", + " rating = self.ratings[index]\n", + "\n", + " return {'user_id': user_id,\n", + " 'item_id': item_id,\n", + " 'rating': rating}\n", + "\n", + " def get_train_instances(self, train, num_negatives):\n", + " user_input, item_input, ratings = [], [], []\n", + " num_users, num_items = train.shape\n", + " for (u, i) in train.keys():\n", + " # positive instance\n", + " user_input.append(u)\n", + " item_input.append(i)\n", + " ratings.append(1)\n", + " # negative instances\n", + " for _ in range(num_negatives):\n", + " j = np.random.randint(1, num_items)\n", + " # while train.has_key((u, j)):\n", + " while (u, j) in train:\n", + " j = np.random.randint(1, num_items)\n", + " user_input.append(u)\n", + " item_input.append(j)\n", + " ratings.append(0)\n", + " return user_input, item_input, ratings\n", + "\n", + " def load_rating_file_as_list(self, filename):\n", + " ratingList = []\n", + " with open(filename, \"r\") as f:\n", + " line = f.readline()\n", + " while line != None and line != \"\":\n", + " arr = line.split(\"\\t\")\n", + " user, item = int(arr[0]), int(arr[1])\n", + " ratingList.append([user, item])\n", + " line = f.readline()\n", + " return ratingList\n", + "\n", + " def create_negative_file(self, num_samples=100):\n", + " negativeList = []\n", + " for user_item_pair in self.testRatings:\n", + " user = user_item_pair[0]\n", + " item = user_item_pair[1]\n", + " negatives = []\n", + " for t in range(num_samples):\n", + " j = np.random.randint(1, self.num_items)\n", + " while (user, j) in self.trainMatrix or j == item:\n", + " j = np.random.randint(1, self.num_items)\n", + " negatives.append(j)\n", + " negativeList.append(negatives)\n", + " return negativeList\n", + "\n", + " def load_rating_file_as_matrix(self, filename):\n", + " '''\n", + " Read .rating file and Return dok matrix.\n", + " The first line of .rating file is: num_users\\t num_items\n", + " '''\n", + " # Get number of users and items\n", + " num_users, num_items = 0, 0\n", + " with open(filename, \"r\") as f:\n", + " line = f.readline()\n", + " while line != None and line != \"\":\n", + " arr = line.split(\"\\t\")\n", + " u, i = int(arr[0]), int(arr[1])\n", + " num_users = max(num_users, u)\n", + " num_items = max(num_items, i)\n", + " line = f.readline()\n", + " # Construct matrix\n", + " mat = sp.dok_matrix((num_users+1, num_items+1), dtype=np.float32)\n", + " with open(filename, \"r\") as f:\n", + " line = f.readline()\n", + " while line != None and line != \"\":\n", + " arr = line.split(\"\\t\")\n", + " user, item, rating = int(arr[0]), int(arr[1]), float(arr[2])\n", + " if (rating > 0):\n", + " mat[user, item] = 1.0\n", + " line = f.readline()\n", + " return mat" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_CJ2wlKPS-w" + }, + "source": [ + "### Utils" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "YhQs5tfvK9Pf" + }, + "source": [ + "def train_one_epoch(model, data_loader, loss_fn, optimizer, epoch_no, device, verbose = 1):\n", + " 'trains the model for one epoch and returns the loss'\n", + " print(\"Epoch = {}\".format(epoch_no))\n", + " # Training\n", + " # get user, item and rating data\n", + " t1 = time()\n", + " epoch_loss = []\n", + " # put the model in train mode before training\n", + " model.train()\n", + " # transfer the data to GPU\n", + " for feed_dict in data_loader:\n", + " for key in feed_dict:\n", + " if type(feed_dict[key]) != type(None):\n", + " feed_dict[key] = feed_dict[key].to(dtype = torch.long, device = device)\n", + " # get the predictions\n", + " prediction = model(feed_dict)\n", + " # print(prediction.shape)\n", + " # get the actual targets\n", + " rating = feed_dict['rating']\n", + " \n", + " \n", + " # convert to float and change dim from [batch_size] to [batch_size,1]\n", + " rating = rating.float().view(prediction.size()) \n", + " loss = loss_fn(prediction, rating)\n", + " # clear the gradients\n", + " optimizer.zero_grad()\n", + " # backpropagate\n", + " loss.backward()\n", + " # update weights\n", + " optimizer.step()\n", + " # accumulate the loss for monitoring\n", + " epoch_loss.append(loss.item())\n", + " epoch_loss = np.mean(epoch_loss)\n", + " if verbose:\n", + " print(\"Epoch completed {:.1f} s\".format(time() - t1))\n", + " print(\"Train Loss: {}\".format(epoch_loss))\n", + " return epoch_loss\n", + " \n", + "\n", + "def test(model, full_dataset : MovieLensDataset, topK):\n", + " 'Test the HR and NDCG for the model @topK'\n", + " # put the model in eval mode before testing\n", + " if hasattr(model,'eval'):\n", + " # print(\"Putting the model in eval mode\")\n", + " model.eval()\n", + " t1 = time()\n", + " (hits, ndcgs) = evaluate_model(model, full_dataset, topK)\n", + " hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()\n", + " print('Eval: HR = %.4f, NDCG = %.4f [%.1f s]' % (hr, ndcg, time()-t1))\n", + " return hr, ndcg\n", + " \n", + "\n", + "def plot_statistics(hr_list, ndcg_list, loss_list, model_alias, path):\n", + " 'plots and saves the figures to a local directory'\n", + " plt.figure()\n", + " hr = np.vstack([np.arange(len(hr_list)),np.array(hr_list)]).T\n", + " ndcg = np.vstack([np.arange(len(ndcg_list)),np.array(ndcg_list)]).T\n", + " loss = np.vstack([np.arange(len(loss_list)),np.array(loss_list)]).T\n", + " plt.plot(hr[:,0], hr[:,1],linestyle='-', marker='o', label = \"HR\")\n", + " plt.plot(ndcg[:,0], ndcg[:,1],linestyle='-', marker='v', label = \"NDCG\")\n", + " plt.plot(loss[:,0], loss[:,1],linestyle='-', marker='s', label = \"Loss\")\n", + "\n", + " plt.xlabel(\"Epochs\")\n", + " plt.ylabel(\"Value\")\n", + " plt.legend()\n", + " plt.savefig(path+model_alias+\".jpg\")\n", + " return\n", + "\n", + "\n", + "def get_items_interacted(user_id, interaction_df):\n", + " # returns a set of items the user has interacted with\n", + " userid_mask = interaction_df['userid'] == user_id\n", + " interacted_items = interaction_df.loc[userid_mask].courseid\n", + " return set(interacted_items if type(interacted_items) == pd.Series else [interacted_items])\n", + "\n", + "\n", + "def save_to_csv(df,path, header = False, index = False, sep = '\\t', verbose = False):\n", + " if verbose:\n", + " print(\"Saving df to path: {}\".format(path))\n", + " print(\"Columns in df are: {}\".format(df.columns.tolist()))\n", + "\n", + " df.to_csv(path, header = header, index = index, sep = sep)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qEl945qyM1FB" + }, + "source": [ + "### Item Popularity Model" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dzjxYN-mM3uv" + }, + "source": [ + "def parse_args():\n", + " parser = argparse.ArgumentParser(description=\"Run ItemPop\")\n", + " parser.add_argument('--path', nargs='?', default='/content/',\n", + " help='Input data path.')\n", + " parser.add_argument('--dataset', nargs='?', default='movielens',\n", + " help='Choose a dataset.')\n", + " parser.add_argument('--num_neg_test', type=int, default=100,\n", + " help='Number of negative instances to pair with a positive instance while testing')\n", + " \n", + " return parser.parse_args(args={})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "xAr3XZ4sM73x" + }, + "source": [ + "class ItemPop():\n", + " def __init__(self, train_interaction_matrix: sp.dok_matrix):\n", + " \"\"\"\n", + " Simple popularity based recommender system\n", + " \"\"\"\n", + " self.__alias__ = \"Item Popularity without metadata\"\n", + " # Sum the occurences of each item to get is popularity, convert to array and \n", + " # lose the extra dimension\n", + " self.item_ratings = np.array(train_interaction_matrix.sum(axis=0, dtype=int)).flatten()\n", + "\n", + " def forward(self):\n", + " pass\n", + "\n", + " def predict(self, feeddict) -> np.array:\n", + " # returns the prediction score for each (user,item) pair in the input\n", + " items = feeddict['item_id']\n", + " output_scores = [self.item_ratings[itemid] for itemid in items]\n", + " return np.array(output_scores)\n", + "\n", + " def get_alias(self):\n", + " return self.__alias__" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0EDavlooLWT0", + "outputId": "a1f13e86-030e-4530-e2d9-34b0d676570a" + }, + "source": [ + "args = parse_args()\n", + "path = args.path\n", + "dataset = args.dataset\n", + "num_negatives_test = args.num_neg_test\n", + "print(\"Model arguments: %s \" %(args))\n", + "\n", + "topK = 10\n", + "\n", + "# Load data\n", + "\n", + "t1 = time()\n", + "full_dataset = MovieLensDataset(path + dataset, num_negatives_test=num_negatives_test)\n", + "train, testRatings, testNegatives = full_dataset.trainMatrix, full_dataset.testRatings, full_dataset.testNegatives\n", + "num_users, num_items = train.shape\n", + "print(\"Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d\"\n", + " % (time()-t1, num_users, num_items, train.nnz, len(testRatings)))\n", + "\n", + "model = ItemPop(train)\n", + "test(model, full_dataset, topK)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model arguments: Namespace(dataset='movielens', num_neg_test=100, path='/content/') \n", + "Load data done [4.3 s]. #user=944, #item=1683, #train=99057, #test=943\n", + "Eval: HR = 0.4062, NDCG = 0.2199 [0.1 s]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(0.4061505832449629, 0.21988638109018463)" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 20 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IaJQ8h8kNWmr" + }, + "source": [ + "### MLP Model" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "F_GYre42NhDX" + }, + "source": [ + "def parse_args():\n", + " parser = argparse.ArgumentParser(description=\"Run MLP.\")\n", + " parser.add_argument('--path', nargs='?', default='/content/',\n", + " help='Input data path.')\n", + " parser.add_argument('--dataset', nargs='?', default='movielens',\n", + " help='Choose a dataset.')\n", + " parser.add_argument('--epochs', type=int, default=30,\n", + " help='Number of epochs.')\n", + " parser.add_argument('--batch_size', type=int, default=256,\n", + " help='Batch size.')\n", + " parser.add_argument('--layers', nargs='?', default='[16,32,16,8]',\n", + " help=\"Size of each layer. Note that the first layer is the concatenation of user and item embeddings. So layers[0]/2 is the embedding size.\")\n", + " parser.add_argument('--weight_decay', type=float, default=0.00001,\n", + " help=\"Regularization for each layer\")\n", + " parser.add_argument('--num_neg_train', type=int, default=4,\n", + " help='Number of negative instances to pair with a positive instance while training')\n", + " parser.add_argument('--num_neg_test', type=int, default=100,\n", + " help='Number of negative instances to pair with a positive instance while testing')\n", + " parser.add_argument('--lr', type=float, default=0.001,\n", + " help='Learning rate.')\n", + " parser.add_argument('--dropout', type=float, default=0,\n", + " help='Add dropout layer after each dense layer, with p = dropout_prob')\n", + " parser.add_argument('--learner', nargs='?', default='adam',\n", + " help='Specify an optimizer: adagrad, adam, rmsprop, sgd')\n", + " parser.add_argument('--verbose', type=int, default=1,\n", + " help='Show performance per X iterations')\n", + " parser.add_argument('--out', type=int, default=1,\n", + " help='Whether to save the trained model.')\n", + " return parser.parse_args(args={})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "BJVmkqWGNoXM" + }, + "source": [ + "class MLP(nn.Module):\n", + "\n", + " def __init__(self, n_users, n_items, layers=[16, 8], dropout=False):\n", + " \"\"\"\n", + " Simple Feedforward network with Embeddings for users and items\n", + " \"\"\"\n", + " super().__init__()\n", + " assert (layers[0] % 2 == 0), \"layers[0] must be an even number\"\n", + " self.__alias__ = \"MLP {}\".format(layers)\n", + " self.__dropout__ = dropout\n", + "\n", + " # user and item embedding layers\n", + " embedding_dim = int(layers[0]/2)\n", + " self.user_embedding = torch.nn.Embedding(n_users, embedding_dim)\n", + " self.item_embedding = torch.nn.Embedding(n_items, embedding_dim)\n", + "\n", + " # list of weight matrices\n", + " self.fc_layers = torch.nn.ModuleList()\n", + " # hidden dense layers\n", + " for _, (in_size, out_size) in enumerate(zip(layers[:-1], layers[1:])):\n", + " self.fc_layers.append(torch.nn.Linear(in_size, out_size))\n", + " # final prediction layer\n", + " self.output_layer = torch.nn.Linear(layers[-1], 1)\n", + "\n", + " def forward(self, feed_dict):\n", + " users = feed_dict['user_id']\n", + " items = feed_dict['item_id']\n", + " user_embedding = self.user_embedding(users)\n", + " item_embedding = self.item_embedding(items)\n", + " # concatenate user and item embeddings to form input\n", + " x = torch.cat([user_embedding, item_embedding], 1)\n", + " for idx, _ in enumerate(range(len(self.fc_layers))):\n", + " x = self.fc_layers[idx](x)\n", + " x = F.relu(x)\n", + " x = F.dropout(x, p=self.__dropout__, training=self.training)\n", + " logit = self.output_layer(x)\n", + " rating = torch.sigmoid(logit)\n", + " return rating\n", + "\n", + " def predict(self, feed_dict):\n", + " # return the score, inputs and outputs are numpy arrays\n", + " for key in feed_dict:\n", + " if type(feed_dict[key]) != type(None):\n", + " feed_dict[key] = torch.from_numpy(\n", + " feed_dict[key]).to(dtype=torch.long, device=device)\n", + " output_scores = self.forward(feed_dict)\n", + " return output_scores.cpu().detach().numpy()\n", + "\n", + " def get_alias(self):\n", + " return self.__alias__" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "lA9s7rv0LspV", + "outputId": "57d64091-be5b-493d-d20a-5f8d991e69ef" + }, + "source": [ + "print(\"Device available: {}\".format(device))\n", + "\n", + "args = parse_args()\n", + "path = args.path\n", + "dataset = args.dataset\n", + "layers = eval(args.layers)\n", + "weight_decay = args.weight_decay\n", + "num_negatives_train = args.num_neg_train\n", + "num_negatives_test = args.num_neg_test\n", + "dropout = args.dropout\n", + "learner = args.learner\n", + "learning_rate = args.lr\n", + "batch_size = args.batch_size\n", + "epochs = args.epochs\n", + "verbose = args.verbose\n", + "\n", + "topK = 10\n", + "print(\"MLP arguments: %s \" % (args))\n", + "model_out_file = '%s_MLP_%s_%d.h5' %(args.dataset, args.layers, time())\n", + "\n", + "# Load data\n", + "\n", + "t1 = time()\n", + "full_dataset = MovieLensDataset(\n", + " path + dataset, num_negatives_train=num_negatives_train, num_negatives_test=num_negatives_test)\n", + "train, testRatings, testNegatives = full_dataset.trainMatrix, full_dataset.testRatings, full_dataset.testNegatives\n", + "num_users, num_items = train.shape\n", + "print(\"Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d\"\n", + " % (time()-t1, num_users, num_items, train.nnz, len(testRatings)))\n", + "\n", + "training_data_generator = DataLoader(\n", + " full_dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n", + "\n", + "# Build model\n", + "model = MLP(num_users, num_items, layers=layers, dropout=dropout)\n", + "# Transfer the model to GPU, if one is available\n", + "model.to(device)\n", + "if verbose:\n", + " print(model)\n", + "\n", + "loss_fn = torch.nn.BCELoss()\n", + "# Use Adam optimizer\n", + "optimizer = torch.optim.Adam(model.parameters(), weight_decay=weight_decay)\n", + "\n", + "# Record performance\n", + "hr_list = []\n", + "ndcg_list = []\n", + "BCE_loss_list = []\n", + "\n", + "# Check Init performance\n", + "hr, ndcg = test(model, full_dataset, topK)\n", + "hr_list.append(hr)\n", + "ndcg_list.append(ndcg)\n", + "BCE_loss_list.append(1)\n", + "\n", + "# do the epochs now\n", + "\n", + "for epoch in range(epochs):\n", + " epoch_loss = train_one_epoch( model, training_data_generator, loss_fn, optimizer, epoch, device)\n", + "\n", + " if epoch % verbose == 0:\n", + " hr, ndcg = test(model, full_dataset, topK)\n", + " hr_list.append(hr)\n", + " ndcg_list.append(ndcg)\n", + " BCE_loss_list.append(epoch_loss)\n", + " if hr > max(hr_list):\n", + " if args.out > 0:\n", + " model.save(model_out_file, overwrite=True)\n", + "\n", + "print(\"hr for epochs: \", hr_list)\n", + "print(\"ndcg for epochs: \", ndcg_list)\n", + "print(\"loss for epochs: \", BCE_loss_list)\n", + "plot_statistics(hr_list, ndcg_list, BCE_loss_list, model.get_alias(), \"/content\")\n", + "with open(\"metrics\", 'wb') as fp:\n", + " pickle.dump(hr_list, fp)\n", + " pickle.dump(ndcg_list, fp)\n", + "\n", + "best_iter = np.argmax(np.array(hr_list))\n", + "best_hr = hr_list[best_iter]\n", + "best_ndcg = ndcg_list[best_iter]\n", + "print(\"End. Best Iteration %d: HR = %.4f, NDCG = %.4f. \" %\n", + " (best_iter, best_hr, best_ndcg))\n", + "if args.out > 0:\n", + " print(\"The best MLP model is saved to %s\" %(model_out_file))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Device available: cpu\n", + "MLP arguments: Namespace(batch_size=256, dataset='movielens', dropout=0, epochs=30, layers='[16,32,16,8]', learner='adam', lr=0.001, num_neg_test=100, num_neg_train=4, out=1, path='/content/', verbose=1, weight_decay=1e-05) \n", + "Load data done [3.8 s]. #user=944, #item=1683, #train=99057, #test=943\n", + "MLP(\n", + " (user_embedding): Embedding(944, 8)\n", + " (item_embedding): Embedding(1683, 8)\n", + " (fc_layers): ModuleList(\n", + " (0): Linear(in_features=16, out_features=32, bias=True)\n", + " (1): Linear(in_features=32, out_features=16, bias=True)\n", + " (2): Linear(in_features=16, out_features=8, bias=True)\n", + " )\n", + " (output_layer): Linear(in_features=8, out_features=1, bias=True)\n", + ")\n", + "Eval: HR = 0.0848, NDCG = 0.0386 [0.6 s]\n", + "Epoch = 0\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.4429853802195507\n", + "Eval: HR = 0.3945, NDCG = 0.2187 [0.6 s]\n", + "Epoch = 1\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.3646208482657292\n", + "Eval: HR = 0.3818, NDCG = 0.2133 [0.6 s]\n", + "Epoch = 2\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.35764367812979747\n", + "Eval: HR = 0.3924, NDCG = 0.2137 [0.6 s]\n", + "Epoch = 3\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.35384849094297227\n", + "Eval: HR = 0.3796, NDCG = 0.2103 [0.6 s]\n", + "Epoch = 4\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.35072445729290175\n", + "Eval: HR = 0.3818, NDCG = 0.2143 [0.6 s]\n", + "Epoch = 5\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3481164647319212\n", + "Eval: HR = 0.3881, NDCG = 0.2171 [0.7 s]\n", + "Epoch = 6\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3454590990638856\n", + "Eval: HR = 0.4157, NDCG = 0.2292 [0.6 s]\n", + "Epoch = 7\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3422531268162321\n", + "Eval: HR = 0.4231, NDCG = 0.2371 [0.6 s]\n", + "Epoch = 8\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3384355346053762\n", + "Eval: HR = 0.4443, NDCG = 0.2508 [0.6 s]\n", + "Epoch = 9\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3335341374156395\n", + "Eval: HR = 0.4677, NDCG = 0.2598 [0.6 s]\n", + "Epoch = 10\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3280563016347491\n", + "Eval: HR = 0.4719, NDCG = 0.2652 [0.6 s]\n", + "Epoch = 11\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.3223747977760719\n", + "Eval: HR = 0.4995, NDCG = 0.2748 [0.6 s]\n", + "Epoch = 12\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.3164166678753934\n", + "Eval: HR = 0.5090, NDCG = 0.2817 [0.6 s]\n", + "Epoch = 13\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.31102338709726507\n", + "Eval: HR = 0.5143, NDCG = 0.2829 [0.6 s]\n", + "Epoch = 14\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.30582732322604156\n", + "Eval: HR = 0.5175, NDCG = 0.2908 [0.6 s]\n", + "Epoch = 15\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.3016319169092548\n", + "Eval: HR = 0.5429, NDCG = 0.2963 [0.6 s]\n", + "Epoch = 16\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.2980319341254789\n", + "Eval: HR = 0.5493, NDCG = 0.2978 [0.6 s]\n", + "Epoch = 17\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.29476294266469105\n", + "Eval: HR = 0.5504, NDCG = 0.3014 [0.6 s]\n", + "Epoch = 18\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.2921119521985682\n", + "Eval: HR = 0.5589, NDCG = 0.3108 [0.6 s]\n", + "Epoch = 19\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.28990745035406845\n", + "Eval: HR = 0.5620, NDCG = 0.3092 [0.6 s]\n", + "Epoch = 20\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.2876521824250234\n", + "Eval: HR = 0.5514, NDCG = 0.3097 [0.6 s]\n", + "Epoch = 21\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.2858751243245078\n", + "Eval: HR = 0.5578, NDCG = 0.3122 [0.6 s]\n", + "Epoch = 22\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.2843063232125546\n", + "Eval: HR = 0.5567, NDCG = 0.3043 [0.6 s]\n", + "Epoch = 23\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.28271066885277896\n", + "Eval: HR = 0.5663, NDCG = 0.3141 [0.6 s]\n", + "Epoch = 24\n", + "Epoch completed 5.6 s\n", + "Train Loss: 0.2813221255630178\n", + "Eval: HR = 0.5610, NDCG = 0.3070 [0.6 s]\n", + "Epoch = 25\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.28002421261420235\n", + "Eval: HR = 0.5610, NDCG = 0.3110 [0.6 s]\n", + "Epoch = 26\n", + "Epoch completed 5.9 s\n", + "Train Loss: 0.27882074906998516\n", + "Eval: HR = 0.5610, NDCG = 0.3095 [0.6 s]\n", + "Epoch = 27\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.27783915350449484\n", + "Eval: HR = 0.5663, NDCG = 0.3115 [0.6 s]\n", + "Epoch = 28\n", + "Epoch completed 5.7 s\n", + "Train Loss: 0.2768868865122783\n", + "Eval: HR = 0.5631, NDCG = 0.3109 [0.6 s]\n", + "Epoch = 29\n", + "Epoch completed 5.8 s\n", + "Train Loss: 0.2760479487343968\n", + "Eval: HR = 0.5631, NDCG = 0.3092 [0.6 s]\n", + "hr for epochs: [0.08483563096500531, 0.3944856839872747, 0.38176033934252385, 0.39236479321314954, 0.3796394485683987, 0.38176033934252385, 0.38812301166489926, 0.41569459172852596, 0.42311770943796395, 0.4443266171792153, 0.4676564156945917, 0.471898197242842, 0.49946977730646874, 0.5090137857900318, 0.5143160127253447, 0.5174973488865323, 0.542948038176034, 0.5493107104984093, 0.5503711558854719, 0.5588547189819725, 0.5620360551431601, 0.5514316012725344, 0.5577942735949099, 0.5567338282078473, 0.5662778366914104, 0.5609756097560976, 0.5609756097560976, 0.5609756097560976, 0.5662778366914104, 0.5630965005302226, 0.5630965005302226]\n", + "ndcg for epochs: [0.03855482836637224, 0.2186689741068423, 0.21325592738572174, 0.21374918741658008, 0.21033736603276898, 0.21431768576892837, 0.21714573069782853, 0.2292039485312514, 0.23708514689275148, 0.2507826695009706, 0.2598176007060155, 0.2652029648171546, 0.2747717153150814, 0.2817258947342069, 0.28289172403583096, 0.2907608027818361, 0.29626902860751664, 0.29775495439534627, 0.3014327139896777, 0.31075028453364517, 0.30917060839326094, 0.3096903348455541, 0.31217614966561463, 0.3043410687051171, 0.314059797472155, 0.3070033682048637, 0.31104383409268926, 0.3094572048871119, 0.3115140344405953, 0.31090220293994014, 0.3092050624323008]\n", + "loss for epochs: [1, 0.4429853802195507, 0.3646208482657292, 0.35764367812979747, 0.35384849094297227, 0.35072445729290175, 0.3481164647319212, 0.3454590990638856, 0.3422531268162321, 0.3384355346053762, 0.3335341374156395, 0.3280563016347491, 0.3223747977760719, 0.3164166678753934, 0.31102338709726507, 0.30582732322604156, 0.3016319169092548, 0.2980319341254789, 0.29476294266469105, 0.2921119521985682, 0.28990745035406845, 0.2876521824250234, 0.2858751243245078, 0.2843063232125546, 0.28271066885277896, 0.2813221255630178, 0.28002421261420235, 0.27882074906998516, 0.27783915350449484, 0.2768868865122783, 0.2760479487343968]\n", + "End. Best Iteration 24: HR = 0.5663, NDCG = 0.3141. \n", + "The best MLP model is saved to movielens_MLP_[16,32,16,8]_1626069383.h5\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU1fnA8e87SzKTkIUsEAggKKAiu5FWUVFaXKq11FoU7aK1om3pz4ob1mrRat1aF6zVutO6rxQrVlsBl7ogCMjmkoCyCCQkJCH7Muf3x50kk2RmMgkzmST3/TwPT2bunNx7bkbPe8+5575HjDEopZSyL0e8K6CUUiq+NBAopZTNaSBQSimb00CglFI2p4FAKaVszhXvCnRWVlaWGT58eLyroZRSvcrq1av3GmOyg33W6wLB8OHDWbVqVbyroZRSvYqIfBXqMx0aUkopm9NAoJRSNqeBQCmlbK7X3SNQSqlQ6uvr2bFjBzU1NfGuStx4PB6GDBmC2+2O+Hc0ECil+owdO3aQkpLC8OHDEZF4V6fbGWMoLi5mx44djBgxIuLfi1kgEJFHgdOBQmPM2CCfC3AP8B2gCjjfGPNxtOtxwrMnUFxT3G57pieTFWeviPbhlFJxVFNTY9sgACAiZGZmUlRU1Knfi+U9gseBU8J8fiowyv9vDnB/LCoRLAiE266U6t3sGgSadOX8YxYIjDFvAyVhinwP+LuxfACki8igWNVHKaVUcPGcNZQLbA94v8O/rR0RmSMiq0RkVWe7PEop1Z369evX6v3jjz/O3LlzAViwYAG5ublMnDiRMWPG8PTTT8ejiu30iumjxpgHjTF5xpi87OygT0grpVSnLV6zk6m3LmPE/FeZeusyFq/ZGfNjXnbZZaxdu5Z//vOfXHzxxdTX18f8mB2JZyDYCQwNeD/Ev00ppWJu8ZqdXPPSenaWVmOAnaXVXPPS+m4JBgCjRo0iKSmJffv2dcvxwonn9NElwFwReQb4BlBmjNkV7YNkejJDzhpSSvVdN7yykU1fl4f8fM22Uuoafa22Vdc3ctULn/D0ym1Bf2fM4FR+/90jwh63urqaiRMnNr8vKSnhjDPOaFfu448/ZtSoUQwYMCDs/rpDLKePPg2cAGSJyA7g94AbwBjzALAUa+poPtb00QtiUY+mKaLF1cWc8NwJXDPlGs49/NxYHEop1Yu0DQIdbY+U1+tl7dq1ze8ff/zxVoky77rrLh577DE+//xzXnnllQM6VrTELBAYY2Z38LkBfhWr47eVnpiOQxw6bVQpm+joyn3qrcvYWVrdbntuupdnLz46VtXisssu44orrmDJkiVceOGFFBQU4PF4Yna8SPSKm8XR4HQ4SU9Mp6Qm3IxWpZRdXHnyoXjdzlbbvG4nV558aLcc/4wzziAvL49FixZ1y/HCsU0gAMj0ZlJcrT0CpRTMnJTLLWeOIzfdi2D1BG45cxwzJwWdxR4T119/PXfeeSc+34ENRx0osUZoeo+8vDzT1YVpLnrjIqoaqnjyO09GuVZKqZ5g8+bNHH744fGuRtwF+zuIyGpjTF6w8rbrEZRU69CQUkoFslUgyPBk6M1ipZRqw1aBINOTSXVDNVX1VfGuilJK9Rj2CgRe6yEynTmklFItbBUIMjwZgKagVkqpQLYKBE09Ap1CqpRSLewVCDw6NKSUii0R4fLLL29+/6c//YkFCxYArdNQjxo1ijPPPJNNmzY1l62vr2f+/PmMGjWKyZMnc/TRR/Paa68BUFFRwS9+8QsOOeQQJk+ezJFHHslDDz0UlTrbas3i5qEh7REopR44Fnavb789Zxxc8m6Xd5uYmMhLL73ENddcQ1ZWVrvPm1JMADz77LNMnz6d9evXk52dzXXXXceuXbvYsGEDiYmJ7Nmzh7feeguAn//85xx88MF88cUXOBwOioqKePTRR7tcz0C26hEkOBNISUjRewRKKRgyBZwJrbc5E6ztB8DlcjFnzhzuuuuuDsueffbZnHTSSTz11FNUVVXx0EMPce+995KYmAjAwIEDmTVrFgUFBaxcuZKbbroJh8NqtrOzs7n66qsPqK7NdY7KXnqRTE+mDg0pZQevzQ9+xd+koQ58Da23+Rqs33nstOC/kzMOTr21w0P/6le/Yvz48Vx11VUdlp08eTKffvop+fn5DBs2jNTU1HZlNm7cyIQJE5qDQLTZqkcA/ofKdGhIKeVKgOQBQNNi72K9b9tL6ILU1FR+8pOfsHDhwg7LdiXNz80338zEiRMZPHhwV6rXjv16BN5M8kvz410NpVSsRXDlzv7dcM8EaKgBVyJc/DakDIzK4X/zm98wefJkLrgg/FIra9asIS8vj5EjR7Jt2zbKy8vb9QrGjBnDunXr8Pl8OBwOrr32Wq699tp26yN3le16BDo0pJRqlpIDE88DcVg/oxQEADIyMpg1axaPPPJIyDIvvvgib7zxBrNnzyYpKYkLL7yQSy+9lLq6OgCKiop4/vnnGTlyJHl5efzud7+jsbERgJqami71JoKxXSDI8GZQVltGvS/+C0YrpXqAaVfBsG/CtOjceA10+eWXs3fv3lbb7rrrrubpo0888QTLli0jOzsbgJtuuons7GzGjBnD2LFjOf3005t7Bw8//DDFxcXNQWHGjBncfvvtUamnrdJQAzz32XP84YM/8N+z/svA5OhFf6VU/Gkaaoumoe6A5htSSqnW7BcI/E8X67MESillsW8g0CmkSikF2DEQ6NCQUkq1YrtA4HV58Tg92iNQSik/2wUCESHTm6n3CJRSys92gQD0oTKlVOxE62nf7mS7FBNg5RvaVbkr3tVQSsXRCc+eEHRkINOTyYqzV3R/heLInj0CHRpSyvZCtQGxaBvWrl3LN7/5TcaPH8/3v/999u3bB8DChQsZM2YM48eP55xzzgHgrbfeYuLEiUycOJFJkyaxf//+qNenLdv2CPbV7MNnfDjElrFQqT7vtpW38WnJp1363Qv+HTxR3GEZh3H1lM6novjJT37Cvffey7Rp07j++uu54YYbuPvuu7n11lvZunUriYmJlJaWAtaKZvfddx9Tp06loqICj8fTpXPoDFu2gpneTBpNI2W1ZfGuilKqjysrK6O0tJRp06YB8NOf/pS3334bgPHjx3PeeefxxBNP4HJZ1+VTp05l3rx5LFy4kNLS0ubtsWTLHkHgQ2X9Pf3jXBulVCx0dOU+btG4kJ89dspj0a5OUK+++ipvv/02r7zyCjfffDPr169n/vz5nHbaaSxdupSpU6fy+uuvc9hhh8W0HrbtEYCmmVBKxV5aWhr9+/fnnXfeAeAf//gH06ZNw+fzsX37dk488URuu+02ysrKqKiooKCggHHjxnH11Vdz1FFH8emnXRve6gxb9giaFrHXKaRK2VemJ/ikkaYRg66qqqpiyJAhze/nzZvHokWLuOSSS6iqquLggw/mscceo7GxkR/96EeUlZVhjOH//u//SE9P57rrrmP58uU4HA6OOOIITj311AOqTyRiGghE5BTgHsAJPGyMubXN58OARUC6v8x8Y8zSWNYJNN+QUoqYTRH1+XxBt3/wwQfttr377rvttt17771Rr1NHYjY0JCJO4D7gVGAMMFtExrQp9jvgOWPMJOAc4K+xqk+g1MRUXOLSoSGllCK29wimAPnGmC3GmDrgGeB7bcoYoGlxzjTg6xjWp5lDHGR4MnRoSCmliG0gyAW2B7zf4d8WaAHwIxHZASwFfh1sRyIyR0RWiciqoqKiqFQuw5uhQ0NK9UG9bdXFaOvK+cd71tBs4HFjzBDgO8A/RNo/4WWMedAYk2eMyWta2/NAZXoyNRAo1cd4PB6Ki4ttGwyMMRQXF3f6IbRY3izeCQwNeD/Evy3QhcApAMaY90XEA2QBhTGsF2BNId1atjXWh1FKdaMhQ4awY8cOojVy0Bt5PJ5Ws5YiEctA8BEwSkRGYAWAc4Bz25TZBnwLeFxEDgc8QLd8gxmeDIprrCsHEemOQyqlYsztdjNixIh4V6PXidnQkDGmAZgLvA5sxpodtFFEbhSRM/zFLgcuEpF1wNPA+aab+nSZnkxqG2uprK/sjsMppVSPFdPnCPzPBCxts+36gNebgKmxrEMogUtW9kvoffnDlVIqWuJ9szhump4u1mcJlFJ2Z9tA0JxvSGcOKaVszr6BwNMyNKSUUnZm20CQ7kkHtEeglFK2DQRuh5v0xHS9R6CUsj3bBgKwhod0aEgpZXe2DgSab0gppWweCEItTKGUUnZi70DgzaSkWoeGlFL2ZutAkOHJYH/9fmoba+NdFaWUihtbB4LmZwm0V6CUsjF7BwKvPlSmlFK2DgSab0gppWweCDTfkFJK2TwQaI9AKaVsHgi8Li9JriTtESilbM3WgQCs4SHtESil7EwDgeYbUkrZnO0DQYZH8w0ppezN9oEg06s9AqWUvWkg8Gayr2YfDb6GeFdFKaXiwvaBIMOTgcFQWlsa76oopVRc2D4QNOUb0vsESim70kDQ9HSxTiFVStmUBgKPJp5TStmb7QNBhtefZkKHhpRSNmX7QJDiTsHtcOvQkFLKtmwfCEREl6xUStma7QMB+J8u1h6BUsqmNBBg3TDWewRKKbvSQICmmVBK2ZsGAqyhoZKaEowx8a6KUkp1u5gGAhE5RUQ+E5F8EZkfoswsEdkkIhtF5KlY1ieUTE8m9b56yuvK43F4pZSKK1esdiwiTuA+YAawA/hIRJYYYzYFlBkFXANMNcbsE5EBsapPOE1PF5fUlJCWmBaPKiilVNzEskcwBcg3xmwxxtQBzwDfa1PmIuA+Y8w+AGNMYQzrE1Lz2sV6w1gpZUOxDAS5wPaA9zv82wKNBkaLyP9E5AMROSXYjkRkjoisEpFVRUVFUa+o5htSStlZvG8Wu4BRwAnAbOAhEUlvW8gY86AxJs8Yk5ednR31Smi+IaWUncUyEOwEhga8H+LfFmgHsMQYU2+M2Qp8jhUYulV6YjoOcejQkFLKlmIZCD4CRonICBFJAM4BlrQpsxirN4CIZGENFW2JYZ2CcjqcpCem69CQUsqWYjZryBjTICJzgdcBJ/CoMWajiNwIrDLGLPF/dpKIbAIagSuNMXFpjTXfkFK9w+I1O7nj9c/4urSawelerjz5UGZOanv7MX56ev2CiVkgADDGLAWWttl2fcBrA8zz/4srzTekVM9vxBav2ck1L62nur4RgJ2l1Vzz0nqAHlHPztYv0r93rL+XiAOBiCQZY6qiduQeJtOTySdFn8S7GkrFRCQNSSwa2c40YJGUveP1T5vr16S6vpE7Xv8s6PlEs5GNpNzt/w5ev1te28wJh2aT4nHjdEjz/iL5e3dH8JOO0iqIyDHAw0A/Y8wwEZkAXGyM+WVUatBJeXl5ZtWqVVHf7+0f3c6Ln7/Ih+d9GPV9KxVPbRsSAK/byY3fO4Jpo7Mpq66nvKaeOX9fTXFlXbvfz0338r/509vts7OBpem4t5w5LsKyDq446VAGpXvZsLOMjV+X89bnoaePjxrQj5w0DzmpHsqq61j+WRH1jS3tW4LLwZzjRnDcqJaZh+98UcSD72ylrsHX6XIuh3DU8P4kup3sLqthd3kNpVX1IevXJCXRRarXzZ7yGhp87dtfj9vBtNEtx37r8yJq6n3tygX7XsIRkdXGmLygn0UQCD4EzsKa3TPJv22DMWZsxDWIolgFgofXP8w9H9/DyvNW4nV5o75/pTojGleplbUNfFFYwfmPrqS0uuMGKpyfTR3BxGHpTBqazqovS/jtyxtCNvDGGIor6zj17ncoqqhtt69+iS5+NnU4bqcDt8uByyHcuyyfsjB1dDmEkQP6sa2kiqq6xnafJyc6OXZkFrvLa9ldVs2e8vbHjQWHwJjBqeSkeslJS2TJ2q8pr2loV65/kptfTx9FeU29FXirG3jx4x0h93tYTkrz60937w9aRoCtt54WcV3DBYKIhoaMMdtFJHBT+2+il2t6lqC4upghKUPiXBvVV0VriMYYw8sf7+C3izc0Xy3uLK3miufX8dA7BZTXNLC9pLrD+tw0cyxpXjepXjdXPLcuaMOd4HTw5Idf8ej/tgJW49f2Qra6vpH5L37CX1fks2NfddDGuklFbQMLl+V3WLcmS+ZOZfTAFDxuZ8hexs0zW/cyRsx/lWCXuAI8+fNvNL8/9+HgIwCRljMG/vXr45rf5x2UEbR+v//uEe2+5w+2FLOztP13lJvu5d+/Ob75/dRblwUtNzg9eheskQSC7f7hISMibuBSYHPUatBDBOYb0kCgYiFUA19b38iEYelsL6lme0kVf/7PZ0HHmS97bi3Xvrye+kZDvc9HsM58g8/w2e4KThmbw6wjhzI6J4Xr/7kh6BVybrqXH33zoOb31552eMihnNPGD+Kz3ftZs72U6xZvCHp+NQ0+hmUkM3VkFsMykrh3WT4lIYaa3r36RBp9pvlcTrrrbXaX1QQtO35IyzOmTY1pR8F0cLo3ZON5zMisVvs/0HKBIq0fwJUnHxr0733lyYd2qdyBiCQQXALcg5UeYifwBvCrqNWghwjsESgVC3e8HryBv9p/td8RY2D2lGG4nA4SnBLyqrrRZ/jLuZNbjlHXGFFD0lEjNjY3jbG5aTywoiDklezDP20ZeeiflBDyuCKCyym4nODFyfxTDou4sZs5KbfDm6TRbmQ70xhHUr+mctBx0OhMcOmqDgOBMWYvcF7UjthDab4hFUsllXVBG88m986exNCMJIb293LGX95lZ2nwq+PfnT6m+f2LH++M+lVqNBvZzh430rKRiHYjG6vGuDNBI5bTYyO5WfwYtB9uM8b8LFaVCidWN4vrGus48okjmTtxLhdPuDjq+1f2VFZdzyPvbOGRd7dSGWLcvO3sj0hn23RmVk609fTnDVR7B3qz+F8Brz3A94Gvo1GxniTBmUCKO0UTz6lOC9YozhgzkMff+5K/vWXduD1t3CDG5aZyz5v5UbuS7o4hg1BifYWqulckQ0MvBr4XkaeBd2NWozjK9Gbq0JBq1tUZPle+sI4Ep4PKuka+ffgALpsxmiMGWwse5aR5ozZE05lySoXTlRQTo4C4rCQWaxmeDL1ZbAPRmMJpjKGitoFbXtvc7gZwfaPBIYaXf3kMk4b1b/WZNtyqJ+owEIjIfqx7BOL/uRu4Osb1iotMbyYFpQXxroaKoXAN/Cljc9hTXsOushpueGVj0Bk+lz+/jgWvbKS8ur7dXPpAdQ2+dkFAqZ4qkqGhlI7K9BUZngxW1qyMdzVUF0VypX9biFww855by2+e7fgYjT7Dd8cPJtXrIs3r5q/LC4I+tRvNh32UirWQgUBEJof6DMAY83H0qxNfmd5MymrLqPfV43a4410d1QnBrvTnv/gJm3aVkeZNYOPXZWzYWc6uIA8tgfWk7BUnjSYnzUtOqod5z62lcH/wh7D+MLMlu8qAFE/MH/ZRKtbC9Qj+HOYzA0Se7aiXaHqobF/NPgYk9cnbIH1WsIe1ahp8PPi2lRbhoMwkxuamUlpVFzQXTG66l7nTWxbH++13gj9leyBz5ZXqqUIGAmPMid1ZkZ4g8OliDQS9hzEm5MNaAqxbcBKpHquHF2ru/YE+DKUNv+rNIpo1JCJjgTFYzxEAYIz5e6wqFS/6dHHvs/HrMm5+NXTqq8Hp3uYgANrAKxVMJLOGfo+1rvAYrNXGTsV6jqDvBQJPS+I51bPtLqvhT298xosf7yDN6+bMSYNZumF3q7ztB5oLRim7iKRHcBYwAVhjjLlARAYCT8S2WvGR4c0ANPFcTxM4GygnzcP43DTe/mIvDT4fPz92BHNPHEVakpvjR2vaA6W6IpJAUGOM8YlIg4ikAoXA0BjXKy6SXEl4nB4NBN2kKw927Sqz5vlPHJLGwtmTGZaZ1FxWr/SV6ppw00fvA54GVopIOvAQsBqoAN7vnup1LxEh05upQ0PdINh0z6te+IT3t+xlWEYyxRV1FFfW8u8Nu6ltaL9MX1FFXasgoJTqunA9gs+BO4DBQCVWUJgBpBpj+uwq7xmeDL1Z3A1ufa39g111jT6e/chavi85wUlmv8SgQQDg6zApnZVSnRNu+ug9wD0ichBwDvAo4AWeFpFqY8wX3VTHbpXpyWRX5a54V6PP2rq3kr+9VcDu8uAPdgmw6cZT8CY4ge5Zpk8pu4skxcRXwG3AbSIyCSsgXA84Y1y3uMj0ZrKxeGO8q9GrBRv7HzmgH/e/VcBr63fhcjpITnAGzc8/ON3bHASge5bpU8ruIpk+6sKaMnoO8C1gBbAgprWKgxOePaHVkNC4ReMAq4ew4uwVcapV7xNs7H/ec2vxGeiX6GLO8Yfws2OH815+sT65q1QPEe5m8QxgNvAdYCXwDDDHGFPZTXXrVqHuC+j9gs4JlurBZyDV4+Kdq6eT5rUe7tIHu5TqOcL1CK4BngIuN8bs66b6qF5sf019yFQP+2samoNAE23gleoZwt0s7nNJ5VRsbN1byaL3vuSF1TtCltGbu0r1XF1ZoUzZUNsbwFfMGE1GSiKP/28ryz8rwu0UTh8/mBFZSdy/Yove3FWqF9FAYHNdXbZx3vPrMEBWv0R+8+1RnPuNYQxIsXISDstI1pu7SvUiGgj8Mj3BF65PcCTEoTbdI9yyjaeOy2FbcRVb9lZy/T83tLsBbID+SW7emz+dBJej1Wc69q9U76KBwC/YFNGH1z/MPR/fw/JtyzlxWN9bniHYDJ+mdXmbpnyGU1pV3y4IKKV6n5j+Xywip4jIZyKSLyLzw5T7gYgYEcmLZX0666dH/JRR/Udx84c3U1nf92bNhkrT0OgzzJ0+irvPnsiSuVMZlOYJWk5vACvVN8QsEIiIE7gP62G0McBsERkTpFwKcCnwYazq0lVuh5vfH/17CqsKuXfNvfGuTtT4fIaX1+zAIcE/z033Mm/GaGZOymX8kHSuPuUwvO7WD5LrDWCl+o5YDg1NAfKNMVsAROQZ4HvApjbl/oCVwuLKGNYlIsFvnE5g1qGzeGrzU5x+8OmMzRrb8Y463Gf8xs9Xbi3hplc38cmOMoakeyisqKOuIfxiLvp0r1J9WywDQS6wPeD9DuAbgQVEZDIw1BjzqojENRCEu3F66eRLWb5tOTe8fwNPn/Y0Lkdkf7Zw+4x1I9o2AJ1/zHBWfVXC6xv3MCjNw52zJjBzYi5L1n2tT/cqZXNxu1ksIg7gTuD8CMrOAeYADBs2LCb1CXXj9KZXN3HK2Olc841ruGzFZTyx6QnOH3t+h1f6xhhueW1z0H3e8fpnXW5Uuzrd8+alm0lwClecNJoLjz24ObGbNvBKqVgGgp20XslsiH9bkxRgLLBCRABygCUicoYxZlXgjowxDwIPAuTl5XUwl6VrQt043VtRx4Qb3uDI4ekMTz2Ke9f8hcb9R/CnpcWtGtr5L33C53v2k5zoYu32UtZuL6Vof23Qfe4sreb5Vds5ZmQWuf4brl1t4Jt6GN+dMJg95TVsL6liwSsb2wUggIzkROZOH9W1P5BSqs8SY2LSrjZlLf0cK2PpTuAj4FxjTNAczyKyAriibRBoKy8vz6xaFbZIl4TKe5+ZnMDMSbm8V1DMp0XbSD74Thqrh1O9/QKs7PntjchKZuLQdJZ9WkhZdX27zx1C89TMgzKTGJzmYdVX+6hvbPkuPG4HV518KMeNyqaqrpGqukbmPvUxxZV17fbndAgOodXvByPA1ltPC1tGKdU3ichqY0zQmZkx6xEYYxpEZC7wOtbaBY8aYzaKyI3AKmPMklgduytC5b2/7vQxzVfmxRW1/PnDMl7ZcT+u1E9oKJ/Qbj9rr59BepL1EFrbK/imff5x5lgOz03lvfxi3iso5s3Ne2jbhNfU+7jxX5uBzR3WvdFnuGjaIQzN8DK0fxJXvrCOPeXteyM63VMpFUzMegSxEqseAVgNd9ODVLkhhmcafY1M/PtkkPZLKEpjCp/87L12++xoyGfE/FfbBYIm986eRFKCE2+Ck0ufXktRRfsGPjfdy//mt+QIDBWAbjlznN4PUMqm4tIj6I2OG5WFz8DvTjucnx93cNAyToczaBAAMM79rd43L3YzCPoNgnLguk/g7s9bL3YzON0bdFgqN93LdycMbn5/7WmH62IuSqmo00AQIL+wAoCRA/p1eR+flnzK0JShJLuTI17sxjd0ASmD2i/54HP1B1qu9HUxF6VULGggCJBfdOCB4Iev/BCALG9W2HKlNaWkJKTgdDipaAi+7k/b7ZH2MNouu9lEl91USgWjgSBAfmEFXreTwWldv6n652l/Ztv+bWwr38bL+S+HLHfcs8chCGmJaWH398/8f5KakEpKQkrEPYxIy0UaMDSwKNW3aSAIUFBUySEDknGESsITgZOGn9T8OlwgmD9lPvtq9lFaW8qznz0bstzv/ve7iI573qvnkeROop87fG/mo90fkexOpp+7X9QDC2jQUKo30kAQoKCwgqOG9++wXKi1CzI9mREf67zDz2t+HS4QLP3+Usrry9lft5+L3rgoZLl+Cf2orK9kb/XesMf92es/i6h+81bMIzUhldSE1LDljDH4HwgEot8bUUrFngYCv8raBnaWVnNO9tAOy0baUEUjYAxN7bg+AH+b8bfm1+MWjQtZ7pGTHqGivoLK+kp+++5vQ5bLL82nvLac8rrysMc96smjSE9Mp7+nf4fDXA2+huY8TRowlOo5NBD4bSmy1hs4kBvFbXVnwIjUlEFTml+HCwRLZrY87xcusJx72Lnsq91HaU0ppbWlYY89+R+T6e/pz4CkAWHLBfYyOjMspZTqGg0EfvlF1jMA0QwEkYp2wOjOwDIvb16r9+GCxiUTLqGwqpC91Xv5tOTTkOWmPDmFgckDyUnKibge2nNQqus0EPgVFFbidAgHZSbHuyohRdqg9dTA8suJv2x+HS5gzDp0Fnuq9rC7cnfY/c1cPJOc5BxyknN0qMkuHjgWdq9vvz1nHFzybvfXp4/QQOCXX1jBQZlJtlqDN9qBBaITNK48qmVpinABY3jacHZV7mJzSfh8TFe+dSWDkgcxqN8gewaMSBvPeJXrjCFToOgzaAxIvuhMsLZ3RTzPOZ5/xzY0EPjlF1VwSHb3Dwv1Nd15Xw7+qXIAABcSSURBVOTuE+9ufh0uYGwq3sSb296k3tc+E2yg+9fez8DkgQxMGti3AkakjWe8ykHkjd03fwFr/tG6jAhMu6pr+4vnOcdin12kgQCob/Tx5d5KZowZGO+q2EZ3BoxXz3wVn/FRXF3M9Oenhyx3/7r7MSHT/7V4avNTDEwayICkAfF9xqKjxs7ng31brfemzfoUvgbY9yU8fwG4POBKBF9j+3LGB64EePPGls+Nz3rddn+7N8CDJ0BtBdRVQE1568YLoLEeqkrgnTsh+1DIOhT6Dw/R2LkhbQj87x7YtQ6+XgslBe3Pt6EWFk6GjIMhYwRkHgKeNHC4rHo1cbggeQB89AhUFUPlXti/q3WZpjruXg9P/KDlb2NMkHNuhLLt8MRZUF8FdZVQuz/IOdfBxpdg8yvgcII4rZ8Y61ht/44lW+H588Hl9R/b1/57EQdMu7r936KLNBAAXxVX0eAzjNQeQY8TratqhzjITsoOW2b1j1ZTVF3Enqo9/OS1n4Qsd8vKWyI65qKNi8jwZDT/i7iX8fh4iqV9QMo0worzP2kpl1RN8Yj2K/Zl+spY8cjJsGeD1SC3I5CUCRWFULrNakgbqq2fvjbH9TXABw/4GzBHSyMmjtaNkyfdujJPyoT0gyCxHySkwLb3rUbVNFq/02+gtW1TwMOWDrf1O8Ea5M9es/6lDYVBE2DCbEgfBkt+DY211pXxCfOtRr24AAo3W+WD9f58DVDwpvUPIDENkjMhKQsqiwBj/W1SB1tBqHqf/29TY/10uqEh4JzdXuvv506y/vUbYAU1ESjZYjXg4oDsw2DY0dbfwBcQSH0N1t9z35ctx07KgqoiK8A01EB9tfXTF5Do0pkAE8+DlOhduGogAAqikGNIxVc0eg5up5vB/QYzuN/gsOWWz1pOYVUhhVWF/HrZr0OW+9OqP0V87Fs+vIX0xDRSXclBgwBgbX/rDqgugaoSik37RYoAih2AD5h4LuSM44TPHqS4rqxduUyPixVnt6R071SPZf9uuGeC1Ui5PPDLD4I3TM3lGq0GbM5bVrmaMtj7hdUL2Pu59a+yEGqbnlsRyD0STvytFQCS2+Tu2v4hrH4MJv0Yjru89WeNDVZD+trVkP9fqwF2OGHkDJj+O6uxTcq0ejrtziURLlrewbn4z/nXH3dczpkAP14cutFue+xL3u14n1HuDYAGAqAl6+ghGgh6re4casryZpHlzWJM5piw5d47+UlKSj5n376tFJdv5zeFy0OWfWXTE+x3dDxR4eQvHiXVQKq4wB263PunLmhJJbL+jqBlDjhH1dCA50Fe+nboHFWhyg3Js/41lR2cDqQHHKmQzNV/CP7dTrsKijYHbxCdLmuI6IyFLUHI4YbvLgzeyKbkWFfYqx8Lf6Ud7XKx2mcXaCDASi2Rk+qhX6L+Ofq6FfsaYPe29h/ktH4qOtNIyOGZZtXBs8Y2SXngOFKAg5o2BBnGafLekLNocLopdwjTvnomZLkjR59BeX2l9cR34ZqQ5eb8Z07YujX58dIf43V5SXInhS3336/+S5IriSR3UkxyVHXp5rwDeOnbQJhEiZEGK4DhQ6D4P7BoXOj7NuECUFfKxWqfnaQtH9aMIR0W6uWiPEtkRea3YO0TrcuJE7JGweOnW/uoLCRzaC7FLme7w2Y6PDDzAUjLhdRca9z5qTCzPGbcgAvIAFgUOhD88fjbml+Hmyn1+CmPU1lfSWV9JVe9fVXIcomuRCobKimqLgpdN+CyFZeF/bzJic+dSKIzEY/TE7bcbStvI9GZaP1zJYYtu2Hvhuay0Q5CMQtAHZSL1T67yvaBwBhDQWEFP8yLLKeP6qHCNfB1ldZMjJIt4O3ffvZHYx1sXAwbXrBuUDbWtb9xCdZYc/lOSEyF0SdB1qGsyBoNSf1h0RktY8eXfhK0696dT3wfOfDI5tfhAsHDJz3c/DpcYHnhuy9Q1VBFVX0Vl/z3kpDlpg2ZRm1jLTUNNRSUBZnh4/dy/svUNtbSEOzv3MbsV2d3WAasIJTgSCDBmRC23LXvXovb4cbtCDO2BizOX4zb4SbBmRA2YGwt24rL4cLtcMekF9QdaVZsHwh2l9dQWdeo9wd6qkiv9I+/EtY80bqMrwE2LYFVj7Te7kyERh/WTA0HZI6EEcdagcPp9v9MgC/esKYsmkZr6uG4WTDzr9askLYiGL9tvnrbvxteuADOevyAAkZ3BpZDMw7tuBCw4JgFza/DBZYPzv0AsNYAr/PVMeXJ0L2lv0z/C7WNtdQ21obNjzVtyDTqffXUNdbxZfmXIcut2r3KKucLfrO9yXX/uy7s503OWHxGROWmPDkFl7hwOVzNyRdDOf/f51vlpHuaaNsHgublKXXqaPc60KGcrMNg/QvW/PLdn1g/G2tb7ys525opkjHCP8fcP8+8vjpgpkYCnP+v4I33kecH3Gx0wbcXBA8C0Lnx25QcuOC1kB/39lQineF0OPE6wi8ENW3otObX4QJBYBD695f/Dlnu9bNeb34dLli9duZr1PvqqffV84MlPwhZ7pbjbqHB10CDr4Eb3r8hZLlZo2fRYBqay774xYshywpCXWMdVb6qkGWiSQNB84yhnptjqE/qaKzeGOtm7OFnwJq/t/7dxjrY8Lz1z5kAA8bA4d+F/iNgxS3W5y4PXPxO8Abekxab2R9hGvd46g2BJR5BqCNDUoZEVO70g09vfh0uEFxx1BWt3ocLBI+d8ljz63DBKlo0EBRWkOpxkd0v/A0rFaEDGspptJ5OXTgZyr+2HnJqR2DA4XD0ryBnvPWwjitgTLhsR2QNdw+YqdFbxSJHVbyCUE8MQPGggaDQmjEkobr8qnNCXelnHGw1/Ls3WE+87tnQfijH7bWGXgaNh0NPtWbapA62ru6f+6lV3pUY/gGdSBvuSK/ge/CVvh1FOwj1hl5QdwQrMabj3Co9SV5enlm1alXHBSPd303/Zfph2dx+1oSo7dPWyr6GhRPa51tp4k6yhnJyxkLaMHjr1pahnBCzbQD41zzrSv/IC+D0O2NXf6X6KBFZbYzJC/aZrXsEZVX17K2o1WcIOtLRcE9FIRQsg/w3YcvyNkHAf4V/7GUwcJx1s9YRMO++fGd0h3KUUp1m60DQtCqZpp/uQLDhHocLkNZBIikTDpkOuXnwn+tbhnLOfb77hnKUUp1m70BQaPNkc5Hc2K2rgsNOg4/bzNzxNcCejTDsmzD9Ohj5LciZAE35cvZ+3utn2yhlF7YPBAkuB0P6h8+z0mcFu9IXpzV184kfQNHnUBYkL484rCv/sx4DT2rwfetQjlK9hq0DQUFRJQdnJeN0SN9aCzWSc9n3FWSNbp9uwTTC3nxr9s7QKTD5x1Y5bwY89cOW1Lrf+2voIAB6pa9UL2LrQJBfWMH4If6sk7FYVi/aweVAnsZ1uK2FOF6aA1+9Z+VrB+scm9ItOFww9kyY+beWIZ5AMUyDq5SKH9sGgpr6Rrbvq+LMybnWhmlXwdonWxfyNVpXxh/+rWVOe2ou5B4V3bVGD7SBzxwF2z6wlgasLYf0odYqSK3OpR6+etdKu3DQVJh6KRx0DHj6w72TrCt9hwtm3BQ8CDT9jXS4R6k+x7aBYEtRJcYEzBhqTifwuH8JPrEecPr47+3nxIuz/RqijfVWY/zAcf41Rn3W8nbB1iSt2w//XWBlwvSkW41z2/VVxWmtWfqf31vrq1YVW0/btttfvbUe6saXwpytAw46Gr57j5Vgre3Dc30gjYJSqutiGghE5BTgHsAJPGyMubXN5/OAnwMNQBHwM2PMV7GsU5P8YMtTTrsKVj1qvXYlWkvR9Rvgb4R3Wg1x089N/7TWSA1c5zQtt2VdVxFrvrzxWePx+KxyCSlQsNzKoxMuBa9phB0r4es11jJ9SVmQlAEZw639GZ91nIOOhqMugsQUK4dOYqr1uqEa/np0S2K1sx478CmcSqk+KWaBQEScwH3ADGAH8JGILDHGbAootgbIM8ZUicgvgNuBs2NVp0AFhRU4BEZkBSSbcybQ3LAHXh0nZ1n/BgU8fTxlThfWOU2EuR9Z5Yyx8uRX77P+rfgjfPEfKzg4XHDEmXDan61GPfAKvtV6qG74waOhG3i90ldKRaDjRVK7bgqQb4zZYoypA54BvhdYwBiz3BjTlGf1AyCydH9RkF9UwdCMJDzugKdct75l/cwZG9kDThPPs3oAkWSwbFtOBBL7WeP5g8bD6Xf7H9LC+nnSTdasnLbDOJEeF6wr/WHf1Ct9pVRYsQwEucD2gPc7/NtCuRAIelkqInNEZJWIrCoqCr+kXqQKCivar0FQsMyaWXPRishmxUTa0EZSLhYNfNOVvs7wUUqF0SNuFovIj4A8YFqwz40xDwIPgpV07kCP1+gzbNlbyfGjswMPYo3dH3w8OCP8s0Q7g6WmW1BKxUEsewQ7gcCFgIf4t7UiIt8GrgXOMMbUtv08FraXVFHX4GvdIyjOt+bWHzK9O6oQnF7BK6XiIJaB4CNglIiMEJEE4BxgSWABEZkE/A0rCBTGsC6tFBQ1rUoWEAjy37R+xjMQKKVUHMQsEBhjGoC5wOvAZuA5Y8xGEblRRJpWe74D6Ac8LyJrRWRJiN1FVdBkcwXLIOMQ6D+8O6qglFI9RkzvERhjlgJL22y7PuD1t2N5/FDyCyvITkkkzeu2NjTUwpfvWDdplVLKZmI5NNRj5RdVcEh2wPMD21dCfZUOCymlbMl2gcAY07xOcbOCZdbc/eHHxq9iSikVJ7YLBEUVteyvaWg9Y6hgmZXQLVxaZaWU6qNsFwhabhSnWBsq98KudTospJSyLdsFgoK2M4a2rACMBgKllG3ZLhDkF1bQL9HFwNREa0PBMisd9OCJ8a2YUkrFif0CgX/GkIj400osg4NPsFJGK6WUDdkvEBRWtDxRXPQp7N+lw0JKKVuzVSDYX1PPnvLalvsDBcusnwefGL9KKaVUnNkqEBQUVQK0TB0tWAZZo601AZRSyqZsFQha5Riqr4Ev/6fDQkop27NdIHA7hWEZSbD9A2td30O+Fe9qKaVUXNkuEAzPTMbldFhppx1uGD413tVSSqm4slUg2FIUkGOoYLm13GNCcvhfUkqpPs4WgWDxmp0cc8ubbNlbybtf7OW199fCnvV6f0AppbBBIFi8ZifXvLSer8tqANhf28Dy1563PtRAoJRSfT8Q3PH6Z1TXN7ba9g2zln2kQs74ONVKKaV6jj4fCL4urW6zxXC8Yz1vN44FR58/faWU6lCfbwkHp3tbvT9MtpMtZaz3HBmnGimlVM/S5wPBlScfitfdklDueMc6APJO/EG8qqSUUj1KTBev7wlmTsoFrHsFX5dWMyNxE2XJozjlmElxrplSSvUMfT4QgBUMZk7KhboquO0CGHtRvKuklFI9Rp8fGmpl23vQWAuHaLZRpZRqYq9AULAcnIkw7Jh410QppXqMvj809MCxsHt9621/HAQ54+CSd+NTJ6WU6kH6fo9gyBRwJrTe5kywtiullLJBIJh2FUib0xQHTLs6PvVRSqkepu8HgpQcmHgeOPyjYM4E633KwPjWSymleoi+HwjA6hU0BQLtDSilVCv2CARNvQJxaG9AKaXa6PuzhppMuwqKNmtvQCml2rBPIEjJgQtei3ctlFKqx4np0JCInCIin4lIvojMD/J5oog86//8QxEZHsv6KKWUai9mgUBEnMB9wKnAGGC2iIxpU+xCYJ8xZiRwF3BbrOqjlFIquFj2CKYA+caYLcaYOuAZ4HttynwPWOR//QLwLRGRGNZJKaVUG7EMBLnA9oD3O/zbgpYxxjQAZUBm2x2JyBwRWSUiq4qKimJUXaWUsqdeMX3UGPOgMSbPGJOXnZ0d7+oopVSfEstZQzuBoQHvh/i3BSuzQ0RcQBpQHG6nq1ev3isiX3WxTlnA3i7+bk+j59Lz9JXzAD2XnupAzuWgUB/EMhB8BIwSkRFYDf45wLltyiwBfgq8D5wFLDPGmHA7NcZ0uUsgIquMMXld/f2eRM+l5+kr5wF6Lj1VrM4lZoHAGNMgInOB1wEn8KgxZqOI3AisMsYsAR4B/iEi+UAJVrBQSinVjWL6QJkxZimwtM226wNe1wA/jGUdlFJKhdcrbhZH0YPxrkAU6bn0PH3lPEDPpaeKyblIB0PySiml+ji79QiUUkq1oYFAKaVszjaBoKMEeL2JiHwpIutFZK2IrIp3fTpDRB4VkUIR2RCwLUNE/iMiX/h/9o9nHSMR4jwWiMhO//eyVkS+E886RkpEhorIchHZJCIbReRS//Ze9b2EOY9e972IiEdEVorIOv+53ODfPsKfoDPfn7AzoaN9RXQ8O9wj8CfA+xyYgZXq4iNgtjFmU1wr1kUi8iWQZ4zpdQ/JiMjxQAXwd2PMWP+224ESY8yt/iDd3xjToxeOCHEeC4AKY8yf4lm3zhKRQcAgY8zHIpICrAZmAufTi76XMOcxi172vfhzriUbYypExA28C1wKzANeMsY8IyIPAOuMMfcf6PHs0iOIJAGe6gbGmLexnhkJFJh8cBHW/7w9Wojz6JWMMbuMMR/7X+8HNmPlAetV30uY8+h1jKXC/9bt/2eA6VgJOiGK34ldAkEkCfB6EwO8ISKrRWROvCsTBQONMbv8r3cDvXkt0bki8ol/6KhHD6UE418TZBLwIb34e2lzHtALvxcRcYrIWqAQ+A9QAJT6E3RCFNsxuwSCvuZYY8xkrLUefuUfpugT/ClGeut45f3AIcBEYBfw5/hWp3NEpB/wIvAbY0x54Ge96XsJch698nsxxjQaYyZi5WmbAhwWq2PZJRBEkgCv1zDG7PT/LARexvqPpDfb4x/fbRrnLYxzfbrEGLPH/z+vD3iIXvS9+MehXwSeNMa85N/c676XYOfRm78XAGNMKbAcOBpI9yfohCi2Y3YJBM0J8Px32c/BSnjX64hIsv9GGCKSDJwEbAj/Wz1eU/JB/D//Gce6dFlTo+n3fXrJ9+K/MfkIsNkYc2fAR73qewl1Hr3xexGRbBFJ97/2Yk102YwVEM7yF4vad2KLWUMA/iljd9OSAO/mOFepS0TkYKxeAFi5op7qTeciIk8DJ2Cl090D/B5YDDwHDAO+AmYZY3r0jdgQ53EC1vCDAb4ELg4YY++xRORY4B1gPeDzb/4t1vh6r/lewpzHbHrZ9yIi47FuBjuxLtifM8bc6P///xkgA1gD/MgYU3vAx7NLIFBKKRWcXYaGlFJKhaCBQCmlbE4DgVJK2ZwGAqWUsjkNBEopZXMaCJTyE5HGgAyVa6OZpVZEhgdmKlWqJ4npmsVK9TLV/kf6lbIV7REo1QH/+g+3+9eAWCkiI/3bh4vIMn8yszdFZJh/+0ARedmfS36diBzj35VTRB7y55d/w//EKCLyf/4c+p+IyDNxOk1lYxoIlGrhbTM0dHbAZ2XGmHHAX7CeUAe4F1hkjBkPPAks9G9fCLxljJkATAY2+rePAu4zxhwBlAI/8G+fD0zy7+eSWJ2cUqHok8VK+YlIhTGmX5DtXwLTjTFb/EnNdhtjMkVkL9ZCKPX+7buMMVkiUgQMCXz0358W+T/GmFH+91cDbmPMTSLyb6xFbhYDiwPy0CvVLbRHoFRkTIjXnRGYE6aRlnt0pwH3YfUePgrILqlUt9BAoFRkzg74+b7/9XtYmWwBzsNKeAbwJvALaF5cJC3UTkXEAQw1xiwHrgbSgHa9EqViSa88lGrh9a8I1eTfxpimKaT9ReQTrKv62f5tvwYeE5ErgSLgAv/2S4EHReRCrCv/X2AtiBKME3jCHywEWOjPP69Ut9F7BEp1wH+PIM8YszfedVEqFnRoSCmlbE57BEopZXPaI1BKKZvTQKCUUjangUAppWxOA4FSStmcBgKllLK5/weA/uHgFsh5DAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1rDDCMkehE3x" + }, + "source": [ + "Thus far, we keep our focus only on the implicit feedback based matrix factorization model on small movielens dataset. In future, we will be expanding this MVP in the following directions:\n", + "1. Large scale industrial datasets - Yoochoose, Trivago\n", + "2. Other available models in [this](https://github.com/ShopRunner/collie_recs/tree/main/collie_recs/model) repo\n", + "3. Really liked the poster carousel. Put it in dash/streamlit app." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "thC-jHYLJKkz" + }, + "source": [ + "## Training neural factorization model on movielens dataset\n", + "> Training MF, MF+bias, and MLP model on movielens-100k dataset in PyTorch." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "U9XYsONJClRh" + }, + "source": [ + "!pip install -q git+https://github.com/sparsh-ai/recochef.git" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2LS69WtgCuxJ" + }, + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "\n", + "from recochef.datasets.synthetic import Synthetic\n", + "from recochef.datasets.movielens import MovieLens\n", + "from recochef.preprocessing.split import chrono_split\n", + "from recochef.preprocessing.encode import label_encode as le\n", + "from recochef.models.factorization import MF, MF_bias\n", + "from recochef.models.dnn import CollabFNet" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "X7-2sy7dDJte" + }, + "source": [ + "# # generate synthetic implicit data\n", + "# synt = Synthetic()\n", + "# df = synt.implicit()\n", + "\n", + "movielens = MovieLens()\n", + "df = movielens.load_interactions()\n", + "\n", + "# changing rating colname to event following implicit naming conventions\n", + "df = df.rename(columns={'RATING': 'EVENT'})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EGLNfBJBCw38", + "outputId": "06429212-3b1c-4a95-df70-927b8e8a3e43" + }, + "source": [ + "# drop duplicates\n", + "df = df.drop_duplicates()\n", + "\n", + "# chronological split\n", + "df_train, df_valid = chrono_split(df, ratio=0.8, min_rating=10)\n", + "print(f\"Train set:\\n\\n{df_train}\\n{'='*100}\\n\")\n", + "print(f\"Validation set:\\n\\n{df_valid}\\n{'='*100}\\n\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train set:\n", + "\n", + " USERID ITEMID EVENT TIMESTAMP\n", + "59972 1 168 5.0 874965478\n", + "92487 1 172 5.0 874965478\n", + "74577 1 165 5.0 874965518\n", + "48214 1 156 4.0 874965556\n", + "22971 1 166 5.0 874965677\n", + "... ... ... ... ...\n", + "98752 943 139 1.0 888640027\n", + "89336 943 426 4.0 888640027\n", + "80660 943 720 1.0 888640048\n", + "93177 943 80 2.0 888640048\n", + "87415 943 53 3.0 888640067\n", + "\n", + "[80000 rows x 4 columns]\n", + "====================================================================================================\n", + "\n", + "Validation set:\n", + "\n", + " USERID ITEMID EVENT TIMESTAMP\n", + "10508 1 208 5.0 878542960\n", + "83307 1 3 4.0 878542960\n", + "8976 1 12 5.0 878542960\n", + "78171 1 58 4.0 878542960\n", + "9811 1 201 3.0 878542960\n", + "... ... ... ... ...\n", + "81005 943 450 1.0 888693158\n", + "92536 943 227 1.0 888693158\n", + "95003 943 230 1.0 888693158\n", + "94914 943 229 2.0 888693158\n", + "92880 943 234 3.0 888693184\n", + "\n", + "[20000 rows x 4 columns]\n", + "====================================================================================================\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "68zLUPlvC5LK", + "outputId": "46c0f8b6-dd84-4c54-8d55-8eb61fb3fc47" + }, + "source": [ + "# label encoding\n", + "df_train, uid_maps = le(df_train, col='USERID')\n", + "df_train, iid_maps = le(df_train, col='ITEMID')\n", + "df_valid = le(df_valid, col='USERID', maps=uid_maps)\n", + "df_valid = le(df_valid, col='ITEMID', maps=iid_maps)\n", + "\n", + "# # event implicit to rating conversion\n", + "# event_weights = {'click':1, 'add':2, 'purchase':4}\n", + "# event_maps = dict({'EVENT_TO_IDX':event_weights})\n", + "# df_train = le(df_train, col='EVENT', maps=event_maps)\n", + "# df_valid = le(df_valid, col='EVENT', maps=event_maps)\n", + "\n", + "print(f\"Processed Train set:\\n\\n{df_train}\\n{'='*100}\\n\")\n", + "print(f\"Processed Validation set:\\n\\n{df_valid}\\n{'='*100}\\n\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Processed Train set:\n", + "\n", + " USERID ITEMID EVENT TIMESTAMP\n", + "59972 0 0 5.0 874965478\n", + "92487 0 1 5.0 874965478\n", + "74577 0 2 5.0 874965518\n", + "48214 0 3 4.0 874965556\n", + "22971 0 4 5.0 874965677\n", + "... ... ... ... ...\n", + "98752 942 933 1.0 888640027\n", + "89336 942 990 4.0 888640027\n", + "80660 942 643 1.0 888640048\n", + "93177 942 155 2.0 888640048\n", + "87415 942 166 3.0 888640067\n", + "\n", + "[80000 rows x 4 columns]\n", + "====================================================================================================\n", + "\n", + "Processed Validation set:\n", + "\n", + " USERID ITEMID EVENT TIMESTAMP\n", + "10508 0 341.0 5.0 878542960\n", + "83307 0 983.0 4.0 878542960\n", + "8976 0 425.0 5.0 878542960\n", + "78171 0 639.0 4.0 878542960\n", + "9811 0 490.0 3.0 878542960\n", + "... ... ... ... ...\n", + "81005 942 314.0 1.0 888693158\n", + "92536 942 154.0 1.0 888693158\n", + "95003 942 183.0 1.0 888693158\n", + "94914 942 176.0 2.0 888693158\n", + "92880 942 132.0 3.0 888693184\n", + "\n", + "[19917 rows x 4 columns]\n", + "====================================================================================================\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VnhEaj5QC8j1", + "outputId": "f15ef434-6b1d-4f51-f11c-4bfbe4af649b" + }, + "source": [ + "# get number of unique users and items\n", + "num_users = len(df_train.USERID.unique())\n", + "num_items = len(df_train.ITEMID.unique())\n", + "\n", + "num_users_t = len(df_valid.USERID.unique())\n", + "num_items_t = len(df_valid.ITEMID.unique())\n", + "\n", + "print(f\"There are {num_users} users and {num_items} items in the train set.\\n{'='*100}\\n\")\n", + "print(f\"There are {num_users_t} users and {num_items_t} items in the validation set.\\n{'='*100}\\n\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "There are 943 users and 1613 items in the train set.\n", + "====================================================================================================\n", + "\n", + "There are 943 users and 1429 items in the validation set.\n", + "====================================================================================================\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xTiGbb5UCpwM" + }, + "source": [ + "# training and testing related helper functions\n", + "def train_epocs(model, epochs=10, lr=0.01, wd=0.0, unsqueeze=False):\n", + " optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n", + " model.train()\n", + " for i in range(epochs):\n", + " users = torch.LongTensor(df_train.USERID.values) # .cuda()\n", + " items = torch.LongTensor(df_train.ITEMID.values) #.cuda()\n", + " ratings = torch.FloatTensor(df_train.EVENT.values) #.cuda()\n", + " if unsqueeze:\n", + " ratings = ratings.unsqueeze(1)\n", + " y_hat = model(users, items)\n", + " loss = F.mse_loss(y_hat, ratings)\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " print(loss.item()) \n", + " test_loss(model, unsqueeze)\n", + "\n", + "def test_loss(model, unsqueeze=False):\n", + " model.eval()\n", + " users = torch.LongTensor(df_valid.USERID.values) #.cuda()\n", + " items = torch.LongTensor(df_valid.ITEMID.values) #.cuda()\n", + " ratings = torch.FloatTensor(df_valid.EVENT.values) #.cuda()\n", + " if unsqueeze:\n", + " ratings = ratings.unsqueeze(1)\n", + " y_hat = model(users, items)\n", + " loss = F.mse_loss(y_hat, ratings)\n", + " print(\"test loss %.3f \" % loss.item())" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "LxhbI4ECC_Jb", + "outputId": "fa326841-1e15-4900-c0e4-fc7790beb762" + }, + "source": [ + "# training MF model\n", + "model = MF(num_users, num_items, emb_size=100) # .cuda() if you have a GPU\n", + "print(f\"Training MF model:\\n\")\n", + "train_epocs(model, epochs=10, lr=0.1)\n", + "print(f\"\\n{'='*100}\\n\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training MF model:\n", + "\n", + "13.594555854797363\n", + "5.292399883270264\n", + "2.558849573135376\n", + "3.584117889404297\n", + "1.0360910892486572\n", + "1.9875222444534302\n", + "2.920832633972168\n", + "2.4130148887634277\n", + "1.2886441946029663\n", + "1.112807273864746\n", + "test loss 2.085 \n", + "\n", + "====================================================================================================\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fnbkknGIDAs6", + "outputId": "e8466582-7078-49ab-dda7-eeffaa65c8de" + }, + "source": [ + "# training MF with bias model\n", + "model = MF_bias(num_users, num_items, emb_size=100) #.cuda()\n", + "print(f\"Training MF+bias model:\\n\")\n", + "train_epocs(model, epochs=10, lr=0.05, wd=1e-5)\n", + "print(f\"\\n{'='*100}\\n\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training MF+bias model:\n", + "\n", + "13.59664535522461\n", + "9.730958938598633\n", + "4.798837184906006\n", + "1.3603413105010986\n", + "2.697232723236084\n", + "4.214857578277588\n", + "2.871798276901245\n", + "1.3329992294311523\n", + "0.9624974727630615\n", + "1.459389328956604\n", + "test loss 2.269 \n", + "\n", + "====================================================================================================\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "N9ltu-ISDCUY", + "outputId": "06c01140-05a4-4b06-9819-546a7ecdba66" + }, + "source": [ + "# training MLP model\n", + "model = CollabFNet(num_users, num_items, emb_size=100) #.cuda()\n", + "print(f\"Training MLP model:\\n\")\n", + "train_epocs(model, epochs=15, lr=0.05, wd=1e-6, unsqueeze=True)\n", + "print(f\"\\n{'='*100}\\n\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training MLP model:\n", + "\n", + "12.962654113769531\n", + "1.4028953313827515\n", + "15.373563766479492\n", + "2.177295207977295\n", + "2.6291019916534424\n", + "5.752542495727539\n", + "6.88251256942749\n", + "6.2746357917785645\n", + "4.8090314865112305\n", + "3.095308303833008\n", + "1.6791961193084717\n", + "1.1257785558700562\n", + "1.678966760635376\n", + "2.615834951400757\n", + "2.80102276802063\n", + "test loss 2.559 \n", + "\n", + "====================================================================================================\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UxWgUAp7vnIM" + }, + "source": [ + "## Neural Matrix Factorization on Movielens\n", + "> Experiments with different variations of Neural matrix factorization model in PyTorch on movielens dataset." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hfac3W-Z4yEs" + }, + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "74OaJCfn5H4Z", + "outputId": "9ec26cd4-a004-49cf-aefb-9a24c670bf11" + }, + "source": [ + "data = pd.read_csv(\"https://raw.githubusercontent.com/sparsh-ai/reco-data/master/MovieLens_LatestSmall_ratings.csv\")\n", + "data.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdratingtimestamp
0114.0964982703
1134.0964981247
2164.0964982224
31475.0964983815
41505.0964982931
\n", + "
" + ], + "text/plain": [ + " userId movieId rating timestamp\n", + "0 1 1 4.0 964982703\n", + "1 1 3 4.0 964981247\n", + "2 1 6 4.0 964982224\n", + "3 1 47 5.0 964983815\n", + "4 1 50 5.0 964982931" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 2 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "F0Irmpk2oUna", + "outputId": "082a3deb-3f36-4d93-8917-77c27db5fc55" + }, + "source": [ + "data.shape" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(100836, 4)" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 3 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AsSMbrTG6LQr" + }, + "source": [ + "Data encoding" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "m_wEgrHx5U93" + }, + "source": [ + "np.random.seed(3)\n", + "msk = np.random.rand(len(data)) < 0.8\n", + "train = data[msk].copy()\n", + "valid = data[~msk].copy()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "fmeQCQXP6Vtv" + }, + "source": [ + "# here is a handy function modified from fast.ai\n", + "def proc_col(col, train_col=None):\n", + " \"\"\"Encodes a pandas column with continous ids. \n", + " \"\"\"\n", + " if train_col is not None:\n", + " uniq = train_col.unique()\n", + " else:\n", + " uniq = col.unique()\n", + " name2idx = {o:i for i,o in enumerate(uniq)}\n", + " return name2idx, np.array([name2idx.get(x, -1) for x in col]), len(uniq)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "OUfpCvFJ6W72" + }, + "source": [ + "def encode_data(df, train=None):\n", + " \"\"\" Encodes rating data with continous user and movie ids. \n", + " If train is provided, encodes df with the same encoding as train.\n", + " \"\"\"\n", + " df = df.copy()\n", + " for col_name in [\"userId\", \"movieId\"]:\n", + " train_col = None\n", + " if train is not None:\n", + " train_col = train[col_name]\n", + " _,col,_ = proc_col(df[col_name], train_col)\n", + " df[col_name] = col\n", + " df = df[df[col_name] >= 0]\n", + " return df" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "TZDUY2rt6Z9B" + }, + "source": [ + "# encoding the train and validation data\n", + "df_train = encode_data(train)\n", + "df_valid = encode_data(valid, train)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "c9VIAfR6otTe", + "outputId": "dc5ad891-2004-4fda-acfa-22d04525df3b" + }, + "source": [ + "df_train.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdratingtimestamp
0004.0964982703
1014.0964981247
2024.0964982224
3035.0964983815
6045.0964980868
\n", + "
" + ], + "text/plain": [ + " userId movieId rating timestamp\n", + "0 0 0 4.0 964982703\n", + "1 0 1 4.0 964981247\n", + "2 0 2 4.0 964982224\n", + "3 0 3 5.0 964983815\n", + "6 0 4 5.0 964980868" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 8 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iy77qSeCo2WY", + "outputId": "c4ce1748-6c39-44f6-c094-476873135fdb" + }, + "source": [ + "df_train.shape" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(80450, 4)" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 9 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "sOSEDMQPo2Tq", + "outputId": "4327e4a5-b01b-4d03-c829-0220e7c8b36c" + }, + "source": [ + "df_valid.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdratingtimestamp
403885.0964982931
509953.0964982400
2908414.0964981179
3005674.0964982653
3204024.0964982546
\n", + "
" + ], + "text/plain": [ + " userId movieId rating timestamp\n", + "4 0 388 5.0 964982931\n", + "5 0 995 3.0 964982400\n", + "29 0 841 4.0 964981179\n", + "30 0 567 4.0 964982653\n", + "32 0 402 4.0 964982546" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 10 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2uiymy6po2Lo", + "outputId": "f1ebfdf2-a139-46d4-d8e7-850701cb57a2" + }, + "source": [ + "df_valid.shape" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(19591, 4)" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 11 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "y3QSDyZj61Iy" + }, + "source": [ + "Matrix factorization model" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HBPnUZl-6z1g" + }, + "source": [ + "class MF(nn.Module):\n", + " def __init__(self, num_users, num_items, emb_size=100):\n", + " super(MF, self).__init__()\n", + " self.user_emb = nn.Embedding(num_users, emb_size)\n", + " self.item_emb = nn.Embedding(num_items, emb_size)\n", + " self.user_emb.weight.data.uniform_(0, 0.05)\n", + " self.item_emb.weight.data.uniform_(0, 0.05)\n", + " \n", + " def forward(self, u, v):\n", + " u = self.user_emb(u)\n", + " v = self.item_emb(v)\n", + " return (u*v).sum(1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "dBQhfy2l7AAn", + "outputId": "d667d3a3-2baa-467b-e8ab-905c3282780f" + }, + "source": [ + "# unit testing the architecture\n", + "sample = encode_data(train.sample(5))\n", + "display(sample)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdratingtimestamp
63234003.0961596392
96012113.0840875700
31417222.0955944735
17473330.51516141230
66983444.01328232721
\n", + "
" + ], + "text/plain": [ + " userId movieId rating timestamp\n", + "63234 0 0 3.0 961596392\n", + "96012 1 1 3.0 840875700\n", + "31417 2 2 2.0 955944735\n", + "17473 3 3 0.5 1516141230\n", + "66983 4 4 4.0 1328232721" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "9tmmtDTuqnIB" + }, + "source": [ + "num_users = 5\n", + "num_items = 5\n", + "emb_size = 3\n", + "\n", + "user_emb = nn.Embedding(num_users, emb_size)\n", + "item_emb = nn.Embedding(num_items, emb_size)\n", + "\n", + "users = torch.LongTensor(sample.userId.values)\n", + "items = torch.LongTensor(sample.movieId.values)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 102 + }, + "id": "H557JwqSqimK", + "outputId": "b45ad07e-0cf7-4cb0-f9ad-5de2f8a86c08" + }, + "source": [ + "U = user_emb(users)\n", + "display(U)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "tensor([[ 2.2694, 0.9679, 0.3305],\n", + " [-1.1478, -0.7004, -0.8113],\n", + " [-1.2287, -0.7210, 0.3875],\n", + " [ 0.9106, 0.0427, -0.7128],\n", + " [-1.0396, -0.2739, 0.7271]], grad_fn=)" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 102 + }, + "id": "IsdtlmtBq3cj", + "outputId": "418f529d-1481-475e-ad2c-39578baa4cdf" + }, + "source": [ + "V = item_emb(items)\n", + "display(V)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "tensor([[-1.9371, -1.1172, -1.5967],\n", + " [-2.4336, -1.1177, 0.6197],\n", + " [ 0.5889, 1.4830, -1.0103],\n", + " [-0.8294, 0.5744, -1.7315],\n", + " [-1.6733, -0.2447, -0.2630]], grad_fn=)" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 102 + }, + "id": "TKtMvkNfq0q2", + "outputId": "c7bb19a7-3a7b-45db-f4c1-b95f0994750f" + }, + "source": [ + "display(U*V) # element wise multiplication" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "tensor([[-4.3959, -1.0813, -0.5278],\n", + " [ 2.7932, 0.7828, -0.5027],\n", + " [-0.7236, -1.0693, -0.3915],\n", + " [-0.7552, 0.0246, 1.2343],\n", + " [ 1.7397, 0.0670, -0.1912]], grad_fn=)" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "5_AN_dhQq0nE", + "outputId": "7cc9d053-c9f3-49d0-e2a6-f0ea809ecd8f" + }, + "source": [ + "display((U*V).sum(1))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "tensor([-6.0050, 3.0733, -2.1844, 0.5036, 1.6155], grad_fn=)" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W01e58dr86WY" + }, + "source": [ + "Model training" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VC5vARcP7QAc", + "outputId": "b4cea803-bbac-4301-bb1c-12683689948d" + }, + "source": [ + "num_users = len(df_train.userId.unique())\n", + "num_items = len(df_train.movieId.unique())\n", + "print(\"{} users and {} items in the training set\".format(num_users, num_items))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "610 users and 8998 items in the training set\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "yRi5sy-K8-fr" + }, + "source": [ + "model = MF(num_users, num_items, emb_size=100) # .cuda() if you have a GPU" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "4nAGJ4l08_83" + }, + "source": [ + "def train_epocs(model, epochs=10, lr=0.01, wd=0.0, unsqueeze=False):\n", + " optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n", + " model.train()\n", + " for i in range(epochs):\n", + " users = torch.LongTensor(df_train.userId.values) # .cuda()\n", + " items = torch.LongTensor(df_train.movieId.values) #.cuda()\n", + " ratings = torch.FloatTensor(df_train.rating.values) #.cuda()\n", + " if unsqueeze:\n", + " ratings = ratings.unsqueeze(1)\n", + " y_hat = model(users, items)\n", + " loss = F.mse_loss(y_hat, ratings)\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " print(loss.item()) \n", + " test_loss(model, unsqueeze)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "7l_3G5gn9GH3" + }, + "source": [ + "def test_loss(model, unsqueeze=False):\n", + " model.eval()\n", + " users = torch.LongTensor(df_valid.userId.values) #.cuda()\n", + " items = torch.LongTensor(df_valid.movieId.values) #.cuda()\n", + " ratings = torch.FloatTensor(df_valid.rating.values) #.cuda()\n", + " if unsqueeze:\n", + " ratings = ratings.unsqueeze(1)\n", + " y_hat = model(users, items)\n", + " loss = F.mse_loss(y_hat, ratings)\n", + " print(\"test loss %.3f \" % loss.item())" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EztQtZKl9M53", + "outputId": "25713f3c-edff-4333-e45b-ffb2c79375fb" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.1)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "12.914263725280762\n", + "4.8582916259765625\n", + "2.5804786682128906\n", + "3.109440565109253\n", + "0.850287139415741\n", + "1.819737195968628\n", + "2.657919406890869\n", + "2.138274908065796\n", + "1.0904945135116577\n", + "0.9722878932952881\n", + "test loss 1.851 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "AoSgUhWV9O1q", + "outputId": "48e887fa-b55c-4465-e2d0-05789b5a7419" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.01)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "1.6430705785751343\n", + "1.0046814680099487\n", + "0.712002694606781\n", + "0.6611021757125854\n", + "0.7258523106575012\n", + "0.803934633731842\n", + "0.843424379825592\n", + "0.8351688981056213\n", + "0.7928505539894104\n", + "0.737376868724823\n", + "test loss 0.827 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "erRnsApY9Q7e", + "outputId": "1e6a68f3-13b1-4963-b187-5d1b1bc3e438" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.01)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.6877127289772034\n", + "0.6256141066551208\n", + "0.6374999284744263\n", + "0.6272100210189819\n", + "0.6171814799308777\n", + "0.614914059638977\n", + "0.6113061308860779\n", + "0.6033822298049927\n", + "0.595890998840332\n", + "0.592114269733429\n", + "test loss 0.764 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HAwA9Rts9UI1" + }, + "source": [ + "MF with bias" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Dur1n3lo9S3C" + }, + "source": [ + "class MF_bias(nn.Module):\n", + " def __init__(self, num_users, num_items, emb_size=100):\n", + " super(MF_bias, self).__init__()\n", + " self.user_emb = nn.Embedding(num_users, emb_size)\n", + " self.user_bias = nn.Embedding(num_users, 1)\n", + " self.item_emb = nn.Embedding(num_items, emb_size)\n", + " self.item_bias = nn.Embedding(num_items, 1)\n", + " self.user_emb.weight.data.uniform_(0,0.05)\n", + " self.item_emb.weight.data.uniform_(0,0.05)\n", + " self.user_bias.weight.data.uniform_(-0.01,0.01)\n", + " self.item_bias.weight.data.uniform_(-0.01,0.01)\n", + " \n", + " def forward(self, u, v):\n", + " U = self.user_emb(u)\n", + " V = self.item_emb(v)\n", + " b_u = self.user_bias(u).squeeze()\n", + " b_v = self.item_bias(v).squeeze()\n", + " return (U*V).sum(1) + b_u + b_v" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "WAyaietL9ZAq" + }, + "source": [ + "model = MF_bias(num_users, num_items, emb_size=100) #.cuda()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5nEO-IVp9acn", + "outputId": "773ee576-ea2e-40ac-97c7-7d78606595e1" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.05, wd=1e-5)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "12.91020393371582\n", + "9.150527954101562\n", + "4.3840012550354\n", + "1.1575191020965576\n", + "2.46807861328125\n", + "3.7430803775787354\n", + "2.4481022357940674\n", + "1.0781667232513428\n", + "0.816169023513794\n", + "1.3183783292770386\n", + "test loss 2.069 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "nD2fD5A59cK4", + "outputId": "03df230c-70ff-4767-e088-928d54f6cd37" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.01, wd=1e-5)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "1.8935126066207886\n", + "1.3250681161880493\n", + "0.9350242614746094\n", + "0.7446779012680054\n", + "0.722224235534668\n", + "0.7774652242660522\n", + "0.8231741189956665\n", + "0.8222126364707947\n", + "0.7816660404205322\n", + "0.727698802947998\n", + "test loss 0.798 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Os53hZxr9e_T", + "outputId": "d9bf93d3-8411-4535-dcc7-ebcf4a3fbc48" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.001, wd=1e-5)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.6853442788124084\n", + "0.6711287498474121\n", + "0.6592414975166321\n", + "0.6495122909545898\n", + "0.6417150497436523\n", + "0.6356027722358704\n", + "0.6309247612953186\n", + "0.6274365186691284\n", + "0.6249085068702698\n", + "0.6231329441070557\n", + "test loss 0.751 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-XhFy6bU9h48" + }, + "source": [ + "Note that these models are susceptible to weight initialization, optimization algorithm and regularization.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NugoowzF9kCk" + }, + "source": [ + "### Neural Network Model\n", + "Note here there is no matrix multiplication, we could potentially make the embeddings of different sizes. Here we could get better results by keep playing with regularization." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qLVWHOxQ9fVX" + }, + "source": [ + "class CollabFNet(nn.Module):\n", + " def __init__(self, num_users, num_items, emb_size=100, n_hidden=10):\n", + " super(CollabFNet, self).__init__()\n", + " self.user_emb = nn.Embedding(num_users, emb_size)\n", + " self.item_emb = nn.Embedding(num_items, emb_size)\n", + " self.lin1 = nn.Linear(emb_size*2, n_hidden)\n", + " self.lin2 = nn.Linear(n_hidden, 1)\n", + " self.drop1 = nn.Dropout(0.1)\n", + " \n", + " def forward(self, u, v):\n", + " U = self.user_emb(u)\n", + " V = self.item_emb(v)\n", + " x = F.relu(torch.cat([U, V], dim=1))\n", + " x = self.drop1(x)\n", + " x = F.relu(self.lin1(x))\n", + " x = self.lin2(x)\n", + " return x" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ljjju7Yy9x7b" + }, + "source": [ + "model = CollabFNet(num_users, num_items, emb_size=100) #.cuda()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "YuG2Hz5e9yyl", + "outputId": "67b716ed-84e4-4dcf-eaf0-f06609542d0d" + }, + "source": [ + "train_epocs(model, epochs=15, lr=0.05, wd=1e-6, unsqueeze=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "14.657201766967773\n", + "2.586819648742676\n", + "6.025796890258789\n", + "2.89852237701416\n", + "1.1256697177886963\n", + "2.0860772132873535\n", + "2.9243881702423096\n", + "2.806140422821045\n", + "1.9981783628463745\n", + "1.1265769004821777\n", + "0.8947575092315674\n", + "1.4373805522918701\n", + "1.795198678970337\n", + "1.4024922847747803\n", + "0.8697773218154907\n", + "test loss 0.797 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "JYyXb1qO90vb", + "outputId": "b3bcc463-51a8-4625-f547-38ebbf105a7f" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.001, wd=1e-6, unsqueeze=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.7495059967041016\n", + "0.7382366061210632\n", + "0.731941282749176\n", + "0.7295416593551636\n", + "0.7321946024894714\n", + "0.7312469482421875\n", + "0.731982409954071\n", + "0.7298287153244019\n", + "0.7264290452003479\n", + "0.7244617938995361\n", + "test loss 0.774 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "o0d-WRvW92h-", + "outputId": "20f81203-7fbb-44db-b421-c6c20239cd22" + }, + "source": [ + "train_epocs(model, epochs=10, lr=0.001, wd=1e-6, unsqueeze=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.7242854833602905\n", + "0.7213587760925293\n", + "0.7197834849357605\n", + "0.7182263135910034\n", + "0.7177621722221375\n", + "0.7155387997627258\n", + "0.7147852182388306\n", + "0.7143447995185852\n", + "0.7133223414421082\n", + "0.712261974811554\n", + "test loss 0.766 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6fpmHCaYAn2d" + }, + "source": [ + "### Neural network model - different approach\n", + "> youtube: https://youtu.be/MVB1cbe923A" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SMuCwuPWPfGz" + }, + "source": [ + "### Ethan Rosenthal\n", + "\n", + "Ref - https://github.com/EthanRosenthal/torchmf" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J31f-camBorB" + }, + "source": [ + "import os\n", + "import requests\n", + "import zipfile\n", + "import collections\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import scipy.sparse as sp\n", + "from sklearn.metrics import roc_auc_score\n", + "\n", + "import torch\n", + "from torch import nn\n", + "import torch.multiprocessing as mp\n", + "import torch.utils.data as data\n", + "from tqdm import tqdm" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ahu8EWCGQJkI" + }, + "source": [ + "def _get_data_path():\n", + " \"\"\"\n", + " Get path to the movielens dataset file.\n", + " \"\"\"\n", + " data_path = '/content/data'\n", + " if not os.path.exists(data_path):\n", + " print('Making data path')\n", + " os.mkdir(data_path)\n", + " return data_path\n", + "\n", + "\n", + "def _download_movielens(dest_path):\n", + " \"\"\"\n", + " Download the dataset.\n", + " \"\"\"\n", + "\n", + " url = 'http://files.grouplens.org/datasets/movielens/ml-100k.zip'\n", + " req = requests.get(url, stream=True)\n", + "\n", + " print('Downloading MovieLens data')\n", + "\n", + " with open(os.path.join(dest_path, 'ml-100k.zip'), 'wb') as fd:\n", + " for chunk in req.iter_content(chunk_size=None):\n", + " fd.write(chunk)\n", + "\n", + " with zipfile.ZipFile(os.path.join(dest_path, 'ml-100k.zip'), 'r') as z:\n", + " z.extractall(dest_path)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "DouK7x1nPsNb" + }, + "source": [ + "def read_movielens_df():\n", + " path = _get_data_path()\n", + " zipfile = os.path.join(path, 'ml-100k.zip')\n", + " if not os.path.isfile(zipfile):\n", + " _download_movielens(path)\n", + " fname = os.path.join(path, 'ml-100k', 'u.data')\n", + " names = ['user_id', 'item_id', 'rating', 'timestamp']\n", + " df = pd.read_csv(fname, sep='\\t', names=names)\n", + " return df\n", + "\n", + "\n", + "def get_movielens_interactions():\n", + " df = read_movielens_df()\n", + "\n", + " n_users = df.user_id.unique().shape[0]\n", + " n_items = df.item_id.unique().shape[0]\n", + "\n", + " interactions = np.zeros((n_users, n_items))\n", + " for row in df.itertuples():\n", + " interactions[row[1] - 1, row[2] - 1] = row[3]\n", + " return interactions\n", + "\n", + "\n", + "def train_test_split(interactions, n=10):\n", + " \"\"\"\n", + " Split an interactions matrix into training and test sets.\n", + " Parameters\n", + " ----------\n", + " interactions : np.ndarray\n", + " n : int (default=10)\n", + " Number of items to select / row to place into test.\n", + "\n", + " Returns\n", + " -------\n", + " train : np.ndarray\n", + " test : np.ndarray\n", + " \"\"\"\n", + " test = np.zeros(interactions.shape)\n", + " train = interactions.copy()\n", + " for user in range(interactions.shape[0]):\n", + " if interactions[user, :].nonzero()[0].shape[0] > n:\n", + " test_interactions = np.random.choice(interactions[user, :].nonzero()[0],\n", + " size=n,\n", + " replace=False)\n", + " train[user, test_interactions] = 0.\n", + " test[user, test_interactions] = interactions[user, test_interactions]\n", + "\n", + " # Test and training are truly disjoint\n", + " assert(np.all((train * test) == 0))\n", + " return train, test\n", + "\n", + "\n", + "def get_movielens_train_test_split(implicit=False):\n", + " interactions = get_movielens_interactions()\n", + " if implicit:\n", + " interactions = (interactions >= 4).astype(np.float32)\n", + " train, test = train_test_split(interactions)\n", + " train = sp.coo_matrix(train)\n", + " test = sp.coo_matrix(test)\n", + " return train, test" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0x9xMyW6PsK6", + "outputId": "00096a84-7542-4d07-920d-48830410244e" + }, + "source": [ + "%%writefile metrics.py\n", + "\n", + "import numpy as np\n", + "from sklearn.metrics import roc_auc_score\n", + "from torch import multiprocessing as mp\n", + "import torch\n", + "\n", + "def get_row_indices(row, interactions):\n", + " start = interactions.indptr[row]\n", + " end = interactions.indptr[row + 1]\n", + " return interactions.indices[start:end]\n", + "\n", + "\n", + "def auc(model, interactions, num_workers=1):\n", + " aucs = []\n", + " processes = []\n", + " n_users = interactions.shape[0]\n", + " mp_batch = int(np.ceil(n_users / num_workers))\n", + "\n", + " queue = mp.Queue()\n", + " rows = np.arange(n_users)\n", + " np.random.shuffle(rows)\n", + " for rank in range(num_workers):\n", + " start = rank * mp_batch\n", + " end = np.min((start + mp_batch, n_users))\n", + " p = mp.Process(target=batch_auc,\n", + " args=(queue, rows[start:end], interactions, model))\n", + " p.start()\n", + " processes.append(p)\n", + "\n", + " while True:\n", + " is_alive = False\n", + " for p in processes:\n", + " if p.is_alive():\n", + " is_alive = True\n", + " break\n", + " if not is_alive and queue.empty():\n", + " break\n", + "\n", + " while not queue.empty():\n", + " aucs.append(queue.get())\n", + "\n", + " queue.close()\n", + " for p in processes:\n", + " p.join()\n", + " return np.mean(aucs)\n", + "\n", + "\n", + "def batch_auc(queue, rows, interactions, model):\n", + " n_items = interactions.shape[1]\n", + " items = torch.arange(0, n_items).long()\n", + " users_init = torch.ones(n_items).long()\n", + " for row in rows:\n", + " row = int(row)\n", + " users = users_init.fill_(row)\n", + "\n", + " preds = model.predict(users, items)\n", + " actuals = get_row_indices(row, interactions)\n", + "\n", + " if len(actuals) == 0:\n", + " continue\n", + " y_test = np.zeros(n_items)\n", + " y_test[actuals] = 1\n", + " queue.put(roc_auc_score(y_test, preds.data.numpy()))\n", + "\n", + "\n", + "def patk(model, interactions, num_workers=1, k=5):\n", + " patks = []\n", + " processes = []\n", + " n_users = interactions.shape[0]\n", + " mp_batch = int(np.ceil(n_users / num_workers))\n", + "\n", + " queue = mp.Queue()\n", + " rows = np.arange(n_users)\n", + " np.random.shuffle(rows)\n", + " for rank in range(num_workers):\n", + " start = rank * mp_batch\n", + " end = np.min((start + mp_batch, n_users))\n", + " p = mp.Process(target=batch_patk,\n", + " args=(queue, rows[start:end], interactions, model),\n", + " kwargs={'k': k})\n", + " p.start()\n", + " processes.append(p)\n", + "\n", + " while True:\n", + " is_alive = False\n", + " for p in processes:\n", + " if p.is_alive():\n", + " is_alive = True\n", + " break\n", + " if not is_alive and queue.empty():\n", + " break\n", + "\n", + " while not queue.empty():\n", + " patks.append(queue.get())\n", + "\n", + " queue.close()\n", + " for p in processes:\n", + " p.join()\n", + " return np.mean(patks)\n", + "\n", + "\n", + "def batch_patk(queue, rows, interactions, model, k=5):\n", + " n_items = interactions.shape[1]\n", + "\n", + " items = torch.arange(0, n_items).long()\n", + " users_init = torch.ones(n_items).long()\n", + " for row in rows:\n", + " row = int(row)\n", + " users = users_init.fill_(row)\n", + "\n", + " preds = model.predict(users, items)\n", + " actuals = get_row_indices(row, interactions)\n", + "\n", + " if len(actuals) == 0:\n", + " continue\n", + "\n", + " top_k = np.argpartition(-np.squeeze(preds.data.numpy()), k)\n", + " top_k = set(top_k[:k])\n", + " true_pids = set(actuals)\n", + " if true_pids:\n", + " queue.put(len(top_k & true_pids) / float(k))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Writing metrics.py\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "I6IqWKWFhAQh", + "outputId": "3bef0869-34b5-488c-ede7-bf9be08c115f" + }, + "source": [ + "import metrics\n", + "import importlib\n", + "importlib.reload(metrics)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 55 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "UEhEE_GhPsH2" + }, + "source": [ + "class Interactions(data.Dataset):\n", + " \"\"\"\n", + " Hold data in the form of an interactions matrix.\n", + " Typical use-case is like a ratings matrix:\n", + " - Users are the rows\n", + " - Items are the columns\n", + " - Elements of the matrix are the ratings given by a user for an item.\n", + " \"\"\"\n", + "\n", + " def __init__(self, mat):\n", + " self.mat = mat.astype(np.float32).tocoo()\n", + " self.n_users = self.mat.shape[0]\n", + " self.n_items = self.mat.shape[1]\n", + "\n", + " def __getitem__(self, index):\n", + " row = self.mat.row[index]\n", + " col = self.mat.col[index]\n", + " val = self.mat.data[index]\n", + " return (row, col), val\n", + "\n", + " def __len__(self):\n", + " return self.mat.nnz\n", + "\n", + "\n", + "class PairwiseInteractions(data.Dataset):\n", + " \"\"\"\n", + " Sample data from an interactions matrix in a pairwise fashion. The row is\n", + " treated as the main dimension, and the columns are sampled pairwise.\n", + " \"\"\"\n", + "\n", + " def __init__(self, mat):\n", + " self.mat = mat.astype(np.float32).tocoo()\n", + "\n", + " self.n_users = self.mat.shape[0]\n", + " self.n_items = self.mat.shape[1]\n", + "\n", + " self.mat_csr = self.mat.tocsr()\n", + " if not self.mat_csr.has_sorted_indices:\n", + " self.mat_csr.sort_indices()\n", + "\n", + " def __getitem__(self, index):\n", + " row = self.mat.row[index]\n", + " found = False\n", + "\n", + " while not found:\n", + " neg_col = np.random.randint(self.n_items)\n", + " if self.not_rated(row, neg_col, self.mat_csr.indptr,\n", + " self.mat_csr.indices):\n", + " found = True\n", + "\n", + " pos_col = self.mat.col[index]\n", + " val = self.mat.data[index]\n", + "\n", + " return (row, (pos_col, neg_col)), val\n", + "\n", + " def __len__(self):\n", + " return self.mat.nnz\n", + "\n", + " @staticmethod\n", + " def not_rated(row, col, indptr, indices):\n", + " # similar to use of bsearch in lightfm\n", + " start = indptr[row]\n", + " end = indptr[row + 1]\n", + " searched = np.searchsorted(indices[start:end], col, 'right')\n", + " if searched >= (end - start):\n", + " # After the array\n", + " return False\n", + " return col != indices[searched] # Not found\n", + "\n", + " def get_row_indices(self, row):\n", + " start = self.mat_csr.indptr[row]\n", + " end = self.mat_csr.indptr[row + 1]\n", + " return self.mat_csr.indices[start:end]\n", + "\n", + "\n", + "class BaseModule(nn.Module):\n", + " \"\"\"\n", + " Base module for explicit matrix factorization.\n", + " \"\"\"\n", + " \n", + " def __init__(self,\n", + " n_users,\n", + " n_items,\n", + " n_factors=40,\n", + " dropout_p=0,\n", + " sparse=False):\n", + " \"\"\"\n", + "\n", + " Parameters\n", + " ----------\n", + " n_users : int\n", + " Number of users\n", + " n_items : int\n", + " Number of items\n", + " n_factors : int\n", + " Number of latent factors (or embeddings or whatever you want to\n", + " call it).\n", + " dropout_p : float\n", + " p in nn.Dropout module. Probability of dropout.\n", + " sparse : bool\n", + " Whether or not to treat embeddings as sparse. NOTE: cannot use\n", + " weight decay on the optimizer if sparse=True. Also, can only use\n", + " Adagrad.\n", + " \"\"\"\n", + " super(BaseModule, self).__init__()\n", + " self.n_users = n_users\n", + " self.n_items = n_items\n", + " self.n_factors = n_factors\n", + " self.user_biases = nn.Embedding(n_users, 1, sparse=sparse)\n", + " self.item_biases = nn.Embedding(n_items, 1, sparse=sparse)\n", + " self.user_embeddings = nn.Embedding(n_users, n_factors, sparse=sparse)\n", + " self.item_embeddings = nn.Embedding(n_items, n_factors, sparse=sparse)\n", + " \n", + " self.dropout_p = dropout_p\n", + " self.dropout = nn.Dropout(p=self.dropout_p)\n", + "\n", + " self.sparse = sparse\n", + " \n", + " def forward(self, users, items):\n", + " \"\"\"\n", + " Forward pass through the model. For a single user and item, this\n", + " looks like:\n", + "\n", + " user_bias + item_bias + user_embeddings.dot(item_embeddings)\n", + "\n", + " Parameters\n", + " ----------\n", + " users : np.ndarray\n", + " Array of user indices\n", + " items : np.ndarray\n", + " Array of item indices\n", + "\n", + " Returns\n", + " -------\n", + " preds : np.ndarray\n", + " Predicted ratings.\n", + "\n", + " \"\"\"\n", + " ues = self.user_embeddings(users)\n", + " uis = self.item_embeddings(items)\n", + "\n", + " preds = self.user_biases(users)\n", + " preds += self.item_biases(items)\n", + " preds += (self.dropout(ues) * self.dropout(uis)).sum(dim=1, keepdim=True)\n", + "\n", + " return preds.squeeze()\n", + " \n", + " def __call__(self, *args):\n", + " return self.forward(*args)\n", + "\n", + " def predict(self, users, items):\n", + " return self.forward(users, items)\n", + "\n", + "\n", + "def bpr_loss(preds, vals):\n", + " sig = nn.Sigmoid()\n", + " return (1.0 - sig(preds)).pow(2).sum()\n", + "\n", + "\n", + "class BPRModule(nn.Module):\n", + " \n", + " def __init__(self,\n", + " n_users,\n", + " n_items,\n", + " n_factors=40,\n", + " dropout_p=0,\n", + " sparse=False,\n", + " model=BaseModule):\n", + " super(BPRModule, self).__init__()\n", + "\n", + " self.n_users = n_users\n", + " self.n_items = n_items\n", + " self.n_factors = n_factors\n", + " self.dropout_p = dropout_p\n", + " self.sparse = sparse\n", + " self.pred_model = model(\n", + " self.n_users,\n", + " self.n_items,\n", + " n_factors=n_factors,\n", + " dropout_p=dropout_p,\n", + " sparse=sparse\n", + " )\n", + "\n", + " def forward(self, users, items):\n", + " assert isinstance(items, tuple), \\\n", + " 'Must pass in items as (pos_items, neg_items)'\n", + " # Unpack\n", + " (pos_items, neg_items) = items\n", + " pos_preds = self.pred_model(users, pos_items)\n", + " neg_preds = self.pred_model(users, neg_items)\n", + " return pos_preds - neg_preds\n", + "\n", + " def predict(self, users, items):\n", + " return self.pred_model(users, items)\n", + "\n", + "\n", + "class BasePipeline:\n", + " \"\"\"\n", + " Class defining a training pipeline. Instantiates data loaders, model,\n", + " and optimizer. Handles training for multiple epochs and keeping track of\n", + " train and test loss.\n", + " \"\"\"\n", + "\n", + " def __init__(self,\n", + " train,\n", + " test=None,\n", + " model=BaseModule,\n", + " n_factors=40,\n", + " batch_size=32,\n", + " dropout_p=0.02,\n", + " sparse=False,\n", + " lr=0.01,\n", + " weight_decay=0.,\n", + " optimizer=torch.optim.Adam,\n", + " loss_function=nn.MSELoss(reduction='sum'),\n", + " n_epochs=10,\n", + " verbose=False,\n", + " random_seed=None,\n", + " interaction_class=Interactions,\n", + " hogwild=False,\n", + " num_workers=0,\n", + " eval_metrics=None,\n", + " k=5):\n", + " self.train = train\n", + " self.test = test\n", + "\n", + " if hogwild:\n", + " num_loader_workers = 0\n", + " else:\n", + " num_loader_workers = num_workers\n", + " self.train_loader = data.DataLoader(\n", + " interaction_class(train), batch_size=batch_size, shuffle=True,\n", + " num_workers=num_loader_workers)\n", + " if self.test is not None:\n", + " self.test_loader = data.DataLoader(\n", + " interaction_class(test), batch_size=batch_size, shuffle=True,\n", + " num_workers=num_loader_workers)\n", + " self.num_workers = num_workers\n", + " self.n_users = self.train.shape[0]\n", + " self.n_items = self.train.shape[1]\n", + " self.n_factors = n_factors\n", + " self.batch_size = batch_size\n", + " self.dropout_p = dropout_p\n", + " self.lr = lr\n", + " self.weight_decay = weight_decay\n", + " self.loss_function = loss_function\n", + " self.n_epochs = n_epochs\n", + " if sparse:\n", + " assert weight_decay == 0.0\n", + " self.model = model(self.n_users,\n", + " self.n_items,\n", + " n_factors=self.n_factors,\n", + " dropout_p=self.dropout_p,\n", + " sparse=sparse)\n", + " self.optimizer = optimizer(self.model.parameters(),\n", + " lr=self.lr,\n", + " weight_decay=self.weight_decay)\n", + " self.warm_start = False\n", + " self.losses = collections.defaultdict(list)\n", + " self.verbose = verbose\n", + " self.hogwild = hogwild\n", + " if random_seed is not None:\n", + " if self.hogwild:\n", + " random_seed += os.getpid()\n", + " torch.manual_seed(random_seed)\n", + " np.random.seed(random_seed)\n", + "\n", + " if eval_metrics is None:\n", + " eval_metrics = []\n", + " self.eval_metrics = eval_metrics\n", + " self.k = k\n", + "\n", + " def break_grads(self):\n", + " for param in self.model.parameters():\n", + " # Break gradient sharing\n", + " if param.grad is not None:\n", + " param.grad.data = param.grad.data.clone()\n", + "\n", + " def fit(self):\n", + " for epoch in range(1, self.n_epochs + 1):\n", + "\n", + " if self.hogwild:\n", + " self.model.share_memory()\n", + " processes = []\n", + " train_losses = []\n", + " queue = mp.Queue()\n", + " for rank in range(self.num_workers):\n", + " p = mp.Process(target=self._fit_epoch,\n", + " kwargs={'epoch': epoch,\n", + " 'queue': queue})\n", + " p.start()\n", + " processes.append(p)\n", + " for p in processes:\n", + " p.join()\n", + "\n", + " while True:\n", + " is_alive = False\n", + " for p in processes:\n", + " if p.is_alive():\n", + " is_alive = True\n", + " break\n", + " if not is_alive and queue.empty():\n", + " break\n", + "\n", + " while not queue.empty():\n", + " train_losses.append(queue.get())\n", + " queue.close()\n", + " train_loss = np.mean(train_losses)\n", + " else:\n", + " train_loss = self._fit_epoch(epoch)\n", + "\n", + " self.losses['train'].append(train_loss)\n", + " row = 'Epoch: {0:^3} train: {1:^10.5f}'.format(epoch, self.losses['train'][-1])\n", + " if self.test is not None:\n", + " self.losses['test'].append(self._validation_loss())\n", + " row += 'val: {0:^10.5f}'.format(self.losses['test'][-1])\n", + " for metric in self.eval_metrics:\n", + " func = getattr(metrics, metric)\n", + " res = func(self.model, self.test_loader.dataset.mat_csr,\n", + " num_workers=self.num_workers)\n", + " self.losses['eval-{}'.format(metric)].append(res)\n", + " row += 'eval-{0}: {1:^10.5f}'.format(metric, res)\n", + " self.losses['epoch'].append(epoch)\n", + " if self.verbose:\n", + " print(row)\n", + "\n", + " def _fit_epoch(self, epoch=1, queue=None):\n", + " if self.hogwild:\n", + " self.break_grads()\n", + "\n", + " self.model.train()\n", + " total_loss = torch.Tensor([0])\n", + " pbar = tqdm(enumerate(self.train_loader),\n", + " total=len(self.train_loader),\n", + " desc='({0:^3})'.format(epoch))\n", + " for batch_idx, ((row, col), val) in pbar:\n", + " self.optimizer.zero_grad()\n", + "\n", + " row = row.long()\n", + " # TODO: turn this into a collate_fn like the data_loader\n", + " if isinstance(col, list):\n", + " col = tuple(c.long() for c in col)\n", + " else:\n", + " col = col.long()\n", + " val = val.float()\n", + "\n", + " preds = self.model(row, col)\n", + " loss = self.loss_function(preds, val)\n", + " loss.backward()\n", + "\n", + " self.optimizer.step()\n", + "\n", + " total_loss += loss.item()\n", + " batch_loss = loss.item() / row.size()[0]\n", + " pbar.set_postfix(train_loss=batch_loss)\n", + " total_loss /= self.train.nnz\n", + " if queue is not None:\n", + " queue.put(total_loss[0])\n", + " else:\n", + " return total_loss[0]\n", + "\n", + " def _validation_loss(self):\n", + " self.model.eval()\n", + " total_loss = torch.Tensor([0])\n", + " for batch_idx, ((row, col), val) in enumerate(self.test_loader):\n", + " row = row.long()\n", + " if isinstance(col, list):\n", + " col = tuple(c.long() for c in col)\n", + " else:\n", + " col = col.long()\n", + " val = val.float()\n", + "\n", + " preds = self.model(row, col)\n", + " loss = self.loss_function(preds, val)\n", + " total_loss += loss.item()\n", + "\n", + " total_loss /= self.test.nnz\n", + " return total_loss[0]" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "iKeUFiRNPsFY" + }, + "source": [ + "def explicit():\n", + " train, test = get_movielens_train_test_split()\n", + " pipeline = BasePipeline(train, test=test, model=BaseModule,\n", + " n_factors=10, batch_size=1024, dropout_p=0.02,\n", + " lr=0.02, weight_decay=0.1,\n", + " optimizer=torch.optim.Adam, n_epochs=40,\n", + " verbose=True, random_seed=2017)\n", + " pipeline.fit()\n", + "\n", + "\n", + "def implicit():\n", + " train, test = get_movielens_train_test_split(implicit=True)\n", + "\n", + " pipeline = BasePipeline(train, test=test, verbose=True,\n", + " batch_size=1024, num_workers=4,\n", + " n_factors=20, weight_decay=0,\n", + " dropout_p=0., lr=.2, sparse=True,\n", + " optimizer=torch.optim.SGD, n_epochs=40,\n", + " random_seed=2017, loss_function=bpr_loss,\n", + " model=BPRModule,\n", + " interaction_class=PairwiseInteractions,\n", + " eval_metrics=('auc', 'patk'))\n", + " pipeline.fit()\n", + "\n", + "\n", + "def hogwild():\n", + " train, test = get_movielens_train_test_split(implicit=True)\n", + "\n", + " pipeline = BasePipeline(train, test=test, verbose=True,\n", + " batch_size=1024, num_workers=4,\n", + " n_factors=20, weight_decay=0,\n", + " dropout_p=0., lr=.2, sparse=True,\n", + " optimizer=torch.optim.SGD, n_epochs=40,\n", + " random_seed=2017, loss_function=bpr_loss,\n", + " model=BPRModule, hogwild=True,\n", + " interaction_class=PairwiseInteractions,\n", + " eval_metrics=('auc', 'patk'))\n", + " pipeline.fit()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dPQaj1FjPsCo", + "outputId": "42275f83-f4e9-43cc-a105-3ecde82efaa4" + }, + "source": [ + "explicit()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Making data path\n", + "Downloading MovieLens data\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 1 ): 100%|██████████| 89/89 [00:01<00:00, 53.63it/s, train_loss=6.88]\n", + "( 2 ): 7%|▋ | 6/89 [00:00<00:01, 57.03it/s, train_loss=6.06]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 1 train: 14.42120 val: 8.68083 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 2 ): 100%|██████████| 89/89 [00:01<00:00, 63.13it/s, train_loss=2.27]\n", + "( 3 ): 8%|▊ | 7/89 [00:00<00:01, 62.84it/s, train_loss=2.23]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 2 train: 4.15028 val: 3.99969 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 3 ): 100%|██████████| 89/89 [00:01<00:00, 59.57it/s, train_loss=1.67]\n", + "( 4 ): 7%|▋ | 6/89 [00:00<00:01, 59.43it/s, train_loss=1.33]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 3 train: 1.84903 val: 2.41240 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 4 ): 100%|██████████| 89/89 [00:01<00:00, 59.96it/s, train_loss=1.05]\n", + "( 5 ): 8%|▊ | 7/89 [00:00<00:01, 61.59it/s, train_loss=0.982]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 4 train: 1.20266 val: 1.78271 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 5 ): 100%|██████████| 89/89 [00:01<00:00, 57.47it/s, train_loss=0.917]\n", + "( 6 ): 8%|▊ | 7/89 [00:00<00:01, 62.99it/s, train_loss=0.861]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 5 train: 0.98022 val: 1.48147 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 6 ): 100%|██████████| 89/89 [00:01<00:00, 61.39it/s, train_loss=0.9]\n", + "( 7 ): 8%|▊ | 7/89 [00:00<00:01, 65.11it/s, train_loss=0.77] " + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 6 train: 0.88477 val: 1.32482 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 7 ): 100%|██████████| 89/89 [00:01<00:00, 62.83it/s, train_loss=0.806]\n", + "( 8 ): 7%|▋ | 6/89 [00:00<00:01, 54.86it/s, train_loss=0.766]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 7 train: 0.83306 val: 1.22818 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 8 ): 100%|██████████| 89/89 [00:01<00:00, 58.63it/s, train_loss=0.776]\n", + "( 9 ): 3%|▎ | 3/89 [00:00<00:03, 25.32it/s, train_loss=0.722]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 8 train: 0.80015 val: 1.16457 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 9 ): 100%|██████████| 89/89 [00:01<00:00, 59.21it/s, train_loss=0.871]\n", + "(10 ): 2%|▏ | 2/89 [00:00<00:04, 19.07it/s, train_loss=0.708]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 9 train: 0.77529 val: 1.12250 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(10 ): 100%|██████████| 89/89 [00:01<00:00, 60.45it/s, train_loss=0.749]\n", + "(11 ): 2%|▏ | 2/89 [00:00<00:04, 19.87it/s, train_loss=0.735]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 10 train: 0.75322 val: 1.09408 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(11 ): 100%|██████████| 89/89 [00:01<00:00, 60.82it/s, train_loss=0.728]\n", + "(12 ): 8%|▊ | 7/89 [00:00<00:01, 62.74it/s, train_loss=0.655]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 11 train: 0.73431 val: 1.06755 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(12 ): 100%|██████████| 89/89 [00:01<00:00, 64.48it/s, train_loss=0.729]\n", + "(13 ): 8%|▊ | 7/89 [00:00<00:01, 61.52it/s, train_loss=0.706]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 12 train: 0.71816 val: 1.05441 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(13 ): 100%|██████████| 89/89 [00:01<00:00, 63.59it/s, train_loss=0.804]\n", + "(14 ): 7%|▋ | 6/89 [00:00<00:01, 57.44it/s, train_loss=0.658]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 13 train: 0.70331 val: 1.04291 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(14 ): 100%|██████████| 89/89 [00:01<00:00, 62.10it/s, train_loss=0.648]\n", + "(15 ): 7%|▋ | 6/89 [00:00<00:01, 55.63it/s, train_loss=0.662]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 14 train: 0.69230 val: 1.03409 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(15 ): 100%|██████████| 89/89 [00:01<00:00, 59.82it/s, train_loss=0.71]\n", + "(16 ): 8%|▊ | 7/89 [00:00<00:01, 63.50it/s, train_loss=0.648]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 15 train: 0.68174 val: 1.02946 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(16 ): 100%|██████████| 89/89 [00:01<00:00, 63.41it/s, train_loss=0.762]\n", + "(17 ): 8%|▊ | 7/89 [00:00<00:01, 66.62it/s, train_loss=0.6] " + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 16 train: 0.67185 val: 1.02574 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(17 ): 100%|██████████| 89/89 [00:01<00:00, 61.57it/s, train_loss=0.709]\n", + "(18 ): 7%|▋ | 6/89 [00:00<00:01, 59.98it/s, train_loss=0.647]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 17 train: 0.66559 val: 1.01690 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(18 ): 100%|██████████| 89/89 [00:01<00:00, 59.60it/s, train_loss=0.657]\n", + "(19 ): 7%|▋ | 6/89 [00:00<00:01, 58.13it/s, train_loss=0.609]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 18 train: 0.65754 val: 1.01814 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(19 ): 100%|██████████| 89/89 [00:01<00:00, 58.23it/s, train_loss=0.609]\n", + "(20 ): 8%|▊ | 7/89 [00:00<00:01, 64.70it/s, train_loss=0.636]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 19 train: 0.65179 val: 1.01196 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(20 ): 100%|██████████| 89/89 [00:01<00:00, 58.38it/s, train_loss=0.693]\n", + "(21 ): 8%|▊ | 7/89 [00:00<00:01, 68.79it/s, train_loss=0.607]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 20 train: 0.64911 val: 1.00926 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(21 ): 100%|██████████| 89/89 [00:01<00:00, 60.85it/s, train_loss=0.75]\n", + "(22 ): 7%|▋ | 6/89 [00:00<00:01, 52.77it/s, train_loss=0.635]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 21 train: 0.64537 val: 1.01296 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(22 ): 100%|██████████| 89/89 [00:01<00:00, 59.46it/s, train_loss=0.702]\n", + "(23 ): 4%|▍ | 4/89 [00:00<00:02, 39.91it/s, train_loss=0.588]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 22 train: 0.64303 val: 1.00838 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(23 ): 100%|██████████| 89/89 [00:01<00:00, 56.49it/s, train_loss=0.683]\n", + "(24 ): 7%|▋ | 6/89 [00:00<00:01, 59.61it/s, train_loss=0.633]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 23 train: 0.63932 val: 0.99910 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(24 ): 100%|██████████| 89/89 [00:01<00:00, 58.42it/s, train_loss=0.709]\n", + "(25 ): 7%|▋ | 6/89 [00:00<00:01, 52.67it/s, train_loss=0.594]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 24 train: 0.63549 val: 1.01004 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(25 ): 100%|██████████| 89/89 [00:01<00:00, 57.48it/s, train_loss=0.786]\n", + "(26 ): 7%|▋ | 6/89 [00:00<00:01, 58.84it/s, train_loss=0.59] " + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 25 train: 0.63468 val: 1.00146 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(26 ): 100%|██████████| 89/89 [00:01<00:00, 55.84it/s, train_loss=0.64]\n", + "(27 ): 7%|▋ | 6/89 [00:00<00:01, 58.98it/s, train_loss=0.603]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 26 train: 0.63316 val: 1.00257 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(27 ): 100%|██████████| 89/89 [00:01<00:00, 60.23it/s, train_loss=0.682]\n", + "(28 ): 8%|▊ | 7/89 [00:00<00:01, 67.37it/s, train_loss=0.584]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 27 train: 0.63269 val: 1.00099 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(28 ): 100%|██████████| 89/89 [00:01<00:00, 59.51it/s, train_loss=0.721]\n", + "(29 ): 7%|▋ | 6/89 [00:00<00:01, 57.41it/s, train_loss=0.573]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 28 train: 0.63194 val: 0.99549 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(29 ): 100%|██████████| 89/89 [00:01<00:00, 58.52it/s, train_loss=0.759]\n", + "(30 ): 7%|▋ | 6/89 [00:00<00:01, 58.95it/s, train_loss=0.564]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 29 train: 0.63050 val: 1.00029 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(30 ): 100%|██████████| 89/89 [00:01<00:00, 59.03it/s, train_loss=0.718]\n", + "(31 ): 8%|▊ | 7/89 [00:00<00:01, 65.42it/s, train_loss=0.563]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 30 train: 0.63016 val: 0.99232 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(31 ): 100%|██████████| 89/89 [00:01<00:00, 57.36it/s, train_loss=0.699]\n", + "(32 ): 8%|▊ | 7/89 [00:00<00:01, 62.85it/s, train_loss=0.58] " + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 31 train: 0.63022 val: 0.99609 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(32 ): 100%|██████████| 89/89 [00:01<00:00, 56.56it/s, train_loss=0.743]\n", + "(33 ): 7%|▋ | 6/89 [00:00<00:01, 59.53it/s, train_loss=0.576]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 32 train: 0.63043 val: 0.99635 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(33 ): 100%|██████████| 89/89 [00:01<00:00, 57.91it/s, train_loss=0.643]\n", + "(34 ): 8%|▊ | 7/89 [00:00<00:01, 64.98it/s, train_loss=0.625]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 33 train: 0.63210 val: 0.99697 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(34 ): 100%|██████████| 89/89 [00:01<00:00, 58.12it/s, train_loss=0.641]\n", + "(35 ): 6%|▌ | 5/89 [00:00<00:01, 49.84it/s, train_loss=0.546]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 34 train: 0.63177 val: 0.99458 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(35 ): 100%|██████████| 89/89 [00:01<00:00, 54.93it/s, train_loss=0.654]\n", + "(36 ): 7%|▋ | 6/89 [00:00<00:01, 57.96it/s, train_loss=0.543]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 35 train: 0.63137 val: 1.00267 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(36 ): 100%|██████████| 89/89 [00:01<00:00, 58.59it/s, train_loss=0.742]\n", + "(37 ): 7%|▋ | 6/89 [00:00<00:01, 59.93it/s, train_loss=0.553]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 36 train: 0.63002 val: 0.99718 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(37 ): 100%|██████████| 89/89 [00:01<00:00, 58.76it/s, train_loss=0.733]\n", + "(38 ): 7%|▋ | 6/89 [00:00<00:01, 57.61it/s, train_loss=0.56]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 37 train: 0.62959 val: 0.99938 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(38 ): 100%|██████████| 89/89 [00:01<00:00, 59.98it/s, train_loss=0.638]\n", + "(39 ): 8%|▊ | 7/89 [00:00<00:01, 61.75it/s, train_loss=0.599]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 38 train: 0.63083 val: 1.00133 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(39 ): 100%|██████████| 89/89 [00:01<00:00, 61.77it/s, train_loss=0.724]\n", + "(40 ): 8%|▊ | 7/89 [00:00<00:01, 60.35it/s, train_loss=0.573]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 39 train: 0.63185 val: 0.99541 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(40 ): 100%|██████████| 89/89 [00:01<00:00, 61.02it/s, train_loss=0.69]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 40 train: 0.63168 val: 0.99467 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "mtI0DewsPr_0", + "outputId": "6cc428e1-211f-4e7e-8264-e8332ad47e8b" + }, + "source": [ + "implicit()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", + " cpuset_checked))\n", + "( 1 ): 100%|██████████| 46/46 [00:02<00:00, 21.50it/s, train_loss=0.361]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 1 train: 0.42040 val: 0.40008 eval-auc: 0.55278 eval-patk: 0.00776 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 2 ): 100%|██████████| 46/46 [00:02<00:00, 22.72it/s, train_loss=0.298]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 2 train: 0.34066 val: 0.35044 eval-auc: 0.60807 eval-patk: 0.01164 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 3 ): 100%|██████████| 46/46 [00:02<00:00, 22.89it/s, train_loss=0.303]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 3 train: 0.27492 val: 0.31180 eval-auc: 0.65543 eval-patk: 0.01804 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 4 ): 100%|██████████| 46/46 [00:01<00:00, 23.75it/s, train_loss=0.192]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 4 train: 0.22703 val: 0.29160 eval-auc: 0.69006 eval-patk: 0.02694 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 5 ): 100%|██████████| 46/46 [00:02<00:00, 21.58it/s, train_loss=0.17]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 5 train: 0.19465 val: 0.27365 eval-auc: 0.71412 eval-patk: 0.03265 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 6 ): 100%|██████████| 46/46 [00:02<00:00, 22.30it/s, train_loss=0.176]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 6 train: 0.17487 val: 0.25775 eval-auc: 0.73276 eval-patk: 0.03973 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 7 ): 100%|██████████| 46/46 [00:02<00:00, 22.14it/s, train_loss=0.202]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 7 train: 0.16267 val: 0.25430 eval-auc: 0.74666 eval-patk: 0.04201 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 8 ): 100%|██████████| 46/46 [00:02<00:00, 22.22it/s, train_loss=0.17]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 8 train: 0.15176 val: 0.24547 eval-auc: 0.75858 eval-patk: 0.04429 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "( 9 ): 100%|██████████| 46/46 [00:02<00:00, 22.55it/s, train_loss=0.141]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 9 train: 0.14359 val: 0.23771 eval-auc: 0.76822 eval-patk: 0.04589 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(10 ): 100%|██████████| 46/46 [00:01<00:00, 23.32it/s, train_loss=0.151]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 10 train: 0.13715 val: 0.22593 eval-auc: 0.77713 eval-patk: 0.04361 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(11 ): 100%|██████████| 46/46 [00:01<00:00, 23.04it/s, train_loss=0.115]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 11 train: 0.13167 val: 0.22131 eval-auc: 0.78402 eval-patk: 0.04772 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(12 ): 100%|██████████| 46/46 [00:02<00:00, 22.63it/s, train_loss=0.134]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 12 train: 0.12781 val: 0.22118 eval-auc: 0.79055 eval-patk: 0.04749 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(13 ): 100%|██████████| 46/46 [00:01<00:00, 23.33it/s, train_loss=0.128]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 13 train: 0.12185 val: 0.21263 eval-auc: 0.79726 eval-patk: 0.05228 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(14 ): 100%|██████████| 46/46 [00:02<00:00, 22.32it/s, train_loss=0.109]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 14 train: 0.11865 val: 0.20135 eval-auc: 0.80326 eval-patk: 0.04977 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(15 ): 100%|██████████| 46/46 [00:01<00:00, 23.13it/s, train_loss=0.117]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 15 train: 0.11352 val: 0.20501 eval-auc: 0.80805 eval-patk: 0.05434 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(16 ): 100%|██████████| 46/46 [00:01<00:00, 23.17it/s, train_loss=0.113]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 16 train: 0.11156 val: 0.20189 eval-auc: 0.81208 eval-patk: 0.05753 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(17 ): 100%|██████████| 46/46 [00:02<00:00, 22.15it/s, train_loss=0.127]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 17 train: 0.10898 val: 0.19678 eval-auc: 0.81534 eval-patk: 0.05936 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(18 ): 100%|██████████| 46/46 [00:01<00:00, 23.03it/s, train_loss=0.13]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 18 train: 0.10363 val: 0.19250 eval-auc: 0.81967 eval-patk: 0.05890 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(19 ): 100%|██████████| 46/46 [00:02<00:00, 22.78it/s, train_loss=0.121]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 19 train: 0.10260 val: 0.18791 eval-auc: 0.82216 eval-patk: 0.06416 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(20 ): 100%|██████████| 46/46 [00:02<00:00, 22.97it/s, train_loss=0.121]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 20 train: 0.10081 val: 0.18382 eval-auc: 0.82357 eval-patk: 0.06370 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(21 ): 100%|██████████| 46/46 [00:02<00:00, 22.89it/s, train_loss=0.0978]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 21 train: 0.09957 val: 0.18360 eval-auc: 0.82604 eval-patk: 0.06667 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(22 ): 100%|██████████| 46/46 [00:02<00:00, 22.88it/s, train_loss=0.105]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 22 train: 0.09936 val: 0.17989 eval-auc: 0.82805 eval-patk: 0.06667 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(23 ): 100%|██████████| 46/46 [00:01<00:00, 23.03it/s, train_loss=0.102]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 23 train: 0.09896 val: 0.17684 eval-auc: 0.83031 eval-patk: 0.07123 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(24 ): 100%|██████████| 46/46 [00:01<00:00, 23.09it/s, train_loss=0.116]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 24 train: 0.09503 val: 0.18290 eval-auc: 0.83277 eval-patk: 0.06758 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(25 ): 100%|██████████| 46/46 [00:02<00:00, 22.64it/s, train_loss=0.081]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 25 train: 0.09565 val: 0.17506 eval-auc: 0.83462 eval-patk: 0.07511 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(26 ): 100%|██████████| 46/46 [00:02<00:00, 22.48it/s, train_loss=0.102]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 26 train: 0.09337 val: 0.17530 eval-auc: 0.83571 eval-patk: 0.07169 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(27 ): 100%|██████████| 46/46 [00:02<00:00, 21.46it/s, train_loss=0.0837]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 27 train: 0.09035 val: 0.17689 eval-auc: 0.83655 eval-patk: 0.07420 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(28 ): 100%|██████████| 46/46 [00:02<00:00, 20.81it/s, train_loss=0.0846]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 28 train: 0.08635 val: 0.17874 eval-auc: 0.83849 eval-patk: 0.07420 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(29 ): 100%|██████████| 46/46 [00:02<00:00, 21.13it/s, train_loss=0.107]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 29 train: 0.08961 val: 0.17910 eval-auc: 0.83905 eval-patk: 0.07237 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(30 ): 100%|██████████| 46/46 [00:02<00:00, 21.09it/s, train_loss=0.0935]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 30 train: 0.08822 val: 0.17294 eval-auc: 0.84065 eval-patk: 0.07717 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(31 ): 100%|██████████| 46/46 [00:02<00:00, 21.52it/s, train_loss=0.0926]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 31 train: 0.08964 val: 0.16762 eval-auc: 0.84098 eval-patk: 0.07466 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(32 ): 100%|██████████| 46/46 [00:02<00:00, 21.57it/s, train_loss=0.0708]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 32 train: 0.08982 val: 0.16215 eval-auc: 0.84217 eval-patk: 0.07055 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(33 ): 100%|██████████| 46/46 [00:02<00:00, 20.14it/s, train_loss=0.106]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 33 train: 0.08753 val: 0.16941 eval-auc: 0.84282 eval-patk: 0.07352 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(34 ): 100%|██████████| 46/46 [00:02<00:00, 20.73it/s, train_loss=0.0781]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 34 train: 0.08659 val: 0.17334 eval-auc: 0.84284 eval-patk: 0.07489 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(35 ): 100%|██████████| 46/46 [00:02<00:00, 20.66it/s, train_loss=0.0971]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 35 train: 0.08623 val: 0.17476 eval-auc: 0.84393 eval-patk: 0.07443 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(36 ): 100%|██████████| 46/46 [00:02<00:00, 20.77it/s, train_loss=0.0864]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 36 train: 0.08559 val: 0.17291 eval-auc: 0.84470 eval-patk: 0.07397 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(37 ): 100%|██████████| 46/46 [00:02<00:00, 20.11it/s, train_loss=0.0751]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 37 train: 0.08506 val: 0.16872 eval-auc: 0.84690 eval-patk: 0.07648 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(38 ): 100%|██████████| 46/46 [00:02<00:00, 18.27it/s, train_loss=0.0964]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 38 train: 0.08522 val: 0.16541 eval-auc: 0.84715 eval-patk: 0.07991 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(39 ): 100%|██████████| 46/46 [00:02<00:00, 19.55it/s, train_loss=0.0962]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 39 train: 0.08316 val: 0.16021 eval-auc: 0.84812 eval-patk: 0.07991 \n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "(40 ): 100%|██████████| 46/46 [00:02<00:00, 19.17it/s, train_loss=0.0943]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Epoch: 40 train: 0.08459 val: 0.16542 eval-auc: 0.84809 eval-patk: 0.07237 \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tbXR1BvPWXKO" + }, + "source": [ + "## Neural Graph Collaborative Filtering on MovieLens\n", + "> Applying NGCF PyTorch version on Movielens-100k." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sG-h_5yQQEvQ" + }, + "source": [ + "### Libraries" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3QJEhOlDwIR7" + }, + "source": [ + "!pip install -q git+https://github.com/sparsh-ai/recochef" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "08tq9wC8pdD1" + }, + "source": [ + "import os\n", + "import csv \n", + "import argparse\n", + "import numpy as np\n", + "import pandas as pd\n", + "import random as rd\n", + "from time import time\n", + "from pathlib import Path\n", + "import scipy.sparse as sp\n", + "from datetime import datetime\n", + "\n", + "import torch\n", + "from torch import nn\n", + "import torch.nn.functional as F\n", + "\n", + "from recochef.preprocessing.split import chrono_split" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "F_-NBXzHS0_o" + }, + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "use_cuda = torch.cuda.is_available()\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "torch.cuda.set_device(0)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uuM8Q9GlP8RN" + }, + "source": [ + "### Data Loading\n", + "\n", + "The MovieLens 100K data set consists of 100,000 ratings from 1000 users on 1700 movies as described on [their website](https://grouplens.org/datasets/movielens/100k/)." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Q-yPokpipXsY" + }, + "source": [ + "!wget http://files.grouplens.org/datasets/movielens/ml-100k.zip" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "6CLWhSTXpbvW" + }, + "source": [ + "!unzip ml-100k.zip" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "KS9OJY75pgwq", + "outputId": "0c1ec91e-2ef6-4b5a-e613-fa4501e2737f" + }, + "source": [ + "df = pd.read_csv('ml-100k/u.data', sep='\\t', header=None, names=['USERID','ITEMID','RATING','TIMESTAMP'])\n", + "df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
01962423881250949
11863023891717742
2223771878887116
3244512880606923
41663461886397596
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "0 196 242 3 881250949\n", + "1 186 302 3 891717742\n", + "2 22 377 1 878887116\n", + "3 244 51 2 880606923\n", + "4 166 346 1 886397596" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 5 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mo6kTBolQYca" + }, + "source": [ + "### Train/Test Split\n", + "\n", + "We split the data chronologically in 80:20 ratio. Validated the split for user 4." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "XTJpJj2uvpD2" + }, + "source": [ + "df_train, df_test = chrono_split(df, ratio=0.8)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "Dpeu1H5xxXcR", + "outputId": "d94c244d-ad25-47c1-d084-e51f6b015645" + }, + "source": [ + "userid = 4\n", + "\n", + "query = \"USERID==@userid\"\n", + "display(df.query(query))\n", + "display(df_train.query(query))\n", + "display(df_test.query(query))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
125042643892004275
132943035892002352
220443615892002353
252643574892003525
327742604892004275
596043563892003459
1215142945892004409
1389342884892001445
163054505892003526
1893043545892002353
2008242714892001690
2038343005892001445
2451943283892001537
2474342585892001374
2486642103892003374
3531343295892002352
488264114892004520
5120343275892002352
6409143245892002353
6827343595892002352
7105543625892002352
7672243582892004275
8681543605892002352
8889143015892002353
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "1250 4 264 3 892004275\n", + "1329 4 303 5 892002352\n", + "2204 4 361 5 892002353\n", + "2526 4 357 4 892003525\n", + "3277 4 260 4 892004275\n", + "5960 4 356 3 892003459\n", + "12151 4 294 5 892004409\n", + "13893 4 288 4 892001445\n", + "16305 4 50 5 892003526\n", + "18930 4 354 5 892002353\n", + "20082 4 271 4 892001690\n", + "20383 4 300 5 892001445\n", + "24519 4 328 3 892001537\n", + "24743 4 258 5 892001374\n", + "24866 4 210 3 892003374\n", + "35313 4 329 5 892002352\n", + "48826 4 11 4 892004520\n", + "51203 4 327 5 892002352\n", + "64091 4 324 5 892002353\n", + "68273 4 359 5 892002352\n", + "71055 4 362 5 892002352\n", + "76722 4 358 2 892004275\n", + "86815 4 360 5 892002352\n", + "88891 4 301 5 892002353" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
2474342585892001374
1389342884892001445
2038343005892001445
2451943283892001537
2008242714892001690
6827343595892002352
7105543625892002352
132943035892002352
5120343275892002352
3531343295892002352
8681543605892002352
220443615892002353
1893043545892002353
8889143015892002353
6409143245892002353
2486642103892003374
596043563892003459
252643574892003525
163054505892003526
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "24743 4 258 5 892001374\n", + "13893 4 288 4 892001445\n", + "20383 4 300 5 892001445\n", + "24519 4 328 3 892001537\n", + "20082 4 271 4 892001690\n", + "68273 4 359 5 892002352\n", + "71055 4 362 5 892002352\n", + "1329 4 303 5 892002352\n", + "51203 4 327 5 892002352\n", + "35313 4 329 5 892002352\n", + "86815 4 360 5 892002352\n", + "2204 4 361 5 892002353\n", + "18930 4 354 5 892002353\n", + "88891 4 301 5 892002353\n", + "64091 4 324 5 892002353\n", + "24866 4 210 3 892003374\n", + "5960 4 356 3 892003459\n", + "2526 4 357 4 892003525\n", + "16305 4 50 5 892003526" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMIDRATINGTIMESTAMP
7672243582892004275
327742604892004275
125042643892004275
1215142945892004409
488264114892004520
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID RATING TIMESTAMP\n", + "76722 4 358 2 892004275\n", + "3277 4 260 4 892004275\n", + "1250 4 264 3 892004275\n", + "12151 4 294 5 892004409\n", + "48826 4 11 4 892004520" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qny2aVoWQknP" + }, + "source": [ + "### Preprocessing\n", + "\n", + "1. Sort by User ID and Timestamp\n", + "2. Label encode user and item id - in this case, already label encoded starting from 1, so decreasing ids by 1 as a proxy for label encode\n", + "3. Remove Timestamp and Rating column. The reason is that we are training a recall-maximing model where the objective is to correctly retrieve the items that users can interact with. We can select a rating threshold also\n", + "4. Convert Item IDs into list format\n", + "5. Store as a space-seperated txt file" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "1iSOiyCqpmYE" + }, + "source": [ + "def preprocess(data):\n", + " data = data.copy()\n", + " data = data.sort_values(by=['USERID','TIMESTAMP'])\n", + " data['USERID'] = data['USERID'] - 1\n", + " data['ITEMID'] = data['ITEMID'] - 1\n", + " data.drop(['TIMESTAMP','RATING'], axis=1, inplace=True)\n", + " data = data.groupby('USERID')['ITEMID'].apply(list).reset_index(name='ITEMID')\n", + " return data" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "D7ZtrUPp22dO", + "outputId": "56290e63-dcf3-448b-b3a5-ce60bd2c23db" + }, + "source": [ + "preprocess(df_train).head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
USERIDITEMID
00[167, 171, 164, 155, 165, 195, 186, 13, 249, 1...
11[285, 257, 304, 306, 287, 311, 300, 305, 291, ...
22[301, 332, 343, 299, 267, 336, 302, 344, 353, ...
33[257, 287, 299, 327, 270, 358, 361, 302, 326, ...
44[266, 454, 221, 120, 404, 362, 256, 249, 24, 2...
\n", + "
" + ], + "text/plain": [ + " USERID ITEMID\n", + "0 0 [167, 171, 164, 155, 165, 195, 186, 13, 249, 1...\n", + "1 1 [285, 257, 304, 306, 287, 311, 300, 305, 291, ...\n", + "2 2 [301, 332, 343, 299, 267, 336, 302, 344, 353, ...\n", + "3 3 [257, 287, 299, 327, 270, 358, 361, 302, 326, ...\n", + "4 4 [266, 454, 221, 120, 404, 362, 256, 249, 24, 2..." + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 9 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "yDMAhrig1Lde" + }, + "source": [ + "def store(data, target_file='./data/movielens/train.txt'):\n", + " Path(target_file).parent.mkdir(parents=True, exist_ok=True)\n", + " with open(target_file, 'w+') as f:\n", + " writer = csv.writer(f, delimiter=' ')\n", + " for USERID, row in zip(data.USERID.values,data.ITEMID.values):\n", + " row = [USERID] + row\n", + " writer.writerow(row)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "XUIFrKsavRzV" + }, + "source": [ + "store(preprocess(df_train), '/content/data/ml-100k/train.txt')\n", + "store(preprocess(df_test), '/content/data/ml-100k/test.txt')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vq2IwCkJtUTy", + "outputId": "bdd5df6b-213f-4e87-deae-b8f29e42ec87" + }, + "source": [ + "!head /content/data/ml-100k/train.txt" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0 167 171 164 155 165 195 186 13 249 126 180 116 108 0 245 256 247 49 248 252 261 92 223 123 18 122 136 145 6 234 14 244 259 23 263 125 236 12 24 120 250 235 239 117 129 64 189 46 30 27 113 38 51 237 198 182 10 68 160 94 59 82 178 21 97 63 134 162 25 201 88 7 213 181 47 98 159 174 191 179 127 142 184 67 54 203 55 95 80 78 150 211 22 69 83 93 196 190 183 133 206 144 187 185 96 84 35 143 158 16 173 251 104 147 107 146 219 105 242 121 106 103 246 119 44 267 266 258 260 262 9 149 233 91 70 41 175 90 192 216 176 215 193 72 58 132 40 194 217 169 212 156 222 26 226 79 230 66 118 199 3 214 163 1 205 76 52 135 45 39 152 268 253 114 172 210 228 154 202 61 89 218 166 229 34 161 60 264 111 56 48 29 232 130 151 81 140 71 32 157 197 224 112 20 148 87 100 109 102 238 33 28 42 131 209 204 115 124\r\n", + "1 285 257 304 306 287 311 300 305 291 302 268 298 314 295 0 18 296 292 274 256 294 276 286 254 297 289 279 273 275 272 290 277 293 24 278 13 110 9 281 12 236 283 99 126 312 284 301 282 250 310\r\n", + "2 301 332 343 299 267 336 302 344 353 257 287 318 340 351 271 349 352 333 342 338 341 335 298 325 293 306 331 270 244 354 323 348 322 321 334 263 324 337 329 350 346 339 328\r\n", + "3 257 287 299 327 270 358 361 302 326 328 359 360 353 300 323 209 355 356 49\r\n", + "4 266 454 221 120 404 362 256 249 24 20 99 108 368 234 411 406 410 104 367 224 150 0 180 49 405 423 412 78 396 372 230 398 228 225 175 449 182 434 88 1 227 229 226 448 209 430 173 171 143 402 397 390 384 16 371 385 392 395 166 366 89 400 389 41 152 185 455 69 383 109 79 380 363 208 450 381 427 382 429 210 432 238 172 207 203 413 167 153 421 431 422 418 142 416 414 373 28 433 364 365 379 391 386 428 424 213 134 61 374 97 447 184 233 435 199 442 444 446 218 443 378 369 440 144 445 401 240 370 215 65 420 426 377 94 419 101 415 417 98 403\r\n", + "5 285 241 301 268 305 257 339 302 303 320 309 258 267 308 537 260 181 247 407 274 6 296 126 275 99 8 458 123 13 514 14 136 292 533 535 12 116 284 220 474 0 256 110 476 245 507 470 150 297 283 124 409 532 236 457 293 471 459 534 404 531 472 20 475 300 307 63 523 7 513 97 426 164 222 134 78 530 509 177 486 176 135 88 49 526 204 512 480 461 186 191 46 168 173 519 317 483 488 497 142 70 11 478 190 496 479 491 503 511 495 210 468 524 196 481 198 529 55 536 499 473 68 520 203 506 31 179 182 518 489 188 22 193 463 494 510 460 184 165 174 487 485 69 132 528 482 215 521 522 434 192 462 431 492 237 493 58 208 130 21 94 502 527 86 490 316 467 505 155\r\n", + "6 268 677 681 258 680 306 265 285 267 682 299 287 263 679 308 63 173 186 602 514 175 179 85 366 264 227 522 434 617 185 418 215 446 529 177 642 31 650 171 649 181 100 473 172 615 428 97 233 487 49 92 525 196 88 99 481 495 513 21 611 203 610 652 658 96 143 483 190 402 656 131 170 180 633 7 494 222 95 22 645 654 635 55 8 490 195 435 81 200 498 655 632 167 422 603 134 272 430 67 204 384 165 614 660 510 182 214 155 607 647 643 197 212 670 68 126 189 43 595 355 542 236 526 512 3 284 135 163 497 237 151 484 592 193 482 612 91 478 491 191 317 392 156 381 583 479 509 420 496 202 429 486 590 6 426 662 152 207 501 567 587 631 78 460 178 629 504 480 27 506 130 228 213 503 433 657 10 588 24 646 160 613 549 469 628 206 210 98 69 626 601 194 528 80 555 673 527 651 70 26 46 547 608 150 536 187 508 216 667 618 274 518 431 627 606 634 9 470 176 120 401 605 209 403 674 201 50 630 89 621 377 211 659 415 454 609 548 153 139 520 678 464 124 462 132 419 125 648 442 543 669 404 604 76 378 229 471 565 117 500 162 161 545 140 580 488 199 636 639 505 644 38 225 591 638 280 383 546 600 596 672 364 231 594 597 51 28 572 447 451 598 90 105 623 619 450 570 586 502 663 71 240 379 561 141 577 599 388 593 77 622 440 400 395 443 571 398 79 414 664 563 676\r\n", + "7 257 293 300 258 335 259 687 242 357 456 340 686 337 688 650 171 186 126 49 384 88 21 55 189 181 180 95 173 510 509 567 176 10 434 175 182 402 143 54 227 78 272 209 6 194 228 685\r\n", + "8 339 241 478 520 401 506 614 526 689 275 293 6 370 49 384 5 297 200\r\n", + "9 301 285 268 288 318 244 333 332 653 526 429 55 512 662 63 31 173 126 492 193 152 557 58 185 517 701 610 628 417 473 384 692 602 706 155 504 655 587 485 663 22 11 403 530 685 370 81 222 123 474 49 708 190 272 609 487 220 705 174 274 696 10 177 21 710 700 167 204 650 184 605 181 233 15 510 697 0 479 196 460 159 115 477 134 178 156 8 508 495 197 601 47 194 210 651 175 3 98 133 68 136 284 356 154 462 97 434 501 496 217 691 199 481 482 215 179 273 163 169 497 69 446 461 99 469 275 132 466 654 483 588 191 478 413 128 202 704 12 198 703 160 694 518 702 656 520 603\r\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "E7YYuu2XuVQa", + "outputId": "fd6fc81c-8ee7-4a34-cb7f-ae5c9ffb27d6" + }, + "source": [ + "!head /content/data/ml-100k/test.txt" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0 207 2 11 57 200 137 65 36 37 139 240 75 225 77 62 231 138 141 74 50 53 43 86 99 8 227 153 85 168 15 177 221 257 265 254 271 270 19 128 220 243 5 17 269 208 31 188 241 170 110 4 255 101 73\r\n", + "1 49 241 271 309 303 299 288 315 307 308 313 280\r\n", + "2 320 327 326 345 347 259 330 317 316 319 180\r\n", + "3 357 259 263 293 10\r\n", + "4 138 388 453 68 161 232 242 258 451 439 437 436 438 188 168 407 100 425 376 62 399 93 408 193 162 393 375 39 409 23 441 387 394 452 456\r\n", + "5 516 80 133 498 194 466 418 131 504 207 356 465 199 172 187 212 273 422 169 525 484 508 515 501 201 469 366 185 500 153 477 202 167 424 18 85 152 27 517 538 464 271\r\n", + "6 675 61 144 551 616 560 569 553 585 540 52 138 589 448 544 558 333 293 259 624 417 541 557 218 671 439 568 562 550 564 566 668 445 637 556 579 665 641 226 582 53 573 449 142 416 625 620 559 230 575 390 539 427 174 72 584 574 576 385 578 519 386 316 661 552 554 30 133 323 257 11 192 640 184 198 356 666 653 432 581 340\r\n", + "7 549 81 187 221 430 683 517 226 240 232 565 684\r\n", + "8 690 285 482 486\r\n", + "9 143 503 59 616 709 431 494 92 509 524 488 229 529 161 6 237 614 695 581 282 699 693 366 84 39 419 711 707 528 182 698 131 32 498 320 293 339\r\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "C-f9mzEf4Ow6" + }, + "source": [ + "Path('/content/results').mkdir(parents=True, exist_ok=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Bxz7aV0Sws1S" + }, + "source": [ + "def parse_args():\n", + " parser = argparse.ArgumentParser(description=\"Run NGCF.\")\n", + " parser.add_argument('--data_dir', type=str,\n", + " default='./data/',\n", + " help='Input data path.')\n", + " parser.add_argument('--dataset', type=str, default='ml-100k',\n", + " help='Dataset name: Amazond-book, Gowella, ml-100k')\n", + " parser.add_argument('--results_dir', type=str, default='results',\n", + " help='Store model to path.')\n", + " parser.add_argument('--n_epochs', type=int, default=400,\n", + " help='Number of epoch.')\n", + " parser.add_argument('--reg', type=float, default=1e-5,\n", + " help='l2 reg.')\n", + " parser.add_argument('--lr', type=float, default=0.0001,\n", + " help='Learning rate.')\n", + " parser.add_argument('--emb_dim', type=int, default=64,\n", + " help='number of embeddings.')\n", + " parser.add_argument('--layers', type=str, default='[64,64]',\n", + " help='Output sizes of every layer')\n", + " parser.add_argument('--batch_size', type=int, default=512,\n", + " help='Batch size.')\n", + " parser.add_argument('--node_dropout', type=float, default=0.,\n", + " help='Graph Node dropout.')\n", + " parser.add_argument('--mess_dropout', type=float, default=0.1,\n", + " help='Message dropout.')\n", + " parser.add_argument('--k', type=str, default=20,\n", + " help='k order of metric evaluation (e.g. NDCG@k)')\n", + " parser.add_argument('--eval_N', type=int, default=5,\n", + " help='Evaluate every N epochs')\n", + " parser.add_argument('--save_results', type=int, default=1,\n", + " help='Save model and results')\n", + "\n", + " return parser.parse_args(args={})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "twi1ZIucR0ga" + }, + "source": [ + "### Helper Functions\n", + "\n", + "- early_stopping()\n", + "- train()\n", + "- split_matrix()\n", + "- ndcg_k()\n", + "- eval_model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aCShFbsCTPzw" + }, + "source": [ + "#### Early Stopping\n", + "Premature stopping is applied if *recall@20* on the test set does not increase for 5 successive epochs." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "tHVTudWxTVZo" + }, + "source": [ + "def early_stopping(log_value, best_value, stopping_step, flag_step, expected_order='asc'):\n", + " \"\"\"\n", + " Check if early_stopping is needed\n", + " Function copied from original code\n", + " \"\"\"\n", + " assert expected_order in ['asc', 'des']\n", + " if (expected_order == 'asc' and log_value >= best_value) or (expected_order == 'des' and log_value <= best_value):\n", + " stopping_step = 0\n", + " best_value = log_value\n", + " else:\n", + " stopping_step += 1\n", + "\n", + " if stopping_step >= flag_step:\n", + " print(\"Early stopping at step: {} log:{}\".format(flag_step, log_value))\n", + " should_stop = True\n", + " else:\n", + " should_stop = False\n", + "\n", + " return best_value, stopping_step, should_stop" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "6JEG5Jlpw3Nw" + }, + "source": [ + "def train(model, data_generator, optimizer):\n", + " \"\"\"\n", + " Train the model PyTorch style\n", + " Arguments:\n", + " ---------\n", + " model: PyTorch model\n", + " data_generator: Data object\n", + " optimizer: PyTorch optimizer\n", + " \"\"\"\n", + " model.train()\n", + " n_batch = data_generator.n_train // data_generator.batch_size + 1\n", + " running_loss=0\n", + " for _ in range(n_batch):\n", + " u, i, j = data_generator.sample()\n", + " optimizer.zero_grad()\n", + " loss = model(u,i,j)\n", + " loss.backward()\n", + " optimizer.step()\n", + " running_loss += loss.item()\n", + " return running_loss\n", + "\n", + "def split_matrix(X, n_splits=100):\n", + " \"\"\"\n", + " Split a matrix/Tensor into n_folds (for the user embeddings and the R matrices)\n", + " Arguments:\n", + " ---------\n", + " X: matrix to be split\n", + " n_folds: number of folds\n", + " Returns:\n", + " -------\n", + " splits: split matrices\n", + " \"\"\"\n", + " splits = []\n", + " chunk_size = X.shape[0] // n_splits\n", + " for i in range(n_splits):\n", + " start = i * chunk_size\n", + " end = X.shape[0] if i == n_splits - 1 else (i + 1) * chunk_size\n", + " splits.append(X[start:end])\n", + " return splits\n", + "\n", + "def compute_ndcg_k(pred_items, test_items, test_indices, k):\n", + " \"\"\"\n", + " Compute NDCG@k\n", + " \n", + " Arguments:\n", + " ---------\n", + " pred_items: binary tensor with 1s in those locations corresponding to the predicted item interactions\n", + " test_items: binary tensor with 1s in locations corresponding to the real test interactions\n", + " test_indices: tensor with the location of the top-k predicted items\n", + " k: k'th-order \n", + " Returns:\n", + " -------\n", + " NDCG@k\n", + " \"\"\"\n", + " r = (test_items * pred_items).gather(1, test_indices)\n", + " f = torch.from_numpy(np.log2(np.arange(2, k+2))).float().cuda()\n", + " dcg = (r[:, :k]/f).sum(1)\n", + " dcg_max = (torch.sort(r, dim=1, descending=True)[0][:, :k]/f).sum(1)\n", + " ndcg = dcg/dcg_max\n", + " ndcg[torch.isnan(ndcg)] = 0\n", + " return ndcg" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sx-Vzl2vTeWN" + }, + "source": [ + "#### Eval Model\n", + "\n", + "At every N epoch, the model is evaluated on the test set. From this evaluation, we compute the recall and normal discounted cumulative gain (ndcg) at the top-20 predictions. It is important to note that in order to evaluate the model on the test set we have to ‘unpack’ the sparse matrix (torch.sparse.todense()), and thus load a bunch of ‘zeros’ on memory. In order to prevent memory overload, we split the sparse matrices into 100 chunks, unpack the sparse chunks one by one, compute the metrics we need, and compute the mean value of all chunks." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "1dysqVKGTjm6" + }, + "source": [ + "def eval_model(u_emb, i_emb, Rtr, Rte, k):\n", + " \"\"\"\n", + " Evaluate the model\n", + " \n", + " Arguments:\n", + " ---------\n", + " u_emb: User embeddings\n", + " i_emb: Item embeddings\n", + " Rtr: Sparse matrix with the training interactions\n", + " Rte: Sparse matrix with the testing interactions\n", + " k : kth-order for metrics\n", + " \n", + " Returns:\n", + " --------\n", + " result: Dictionary with lists correponding to the metrics at order k for k in Ks\n", + " \"\"\"\n", + " # split matrices\n", + " ue_splits = split_matrix(u_emb)\n", + " tr_splits = split_matrix(Rtr)\n", + " te_splits = split_matrix(Rte)\n", + "\n", + " recall_k, ndcg_k= [], []\n", + " # compute results for split matrices\n", + " for ue_f, tr_f, te_f in zip(ue_splits, tr_splits, te_splits):\n", + "\n", + " scores = torch.mm(ue_f, i_emb.t())\n", + "\n", + " test_items = torch.from_numpy(te_f.todense()).float().cuda()\n", + " non_train_items = torch.from_numpy(1-(tr_f.todense())).float().cuda()\n", + " scores = scores * non_train_items\n", + "\n", + " _, test_indices = torch.topk(scores, dim=1, k=k)\n", + "\n", + " # If you want to use a as the index in dim1 for t, this code should work:\n", + " #t[torch.arange(t.size(0)), a]\n", + "\n", + " pred_items = torch.zeros_like(scores).float()\n", + " # pred_items.scatter_(dim=1,index=test_indices,src=torch.tensor(1.0).cuda())\n", + " pred_items.scatter_(dim=1,index=test_indices,src=torch.ones_like(test_indices, dtype=torch.float).cuda())\n", + "\n", + " topk_preds = torch.zeros_like(scores).float()\n", + " # topk_preds.scatter_(dim=1,index=test_indices[:, :k],src=torch.tensor(1.0))\n", + " _idx = test_indices[:, :k]\n", + " topk_preds.scatter_(dim=1,index=_idx,src=torch.ones_like(_idx, dtype=torch.float))\n", + "\n", + " TP = (test_items * topk_preds).sum(1)\n", + " rec = TP/test_items.sum(1)\n", + " ndcg = compute_ndcg_k(pred_items, test_items, test_indices, k)\n", + "\n", + " recall_k.append(rec)\n", + " ndcg_k.append(ndcg)\n", + "\n", + " return torch.cat(recall_k).mean(), torch.cat(ndcg_k).mean()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mvvKJOlpSLn4" + }, + "source": [ + "### Dataset Class" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AzoIqHHuUADD" + }, + "source": [ + "#### Laplacian matrix\n", + "\n", + "The components of the Laplacian matrix are as follows,\n", + "\n", + "- **D**: a diagonal degree matrix, where D{t,t} is |N{t}|, which is the amount of first-hop neighbors for either item or user t,\n", + "- **R**: the user-item interaction matrix,\n", + "- **0**: an all-zero matrix,\n", + "- **A**: the adjacency matrix,\n", + "\n", + "#### Interaction and Adjacency Matrix\n", + "\n", + "We create the sparse interaction matrix R, the adjacency matrix A, the degree matrix D, and the Laplacian matrix L, using the SciPy library. The adjacency matrix A is then transferred onto PyTorch tensor objects." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "s0w9GTdKw7Vj" + }, + "source": [ + "class Data(object):\n", + " def __init__(self, path, batch_size):\n", + " self.path = path\n", + " self.batch_size = batch_size\n", + "\n", + " train_file = path + '/train.txt'\n", + " test_file = path + '/test.txt'\n", + "\n", + " #get number of users and items\n", + " self.n_users, self.n_items = 0, 0\n", + " self.n_train, self.n_test = 0, 0\n", + " self.neg_pools = {}\n", + "\n", + " self.exist_users = []\n", + "\n", + " # search train_file for max user_id/item_id\n", + " with open(train_file) as f:\n", + " for l in f.readlines():\n", + " if len(l) > 0:\n", + " l = l.strip('\\n').split(' ')\n", + " items = [int(i) for i in l[1:]]\n", + " # first element is the user_id, rest are items\n", + " uid = int(l[0])\n", + " self.exist_users.append(uid)\n", + " # item/user with highest number is number of items/users\n", + " self.n_items = max(self.n_items, max(items))\n", + " self.n_users = max(self.n_users, uid)\n", + " # number of interactions\n", + " self.n_train += len(items)\n", + "\n", + " # search test_file for max item_id\n", + " with open(test_file) as f:\n", + " for l in f.readlines():\n", + " if len(l) > 0:\n", + " l = l.strip('\\n')\n", + " try:\n", + " items = [int(i) for i in l.split(' ')[1:]]\n", + " except Exception:\n", + " continue\n", + " if not items:\n", + " print(\"empyt test exists\")\n", + " pass\n", + " else:\n", + " self.n_items = max(self.n_items, max(items))\n", + " self.n_test += len(items)\n", + " # adjust counters: user_id/item_id starts at 0\n", + " self.n_items += 1\n", + " self.n_users += 1\n", + "\n", + " self.print_statistics()\n", + "\n", + " # create interactions/ratings matrix 'R' # dok = dictionary of keys\n", + " print('Creating interaction matrices R_train and R_test...')\n", + " t1 = time()\n", + " self.R_train = sp.dok_matrix((self.n_users, self.n_items), dtype=np.float32) \n", + " self.R_test = sp.dok_matrix((self.n_users, self.n_items), dtype=np.float32)\n", + "\n", + " self.train_items, self.test_set = {}, {}\n", + " with open(train_file) as f_train:\n", + " with open(test_file) as f_test:\n", + " for l in f_train.readlines():\n", + " if len(l) == 0: break\n", + " l = l.strip('\\n')\n", + " items = [int(i) for i in l.split(' ')]\n", + " uid, train_items = items[0], items[1:]\n", + " # enter 1 if user interacted with item\n", + " for i in train_items:\n", + " self.R_train[uid, i] = 1.\n", + " self.train_items[uid] = train_items\n", + "\n", + " for l in f_test.readlines():\n", + " if len(l) == 0: break\n", + " l = l.strip('\\n')\n", + " try:\n", + " items = [int(i) for i in l.split(' ')]\n", + " except Exception:\n", + " continue\n", + " uid, test_items = items[0], items[1:]\n", + " for i in test_items:\n", + " self.R_test[uid, i] = 1.0\n", + " self.test_set[uid] = test_items\n", + " print('Complete. Interaction matrices R_train and R_test created in', time() - t1, 'sec')\n", + "\n", + " # if exist, get adjacency matrix\n", + " def get_adj_mat(self):\n", + " try:\n", + " t1 = time()\n", + " adj_mat = sp.load_npz(self.path + '/s_adj_mat.npz')\n", + " print('Loaded adjacency-matrix (shape:', adj_mat.shape,') in', time() - t1, 'sec.')\n", + "\n", + " except Exception:\n", + " print('Creating adjacency-matrix...')\n", + " adj_mat = self.create_adj_mat()\n", + " sp.save_npz(self.path + '/s_adj_mat.npz', adj_mat)\n", + " return adj_mat\n", + " \n", + " # create adjancency matrix\n", + " def create_adj_mat(self):\n", + " t1 = time()\n", + " \n", + " adj_mat = sp.dok_matrix((self.n_users + self.n_items, self.n_users + self.n_items), dtype=np.float32)\n", + " adj_mat = adj_mat.tolil()\n", + " R = self.R_train.tolil() # to list of lists\n", + "\n", + " adj_mat[:self.n_users, self.n_users:] = R\n", + " adj_mat[self.n_users:, :self.n_users] = R.T\n", + " adj_mat = adj_mat.todok()\n", + " print('Complete. Adjacency-matrix created in', adj_mat.shape, time() - t1, 'sec.')\n", + "\n", + " t2 = time()\n", + "\n", + " # normalize adjacency matrix\n", + " def normalized_adj_single(adj):\n", + " rowsum = np.array(adj.sum(1))\n", + "\n", + " d_inv = np.power(rowsum, -.5).flatten()\n", + " d_inv[np.isinf(d_inv)] = 0.\n", + " d_mat_inv = sp.diags(d_inv)\n", + "\n", + " norm_adj = d_mat_inv.dot(adj).dot(d_mat_inv)\n", + " return norm_adj.tocoo()\n", + "\n", + " print('Transforming adjacency-matrix to NGCF-adjacency matrix...')\n", + " ngcf_adj_mat = normalized_adj_single(adj_mat) + sp.eye(adj_mat.shape[0])\n", + "\n", + " print('Complete. Transformed adjacency-matrix to NGCF-adjacency matrix in', time() - t2, 'sec.')\n", + " return ngcf_adj_mat.tocsr()\n", + "\n", + " # create collections of N items that users never interacted with\n", + " def negative_pool(self):\n", + " t1 = time()\n", + " for u in self.train_items.keys():\n", + " neg_items = list(set(range(self.n_items)) - set(self.train_items[u]))\n", + " pools = [rd.choice(neg_items) for _ in range(100)]\n", + " self.neg_pools[u] = pools\n", + " print('refresh negative pools', time() - t1)\n", + "\n", + " # sample data for mini-batches\n", + " def sample(self):\n", + " if self.batch_size <= self.n_users:\n", + " users = rd.sample(self.exist_users, self.batch_size)\n", + " else:\n", + " users = [rd.choice(self.exist_users) for _ in range(self.batch_size)]\n", + "\n", + " def sample_pos_items_for_u(u, num):\n", + " pos_items = self.train_items[u]\n", + " n_pos_items = len(pos_items)\n", + " pos_batch = []\n", + " while True:\n", + " if len(pos_batch) == num: break\n", + " pos_id = np.random.randint(low=0, high=n_pos_items, size=1)[0]\n", + " pos_i_id = pos_items[pos_id]\n", + "\n", + " if pos_i_id not in pos_batch:\n", + " pos_batch.append(pos_i_id)\n", + " return pos_batch\n", + "\n", + " def sample_neg_items_for_u(u, num):\n", + " neg_items = []\n", + " while True:\n", + " if len(neg_items) == num: break\n", + " neg_id = np.random.randint(low=0, high=self.n_items,size=1)[0]\n", + " if neg_id not in self.train_items[u] and neg_id not in neg_items:\n", + " neg_items.append(neg_id)\n", + " return neg_items\n", + "\n", + " def sample_neg_items_for_u_from_pools(u, num):\n", + " neg_items = list(set(self.neg_pools[u]) - set(self.train_items[u]))\n", + " return rd.sample(neg_items, num)\n", + "\n", + " pos_items, neg_items = [], []\n", + " for u in users:\n", + " pos_items += sample_pos_items_for_u(u, 1)\n", + " neg_items += sample_neg_items_for_u(u, 1)\n", + "\n", + " return users, pos_items, neg_items\n", + "\n", + " def get_num_users_items(self):\n", + " return self.n_users, self.n_items\n", + "\n", + " def print_statistics(self):\n", + " print('n_users=%d, n_items=%d' % (self.n_users, self.n_items))\n", + " print('n_interactions=%d' % (self.n_train + self.n_test))\n", + " print('n_train=%d, n_test=%d, sparsity=%.5f' % (self.n_train, self.n_test, (self.n_train + self.n_test)/(self.n_users * self.n_items)))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J2RxhIxmSYvl" + }, + "source": [ + "### NGCF Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P4vz1IOwTvED" + }, + "source": [ + "#### Weight initialization\n", + "\n", + "We then create tensors for the user embeddings and item embeddings with the proper dimensions. The weights are initialized using [Xavier uniform initialization](https://pytorch.org/docs/stable/nn.init.html).\n", + "\n", + "For each layer, the weight matrices and corresponding biases are initialized using the same procedure." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wo9vgNvJUWvR" + }, + "source": [ + "#### Embedding Layer\n", + "\n", + "The initial user and item embeddings are concatenated in an embedding lookup table as shown in the figure below. This embedding table is initialized using the user and item embeddings and will be optimized in an end-to-end fashion by the network." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ntRUCJGMUeNf" + }, + "source": [ + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASEAAABICAYAAAC5kUM9AAAgAElEQVR4Ae2daVhUV7rvz5f74T733A99zj3dT/p0x3S0p6QTTxJN2k6cYsc4xChx1hijxiHGeR4jzjOKAwqIoOIMIiAKIiLzXMxTMRbFUExFUUVBjft3n12AIuJAGoLEzfPUQ9Xae7/rXb937f9ea+219/o3pD+JgERAItCDBP6tB/OWspYISAQkAkgiJFUCiYBEoEcJSCLUo/ilzCUCEgFJhKQ6IBGQCPQoAUmEehS/lLlEQCIgiZBUByQCEoEeJSCJUI/ilzKXCEgEJBGS6oBEQCLQowQkEepR/FLmEgGJgCRCUh2QCEgEepSAJEI9il/KXCIgEZBESKoDEgGJQI8SkESoR/FLmUsEJAKSCEl1QCIgEehRApII9Sh+KXOJgERAEiGpDkgEJAI9SuAXK0KNufe45O7CqVOncLkWhUJrwNKjqKXMJQJdRECoIyPoEmdPn+D4cSe84srRGoQuMv7zm/mZRchCScRFju/ZyqaNG9mwYUObz2ZOBBZS12TtAgoCNV7LGPb5fA55R5BRUkOj2UrvDVMXIJFM/IIImNBVK8mJvcLG8R/xtWM8Sk3vvcT+zCIk0FirIGjbaN7u898MXX2Fe1EyZDF3cPp+BKM236ao1tAFYmGl6sI3/M+Uw0QV1GISJPn5BZ2BUlFsBAQsBhVXFw5g8qFoitXmXsvlZxYhkZOZ9MNj+MOv/p0hOxNQ1pmwmBqpTjrGUns/FNX6LhOhAXPOkFWh6wJ7vTa+kuO/aAJNBK7+mGkOkgh1MswWMh2+4A+/+r8M3S2jrN5MVbQ/IVmFZBepMZq7ojvW3BIaOOcM2S8qQnoFMT5nObpvHw6ufsjKGjB1hSudpNM1u+tRxN7E/eg+9jm44isrQ2d8iQujLyH2pnsLe1+SSnW8zO52FCN9SRy+7o7s3+eAq28SpVoj3U+8icA1kgh1FI/npLWK0P/hrck7OHLKma3TZ7A3XIna2HG3yVKVSfjd2/j7+eHX9nM7jkKdgScbop0TIaEugTPLJjJ1zRn8A1xZMXYQw+eeIq5C14Ht5xSvpzcLdSScWc6kqWtw9buF68ov+MfwOTjFlqM19bRzT+YvsndbMZlpa1zwu+XKyi8+Zvick8SUa7G5K9SSGXqPpJJ6ml7KYQ+BusSzrJwyjdXOvvi7rmLcx8OZcyKa0vpm4II6i9B7SSjqm7r45ogkQk/WqBdKaRWhf+e9+c5c9fPn0MxJbL+voPYpItSU/4Cr587gfPo0p9t8nD1Dye8wsJ0RITM5Z2cz8I13mLT7An53b2L/xRv812tjOJRYhrbLKr6AWuaFs1sgWbX6x8VNUCPzcsYtMIta/eOSKqhleDm7EZhVS7tNHdI257gz56M/8O7EXZz3DeLm9nG8+evXGH0wntL6LitMh3l3PtFMjvtcPnrzXSbuPIdv0E22j3uTX782moNxSmzuWvLxd9jHxdgKNJ29AySoSfZ2wS0wk5qGtlwF1MneuLgFklnT8EQsHh3zAqptzsFj3iD6vjuRHR43Cbq5gy/7/obfjtpPbInGJjqWglsc2X+RmHINnS3Cs5lKIvRsPk/d+kiEBttHkVdeR+E9b+7laZ56klkbNVRXqqioqHj8UyVeHTu669UJEbIquTD7bX7z6w+Zt/8kbp4XcD68k207zxJWWo/BakKjSCejWIPBLIBVTV5mIeomUyeb2xbk5xYxetxGfApqaGzb6LPIObdoNOM2+lBQ0/jYGJZFfo5Fo8ex0aeAmscO6giwlVLPb3nntV/z4dx9nHTz5ILzYXba78QtTEl9kxWTRkF6ZjF1BhMCVtT5mRSpG3um62ktxXPOO7z2m4HM3XsCtwvN7O13uvFAqaHJrEOVl0Cw/30yVA2d76JZ5JxfPJYvN3qTV92WqwX5+cWM/XIj3nnVT8Si9Rh51fPHJ62lF5nb/7f8ZuAc9hw/w4ULzhzeac9OtweUaJow61TkJQbjfz8Dla6ru2iSCHV0FrxAWqsItY4JWbE0atEZm8gODCBRpcPQrjOtT77EznXL+X7RIha1+Xy//gzRKi1PNqA6IUKWPE7a9eX//WoIW++mkl9eSWVlJZVVdehNGrJDLnNgwWi+PRZnG78y557l+0XHiFLWdZDvs4ovoK/IJS2tgOrGdgIm6KnITSOtoJrGdgNRgr6C3LQ0CqpfRCgs5DlN5I//9SsGbwkiJb+8uSyVVdTpjWiyQ7h8cCFjvj1GbKkGkzkX9x++51hkCXVPQnxWYbpmmyUPp0l/5L9+NZgtgSnktWUvTqkwqUg5u5yxM/YTolB3vhUhNKKSp5FeUNWOq0CjSk5aegFVT8Tiacd0XGRL/ikm//nX/Mcnm7kty6NcrDuVlVTV6TFbBUyqVNxXfMHM/cFddOe3rR+SCLWl0YnvFjIcxrYZmG5WHGtVMNumreZKfhUNbVsJgKW+FHlWBmlpaY9/shSoDeYOWiSdECFBS+imwbz+q9cY/uNt5GoDVksNyQ/iUGh1aCqC2frZxyzyzKRSb0bpOZdPF54jraIBq9BASYKMwoam5vELkYK5gqRb1wlIVKI1dF/3x6xK4tb1ABJLtDzKRkAbupkhff6D14ZtISCnFoPVQk1yGHEKLVpNBfd+HMkniy6QXtGAufQi8/65CI/UcnRiGAxKQo+vYpH9NdKqmrsp5qo0Lu3bgZNPDIU64zPHNMwqWQc+PaNqCFoebBnKG//xGsO23CKntpl9Slg8Cm0jZque5MPjGTT3DLIyHRYE9MpEkgt1ND7sKZlRyQLwCkigpL4bJ6SaVcgCvAhIUFD/CDiC9gFbh/+B/3xtKJv9sqlpsmKpSSEsXkG93oxVn4LDhI+Z6yoOVjfXB0GdzbWdi9jokUh5y7gR+kwubV3L3nP3yalrQmx0P//vSRFqUsYT/CCNcq2hg/Pi+RZ7Yo+f+Ra9hZJIT7aO/yO/+t//i//+eCaLlq5izeplzB33AW8MXIF/UU3nr3hPkOuECCGgzbnBjxPfp2/f9/nn5DksXLiY9c5hKOsNmEvO8c2grzmRVIZGncjRyX9nylFxfEVLScJ17L9Zy9XCR8JpVfmy5rMP6D96O8GKzraWnijIUxKsqHzXMvKD/ozefpdStfHhfoI2Bx/7SXzQty/v/XMycxYsZPF6Zx6UiN3JEs7P/pivjydQWqcm0XEKg6YeJU6paR4XsTaQcHg2Qz4azKLzMlQNFoSmbFx2OxKWVYL2mRM+m336fEB/RtvfRak2PPTp6V8EtLk+2E/+gH793mPE5DkssLEPRaExYLWWcXn+YGY4xlKiMdsEyGvHt6y7kodK13KWWivxWzeKAf1HYx9UhLprB10eum6t9GPdqIH8z6htBBbWPqqjgpbcm9uZMqAf/d4bwaRvF7Bg8XpOhxbbJt5ay66wYMgMjkYrqGsdlmqIwnHqQN6Z5oSsTIsVPTlBB/j673bsCcmkwtDREMNDV9p8aS9CZjJd5jDi0yVcyFI9cTFvc+BL9fVnFiFxsmIpeWnxxERHEZuUSnpGJpmZ6aQkxhKTUkSdwfzYmMhPo9UsQuI8ocwXuUVvaUCVn0b0XT9u3n5AnCwNeXk9RouALnA1g8bYE5KfTbS3PXZ/G4n9nTRKGo00qNM4NusHzudVoG05JwRjFTmRV9iw7BARpW0q608ryFOOEjBW5RB1ZSPLDkVQUdv2hLfQoMonPeYufjdv8yBWRpq8nHqjBUEXxJp/jMU+WE52jDfbv3qHz+0DSFXosYgtIaEaP6eTeDnNZ/QXq7iYUklDfQjHHO9SUq1/zpW1xaerok/hlD/m01OKISbb2KcT08I+tg17oe4Oq4ZMYGdAMkU6C0a9mvQTs1lyPpey+pY+u2CkKieKq5uWcyi8hJpuEiExrrlRV9m0/BDhJY9fKC0NKgrSYwj28+X2g1hkaXLKNQbMgoDmzmqGTdjJLVkhWovYlgNDsidO+xYwcNBKvHJUVGffw+/WQSZ/8gOXMspo1ddnUGvZ1H6ekIBOEUfI/ZTubRU+37FO7fEzi1CnfPsXdhZFaBb9Jx62XeVftFNkNRtoMpiwPJxhLT7+sYj3PhjDoq0n8b64hqF/Gcqc/TdIVYvdhVLc5y/DM/+RCCHoyA2+yMXAdFQN4uBv9/wJulyCL10kMF1Fg6mDXKxmjE0GTBbhoQ9CjTffvz+AMYu2csL7ImuH/ZVhc/bhnVJLo9j+b4zE5fR9FMVpeG+wY/yaSySGHOHAZTmVL3BnTfTp3qVLT/fpGSisZmM79mBKOcjYd4cxa7snceLAtGCl7NwiVnjKH4kQAjr5PS5fCiStQtd9A+yCDvm9y1wKTKOiwwFmK2ZjEwaThYfVBxOph8fRf/gs7C/E2o4TMCC74MqdJFe+HTSenX4BXPF+QEbYASZ+tYt7eVWPWlnP4GXbJNRwdf4AJh+MejRjWrBgElusHVSJ55nrqe2/WBGqvraYQQNHMH3ZNhw8Iyj+iQ+wmmpyiQmPJiVHQXVlNtERiWQVVqIzWRE6EiGMaKsqqNYam1sX3RVZo5aqimq0RstzWihtHDDVII+JIDolh+KqSrJjIkjKLKRSJwovmDPdOXGjgGqtCV2BL5snjmfO9DnsDC2l9kVaGD/Fpzbutf8qaOTERCSQkd/SksNKqUd7EQKjtoqKai1GW3OuvZWu+v1T4ipQL48lIiGDfFvL2gpNSZx3CaSgIpkTUwcyct5eriSVUnNvM6O/c0bWclv/mV6LD7AGenJy1xLGDXyLiYfjUda96KX2mZZ7ZOMvVITAVFNAqkxGSmYuBUrxSv+i/ez2cRCwmC1YxUuLYMEsTgmwXWWs1GRdZ+Wno1h5ORalvrXVI/w8VyHhUQunvcfP+i1YzFis4rFtygXoy1Lw2z6PZacjya8zYrU2UOi/lQkjFnMuQ9U8cP0sw+K2n+jTU80+xhusNdl4rf6MMSsvEqVooLUBKPwsl/2fFtdHvIGmClKubmLFofvkVdcStnsu6zyikCvyuL97PAMm7+eeXP0CUxFMaCuVFMkzSE5KIlucYf5iI9lPRd2TG36xItT9UAWM9eXkpqSQW6a2zVfq/jy7LwdLYx0V8kxyleKkyGbBtjaUkibLpUrf1fNbflo5BKOWcnkqKbllqJs60QL8adl1/VHWRupK88gtUdNosqArK0BZ14jJ2EBVQTpJGcXUiHfUuj7nl9qiJEIvdXh+TufE5p3Y2ns8T8HaLuHxzdKvLiLQQr+LrPUuM5II9a54Sd5KBH5xBCQR+sWFVCqQRKB3EZBEqHfFq+e9lXpnP1MMXh3Qr7wImYrCCYwvRftyvifiZ6rwL5aNtUZGUFgu6i6a/2QqjiAoXkl9Y++9vfxi5Dq5l7WG5Lth5NbqOvl8YifzeUl2f+VFSNAreHB2P/tdbyEr7c0vMuvGGmWppzDyMg67jnMzpcw2R6orchP0JYS5H+CAqz9Jyt73IrOuYPC4DQv1RVFcObKb4z4ySrXtHnR+fOdfzK9XXoTAgrYslUDXbSyYNoVZi1azdY8DTm6eXPP2wdfPhwvOLpy/4YuPpzPO573wPneKo04e3PQ6h7PzeXy9z3H6qBMe17045+zMeS8vzp06ipPHdbzONR/j1XKMj+8NLnleITDoHuExSaTLlagbzbzYTSgTuqpictMSiA67z51rl7ly0x8fz9M2f3xa/fF6cX+uu59sPtbm1yX8fH3xu+mD95ULnDlxmN2bVzB36iRmLtvHhfuZVDR05Ykh3qZOJfCMPQunN7Pfsvvwk+y9fbnh6YLLeS+8zp3G0ckDsawuLufxFcvq6ITHNZG9SzP70444eVyzsReP8b3hiYvrJe4EhRKdlI5cqX7xeWNmHVUKOWkJ0YQHXcfNzZMbvjfwdOk4r1b/rrmffMxPW12x+XmTG5c8uXTDl5u+vvh4X+HCmRMc3r2FFfOmMunrZew9H0JGuXhBfDW6ZJII2a4nZhoqC8mIDuDC8Z2sWTiLSWM/Y9jHA3nv3bfo26cPb/T7I/3eeJ3X3+hL3z6/47e/60O/vn14/fU3bP9//9vf0efNvvT5/eu80bcvfX73W37X50369vn948f060fft/rx57/+hbfefof+7w1k0NBRTJq7mr1ng8hQ6R9/9EDQUxJ/E+ddK5ht9xlDBg3gvf5/4+2//pU/9fsbffv9mX5v/P4n+/Nma1lEv97ux1/efpv+H3zIJ8NGMm7ybBav38upq0HEZiiofWGx7MxF2kxDVREZ0bfxPL6TtYu+YdIXIvsPeb+Vfd9+/xL7fn3f4PXX/8Cf/vxX3n6nP+8NHMSQUZOYu3oPboHpTwiroFcS7+vC7pXfYjdyCIMGvEf/v73NW39+kz593qBvv3688fqz4/yQ6xN1pB99+75Fvz+9xdvvvs+Hnwxj5LjJzF68nj1OVwiKSae4Vnwk6NUQILGmSCLU9nyxGtDWlFOcl01GiozE+FiioyKJiIggPDy8yz4RURGEh90nJDiI277XOO96kqMHdrJ55QJmfbsap/sFaAxWrJp0vPYsZd53i1mzdRcHjp7E7aIXfrcDCQ4JJSw8iojwrvNN9CsiMpKomFjiE2WkZuSQr6iwjQF1+znRyj4/p9vYP7gfQnDQbXyvncfV6SgHdm5m5YJZfLvaiXv5dbYJp5p0b/Yum8f8xavZsusAR0+64Xndj9uBwYSEhnVNHYiIIiIiksioGGLjE5GlZpCTr6CiVnxx26sjPq2nniRCrSR+7v+CgNVqwWxqQq/VUFtVjiI/i+RIP9wOOHA9IQGv4w64XA0kSpZFQUkFVbUadHoDJvExklewsv6rIRIEK1aLGVOTHq2mlqpyBflZyUT6n+WgwzXi4704ccSFq3cikWUVUFJeRa1Gh95gsj2uIyH/VyPQ8fGSCHXMpedSzQ1UFSUSdOM6vuGvXtO8J8Cb9dUUJQVxw8uX8LQiarql29kTJesdeUoi9DLGSTBSX1XViQHrl7EQvcsnwVhPVZU4YN38zp/e5X3v9lYSod4dP8l7iUCvJyCJUK8PoVQAiUDvJiCJUO+On+S9RKDXE5BEqNeHUCqARKB3E5BEqHfHT/JeItDrCUgi1OtDKBVAItC7CUgi9JLGz1BTTHr0fRIKdTS1rlf1Qr4K6CvzSI4IJ6Ws4bGFETtOfyGjv+CdDNQWpxMdmkChtunxdemfV2pBT1VeChHhKZTq2i68KKCvyiNFjEGprk0Mnmfw1dwuidBLGXcBbUEQe7/+kg3eBVTrOzOV30pt1jW2TJrBvhBxhdrWY9unWykLOYXDNRmV2keLJ3YLDms5IaePcC2pkuasftpL47vFN0FH4d39fDN+A9fzKukc6lqyr29l8sy9BD+2aKeV2uzrbJ08k73B4uTHMu47H+Vakor6bkbdLYy62agkQt0M+Keat+jlOE39mO89c1C1Xxf7OUZN9bHsGTuaLXcKqH4oQtA+XZt+m5tRxWi6+30+gpb0275EFWlotJjJvb6NfdezqWxZFvk5xenmzRb0eaeYPngxFzLLO7HwoOiWCW3cPsaN2UKAvIqmVr0Xt2jj2DduDFsC5FQ11ZNxRyx/HfpOtWq7uegviflXRoQsdUpyszLJF59SN6lR5maRma9CLy5WJ1aamgJkMdHEpRVT27K+vaBXkZucSHpxLU1mK1ZTPaVZOSjraynOzLItVWwWTNQUyIiJjiOtuBajuf1aCQJ6VS7JieIjGE2YrVYMagUZuWXotdUUpSWSodRgMDWgypWRkKFsXi3VquL87KEsORdNVGQ8KYU1Nh+a6017my21SWigLDOemJibbPxsJJtbRaij9CYj9cUyotNKaWjUU1eSSW6ZnobqYtLiE5FXtnma31BNXlI00SlysiNvcMEnltL6JgwPmRVR0/T0VSIEUz0KWTRppQ00akrwWjmYLzddJTiuCLV4nFWPKjeZxPRiam2/DagVGTZ/tNVFpCVmoNQYMDWoyJUlkFFSj6GjJW6EdnawYqgrIbMta3E57IesS6g3mLGoLjBn+BI8xIdK41MorGls8yYDkbWclKR0imxPt7dEoKGcrPgYYm5u4vNRmx+JkNBAeVZzDDZ9PorNAXIqtfUokmNIU+poMhoestZVF5OekIhcXNix9b1uImtZjI11VtQNPH1iUWqaMNQWkhwbTVzqs1m/JLrSKTdeHRGqkuGydDxL3NJQaSpIPrMcu6VnSVfpMFWGcNj+OL7BAZxcu4sbRTXUFd/F+fgFbgf7c3L9chzvF5KfcJFNU6byw/ZdbFq6BMdQJfKAQ9gf9yX4lhNrd3mjqm5oEwAzirsunLgQQLD/SdYvdySkMIdI15VMmLmGw8dcOOu4iunTV7D32GnczzqwfNo8jkWUomlScf6bAYz94QDO513ZtmgBu29mU6M3oAhutenEhhWOhBSpaTIUE+R0kNNeQYT4HmLa/wxl7a18qnUdp1dVZOD14wymbg+gIDsKj7UTmbHqEMdOueNsP5dJK86RotJhacjg0p5DXLr3AP9jy/lu6yku306lqiiQwzuOc/PuLZzW7ca7oOopa5MJqDO92DZzKttvFaIqknHq2/cZsfgY5/xklNblEehyggsBd/F32sAKx3vkZ0fiusqOmWsOcczlLI6rZjBjxV4cT7tz9sgKpn/nSFhJHY/1bMwlBLuefMxOYU0x8W6rsft6NYdaWM+YsYK9jiLrI6yY/h2OYSXUllzg2w+/4Id9pzjnas+ihbvxyaymwWKi5J4rJy8EcNffiY0rHblXUIu26C6nDp3meuA9fA/N4P2ha/ETW0JGBXdPHeL09UDu+R5ixvtDWeuXS26SF/ZfT2O7fwGViiTOr5/EzFUHcTzljov9PCav8EBWrsXckMHlfYe5GByK//EVzN96iosBKVQWBXJk5wl8gvxxWr8H7/xKtO2vdW1qXW/7+sqIkGCqxWfJh0w+kkippola32X8fepRZKUajFknmPz5Qk4GJpMSFkuupoLAHRP5av4W9h89wvoJHzB0gz/5BeHs/eJDZh0JJDohjcKaBtKOTebzhScJlKUQFpuLvvHRqSFo7rNz0kTmb9nHkSMbsPtgKOv9Msnx28Cwf67iYmQahfJLfP+PiewKiCNbkcuZbwYx3yOLCm0F57/5mDnHH5BWpCT51ByGzTxKRIYP20Sbm1tsDhBt5lFwZyfTFh/jQVYp6spwto8axeY7eRQE7uogXVxltQ7ZQTuGr/Imt6SU0G0j+XSZB2EpBRRH72PCP9fhK69Cm3SESVN3EpRdSnnoNsZN3MXtzAp0mSeZOnoRJ+4kkSwyq2986qtITRoZh+w+ZZV3Dqq6GgLWDGbS3hDSiuqovLeDSRPns3nfEY5ssGPAsPX4ZmTjv/FTPlvlSXhqAfLLi/lk0i78Y7JQ5Lrx7ccLcM8oQ/uw+yOgCd3NlEnz2by3jR25EuXdTYwYuQrP8FQK5JdZPHgSu/xjyFLk4vbtxyxwz0CZf545g+dy7H4KhcoUTs/9lK+PhFNcfJfdUyYxf9NeW/y+GjiMdT4JXP9xBj843idDqaYyYgdjRostoUoqH+xhxg+O3M9Qoq6MYMeY0baWUKlKxuGJI1h1PZsKdQ1h20fzz2XuhMryKY7ez1cj1+GTU0m9zJGp03dyJ7OE8gfbGT95F7fSy9BlOjF97CKOBySSHB5Hrkb/VNa9TYBEf18ZEQIDwWs/ZrqjjDKNBcP9DQyecZyUsnosulx8j6zju8ni1fcsMco4DowfwkKXMGQ5hchTY4jLrqSpSY7TlBGsuSGnqlE8AwR0ub4cWfcdk+1msvpsNNW6RyJkznRkwpCFuDxIIqdQTmpsHNmVehri9zB6wi5Ci2oxGh6wedg0jiQo0VgaubX0I75xzqBMU9HcHbvUPCbUFL6F4SM34ee/lTFDF+Lc1qaqntRjExmy5BLZqgYESxaOdl/wY2AOkYc7ShfHiiwUu85g5LqbyKt0pDlM4IttQRTUGLCWuTNr+HKu5qjQlnixxO57XMJTkXmuYPLKi6RU6DDrcvE7uoH5U75i5mo3osq0Tz8xLApcZ37Oupu5VDY28WDzcGY6JqGsM5B57CuGLXQmNDGbQnkqsXHZqPQNxO8di92u+xTWGDGEbeHT6UeIU6gxNwWw/B+zOZ2qpP5ha8BM5rGJDFt4mvtt7TQYaUrcxxd2uwgpqMFoCGPLiOkciVOgNjcRsPwfzD6dSon8vK075pkljgk1EfHjCEZtukXWg4NMHL6I0yEJZLfGrzSJIxOHscQzg/IGAUv2cSaN+5Hb8nJSTk5h2BJP21sRBUs2xyeN48fbcqr0xZz5ejTrbohlM5N+9Cu+3HaHvKomrGXnmD1iOVcyy6kv8Wb5pMU4hyYj81zF1FWeyMq0mHVybjlubGF9hsjS+qez7oUq9EqJ0P0Ng5l4MBZlnRl94Cr+McWBRGUdalk44ZkZJEVcZt34Cey4Hc6hiR/wxfYgCtQGrIIVo9GMYM7j5OSRbPDLp1ochRS0yMLDycxIIuLyOsbb7aBEWfuwGlgUZ5k5YBzbg/JRG8S164225XqNCbsZNWE3YcW1mAwP2CSKUHwJdZZG/JcM5JtT6ZRpytuIkJXK64sYtfgcyXHHmD5wHNsD86h9aNNMifssPpx6hBilBrMplUPjRrHplpzk0193kC76b6HIZTqfrfVBXllP6uHxjNt2t1mESt2ZNXQpl7JV6KpvsXPdIc6JL1PzDyIyR0WDyUy9LIKIzHQSI66wfsJX7LhTSE3bkdmHFABLES4zRrL2RqsIDWOqQxyKOiMK0e9x9tzJq8VgFbAajZgFIwl7xmC3O/ShCA2f5kBssRpz4y2W/X02TilKNA9FyGIr/0fj7Lktr2ljR8CYuIexX+0itFWEPp2OQ2wxanMjt5b9ndlOKSjaipC1Eq8fxrDYI5mSTDe++ehL7G/nUtPK2lCCx+xBTHWIpLjOjCnVgfFjNuEnryDv3GwGTaDKfY0AAAZUSURBVHUgsrjOFgOH8WPY5CensqHQJsJrvUURMpJ2xI7x2wLJqxIF/xyzP13KxcxytNUB7Nl4CA/P6/j5BxKZXUGD0Ux9cmQz68grbPhqIjsCWupfW8a9+PsrJEIWiq8uZtT4H9hx5DSu2yfz3oAp7LiRSvGVTSze7cHVq8dZOn0p7okKMm7aM33UBOZvPcgJFw/8EgvJeXCcOe//kU9/OM29fC0GSx2+Py5mt8dVrh5fxvSlZylVaR9VB6OCW9tnMNpuPlsPnsDFw4/EwixCDkzh7b+NZ5tvCmn3dmH3l/eY5nCXjJQANn3+Jh/OOkFIfg6uc4Zgt+Io5y66sn/zDs6EF1GnLeLWjmabW1psJpRq0RX4sHHyBOZtPsBJF0cWDH6X0SvdeRDlyZqJT6ZHJIZy8rsP+eOnS3C+fo3dU9/lrbFbuJFSgMxzCZ/0G8zCM+EoCq+wZtYCVq9Zy4YtOzns4kO8UkulzzZ+2OPOlSsnWDZjKWcTytB2eOfHQnm8Kws++hMjlrgRpqgj8fAEhtgtZqtTMJmp3mybORq7+Vs4cMIFD78ECrNCODj1Hf725Y/cTE7j3u6v+Ov7UzkUmEZywBZG9/uQWceCydc8Wp7aqAhg58wxLXac8fBNoLQmn/DD03jnnS/50UdG6r3dTHzrfaYeCiQtOYAto/vx4axj3I08wewhX7HCwQNP1wNs2XGGsAI1TU0Kbu/8mjF237H5wHGcPXxJUNaQc2MTU+zmsWn/SVwcFzKk/2hWuEVQmn6dzVPsmLdpvy0GC4f0Z/SKM/j4HmXu3//MiB/OEBwVxMGv3+PtsZvwkuUju7iMIX8awgKXBxQVXGXd7IWsXt2GdUk9lTe3s3TPWS5fOcHymctwi1NSb3rYF31U33rpt1dIhMBUW4AsKpKE9FwKcuIJj0gmp6yexopckpNTbJ+U9Hyq9CZMunJyEsIJDY8hOauAUrWeevEuSUwk8akFqLQmLIKZKnkyySniJ4X0/CqMpoeXZ7EJgK48h4TwUMJjkskqKEXdoKFCnkx0VCLZZXWoK3KRRccgy1OhqS0lKzGa2OR8Khu0FKbEIpMlkZqZTXZuScs7npttJra1Kb6Ey1yPMjOB6LhksvPzkUU9ID5TQW2DGkXGk+k1tRXkpcYSGZ9GgUJBtiyaqIQsyup01BRnEB8ZR2pRNeoUd/YccOeK9y1u+3vjcWAZ83b4I89ORfYYMyOWDs8LAX11AamxkcSnFdnY6pRJhIXHk55Xga6pnrKcRMJDw4lJzqKgVE2DpgJ5cjTRidmU1ampyE0mOkZGnkpDbWk2SdGxJOdX0tByZ9N27ll0VOQkEvHQTi2NRi2VeclERz9indyGdXZSM2tVbT7JsTJkSSlkZmeTW1Lb8iJ8C7qKHBIjWuKXX2qLgbG+lMyEaOKSs8jLlxEVFk+GeGfUoKG0JQZZeWIMwojPKKJUmUuKrfyFqKrLyE2JsbEurdM2s46KI7WwCnWqB/sOunP5MdZ+5GSl2uqXWEfF+lnZ8DTWvVOFXikREkMkWMRXdYqT5SxY2uiFYDFiMIjC0uZMEiyYDEbbS8dtqYJ4XOunNeACFqMBg3hCtDm0dWtrngajuKJG8w6PbLTaavuf5jwQsFqsNn/F2/7tbYvlaGuzOT8Bi7k5H4ul7cu5Okh/WI62ebf/biL52BxWuMWTX16PTquhLPooq/YGoqhuxCr60J5Z24K3fH+svLY0K2bx5WEPeQlYTGJXtbWc7f1o95vm309m1d5OC8unlrV1uxWLtflY8QXzD91q9d9isnXHW+NnSxYsmG3+WrDY6lOrN+1YPzXvdmUSTKQc/46VZ2KRl7WwjnFkzd47FFXqX5h1qxe96f8rJ0K9KTg976uAKsSBlUs3sOekB5e9buDjd4eonEoaH2vx9bynvd8DAdX9o6xetoHdJ9y5fL2FdbY4l63N1bL3F/SJEkgi9AQSKaEtAUuDivysLHILiilRllJWWffia3a1NSR9fy4BkXVBtsi66JViLYnQc6uGtINEQCLQnQQkEepOupJtiYBE4LkEJBF6LiJpB4mARKA7CUgi1J10JdsSAYnAcwlIIvRcRNIOEgGJQHcSkESoO+lKtiUCEoHnEpBE6LmIpB0kAhKB7iQgiVB30pVsSwQkAs8l8P8BQIzxTVPvPikAAAAASUVORK5CYII=)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HKqah7XFUhan" + }, + "source": [ + "#### Embedding propagation\n", + "\n", + "The embedding table is propagated through the network using the formula shown in the figure below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZvrR6lvUUuIm" + }, + "source": [ + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAXUAAAApCAYAAADKxrnwAAAgAElEQVR4Ae2dd3iUVdr/33/23d/qirvrvjZcxV1R0bXRV6SDFOlSVYr0IiBShFBCCb2ELtJbSEJIMKRDAmlAAiG99zLpk2QyM5k+z+d3PZOZZBKSEAUbO3Ndc81TznPK99zP99znvu9z5n+wfWwI2BCwIWBD4LFB4H8em5bYGmJDwIaADQEbAthI3SYENgRsCNgQeIwQsJH6Y9SZtqbYELAhYEPARuo2GbAhYEPAhsBjhICN1B+jzrQ1xYaADQEbAjZSt8mADQEbAjYEHiMEbKT+GHWmrSk2BGwI2BCwkbpNBmwI2BCwIfAYIWAj9ceoM21NsSFgQ8CGgI3UH1MZEBRJ3LpXgk5nfExbaGuWDYH/PgSM0hgikirR6IRmG28j9Wah+R3fEGREnT/EpXgpGkPznf87buFvt+qGMlJTi9DqDaCWkp0QQVBYEpqaXBKTS9HobYPsb7fzfqWaGcpJSy1Co6uVmZzESK6HJaFW5pCUXIpavG7+CKoMvI6e56ZEjrYZUXpoUjeUpZBSpMNgBHV5NgkRQYQlV6DOjSepVGu6bqmQ7feXQUCTeJLVmz1Jq9BgfJw4vY4wRWGzEGYyWq3+JwCrpyAxmVKNnmbejZ+Qp5KEi4dxjiqhRicgVKdzafUYhix1QyorIeb8AS5ElaLS/5Y7xUB5aipFWh0G1EhzEom8Hk6SVM1PqbZekkRyqZpfbcLYFGGGJ6HW/DSZkSQlU6rWPVKZSbx0BJe7RSh0RoTqDDzWjuOTpa6UVhQTe+EQzneLUVo0c0FDYeAu1h6NoFihoylJejhSV8bjcuAC98o0GASB6rSLrBw5iGUehShK73F2nxPRUvHeT3g/bI/8NAQEKQGrp7HOLw+59nECvp4wRVIUCdPNbjRDll6iQq5pJVZ6KgsKkekNCBipijnPgQv3KFPpm3w5WplpXTJ18hm+XX2G6OIaEwEK2kLOftGBARtvIlVoUaScY+U3R7lbXnu/7sHf0IEywY3vnKMoVuoQhGoy3Nfw6dCluOXIULdSnPRVEgplepNCZ6yKxengBaJKarDw0i/XXCWJbkdwiRJJsZYw3deO5ZOlFymtUreyGnqqJIXIdAaMoszEOnHowl1KRHxamUNLyTQp57Bbe4YoicIsM0Wcn/oOH28IoaRaiyLVCbtlR4ksUdThZ5TdZNtsO9zTK1A1oZE8BKmrSTq5lJVn4yhTiy+JgFZymomv9cMhUopKJyf5zHIWH42mSlM/fWipgbZ7D4+AUeLKrGHfcCVPzuPE6SbCtDtDjDVhfv4mAzbdpEL5IK1LQJl7k0v7lzLpS0duiaQKGBUpnPt2CUejpI9Ae1YStmkC888kU6Eyy7vmNut6fsRS3wKqxc7Q53N+3qes9c3/bQ646hTOrlzDmXtFKEW1XNBSeG4ybw/cSFiR3IRZSxIqKHO55X6AZZ9NxzG8rFa7NCpIOb+KpUfvUqZ8NINnS3WwvqdOOcuqNWe4V6i0Isx/8/GGUErlrZGZW3gcXM7n0x0JM5OqKDNOdks5eqe0Xnu2LvRHHSsJ3/IZX52Op6zGIjORbOjbm6VeOVRpRJkp4MJX41nnbT4X8xfkhK77lJknYihT3s+tP53UFSHYj5nL2fTqOtuO5tZqundfytXiWhD1eWeZNXINAcW/xij9o9B9TBIbkVyYyn+mnCJDpn6EU8RfGx4FYRvHmwizUm1WTUTC/KiHiTDlD1QBBdRlOST5rqBnj2+5WixHZ2qSnvzzcxmz1o8ChfbhNC91OGv7jsUxphzLe6ZP3c/wLjNwyqiittp6Mg98Sj+7q5TLtb82qPeVrwx3YOKC0yRKVdRShYaI9b3pudSHPNmD8RHUZeQk+fFt756s8C+k2txEfb4T88euxTevGpGnfpmPkvDNk1hwOpHyOsKMYEOfXvWE2WJFzDLjt5K+vVbgVyAzK0l6Ci4sYNw6H3JlmoeTGc1N1g8cz56oEixjjD7tEKO7z+RsitSshevJPDSegXZ+FFdZZqQClT/Mp8/n33GvVGnuq/rG/GRSV4euoseovSTWkYeelL1D+GC6M7lyswDoMtg7ohd2QeXUPGhgrK/TL3sk6FCU5ZEYcYvkMu3v21QklHFpxrsM2BxZL8h1aBqpTIwmQ6H7ddtoKCQuOhelTpzdtfKjDmdNn09xjJVieT9FwhzWeQZOmVVompiCNs7ZaDCgy9jH8D6rCCyxkDroMw8wpp+dFdE3frJ158aCY4zrOAvnvCqzmcJIyYWpdBx7kHhpjfnFE1D+MIf3JxyhtFLVuox/sVRqbq7rz7g90fXanz6VAyO7MvN8OhWWwbSl+hgNGHSZ7B/Zj1VXi+pIHX0mB8f2xy5AJPpW93pLJT34nvom9v3HsedeKQqzMqtPO8iobjM5l2ohzJazMclM5gFG97cjQCLDMgzrMw8xbqCdFdG3nE9zd42SE0zqOhunLCkqEyxGSlym03XcfqJLFPUyc2U+XSYdRlKmrMvKkHWI0Z2+4GhcWd07YbnZNKkbCrjpcoQ9Wzfj4ODQ8Lv1NGElNWR/N4p3Zlystb2JuRmLOf/Zu4w5lILM8pYJSjxmvM247zORP/QQLaBMucrpgzvZtnUrW3cdxPlmAaqHjSZQZRJ0eAmjeo1l5+1KLDNnC0At/grVJPmf4uCubWxx2MTWPYc4cngfjoddCc+SoX+Al1JQpnD1zCF2bdvK1q27OOh8k3yVxXFnpDTqEkf2bGfr1q3s3H+Sq6k1aA1gLLvH5WP72LFtKzv2X+ROsdmJpQ5mRae3+PJCHtUN8BZQJLqw64A/mXJtixq8TlZASnQU8VllKPXG1hNvU0Dp8wi7cATH7dtMbdj93WWiSsqIPr+DAwFZKEzuewMFty7yveNWNjeWNYetnAopQpH9PWM7zsQ5X2bW9ETCnELHTw+SUKEy2TpLo/3xcHPB2dnZ6uvCpdDMuugBQ9Z+RjQidUF5mdnvj+dIagWK3DDOHdrFdlG+dh7ENUKCojCKH47vZ6elj9wiKVTpMBrLiblykgM7t7P74BkCA7cx+P25uBVY6qjg6pKufLwpnBJFvUaj8VnAu6P2UVJR0xRij+aaXkZBSjT34jMpVepMznKD5DZuRx3Ztvn+d3rrqRAk8iyOje/MrAs5VJqN58YSZ6Z1HsuB2NrZh7E0hoDLbrg0wNgZl0uhZCi1tSRkyOLgqH7YWZO6oMRzbkcmfJeMtKYVI/ADUTAgibjEsb3b2LK5ET85bOFksAR51jEmdJmFU3Zl/SDr/CVdxu0nxqTdGimNCeBykzKTgcpsMhbJc0wjUheUnszrPJHDicWkhDrx3Z7tJk7aedCV2wVyCu95cvJAPU9djJCg1BoxSmPxPnWQXdt3c+B0EMlxexjRdR6uORXmOioIXPYhQzaGIKmut9lrfBfRcYwjklJFHTKC8ipLunRinmt2XX9ZbjZN6oKC4hQPlvdpz8sv9+dbt2vcCAsj0H030z4axMbwMhJ3DeadOZdNQmPKTO7Pwo792RQhtSJGDV5zOjBifxrVrRnpLbVq8ldAV5lPwqVl9HnjHcbvvEZcvvyBxNlkVtYX9dVIbm6k3/NdWRn8I2cUgpbKvHhcv+nNG50msz/wDrEJUfjtm8vIT1fhnt582JFYBUFXSX6CO8v7vsm743dyLS4feR2RCqhK0/G3H0KHt0ayyS+KzEq96QUVVGVk3TvDvL5j2eQTQ76y9rohfT9DX/6INWEiIVs1siae44sXsfdGDtXNhSEISlI9d7JyzT5cvLxw3raAmfbupMg1903vrHJu+dAopyjJjaV93uTVfw5j49V4JDU6FFme2M9ehVtKNRqjgKI4hR++7c+b7V6m/3IX/K+HERbozp4ZvRmyIZii+D0M6ziXSxLL9F3B1a+78vHGm5SYGmqkLD4Ib8/LeHh4WH0vc+V2Tm2oGNAUqaPxZv47o9iXVIGyuog412/o3+FVXum3Ep80KRplGVmhe5jU8TVe6fAZ+25nUyX2kaCmPN2Zxf1HseZSJBnZznzZ8QtOZFbWal26KDb17cESj2jSS8RBwNTjSM9PpuscV0pllql0yxD+uLsCytQr7F61ln3OV/By3s7CWfZcSpKhqi4m1XMVA01tW4aTbxBhYYF4OM6k75D13JDE4TiyM3Mv5lFlJnXFtW/oPqjenm4siyfIx5PLDTD24PKVW2TXtEDqaPD56n1G741HqmyB1I1qqgpSiIqMIjmvApVB9NQ19RFQlqTiafcxb736Cv2WOuEbJMqMB44z+zJ0/XUK4hwZ2WUuF3MtpK7g2tIPGbwxhEITYRopT7iOT1Mycysblag9mWTmflJH48vCjmPYG1dKsSQet2UDefuf7ei74grJZWqUZVmE7f2cLq+3o8PEPYRlVKAzCgjqctJdlvDx6NW43kqnvNSN2d2mcDy1HNNYp7vHloG9WOJ2h9RiS+SgQMWFaXw415lCqdXsTp/K3qHt6W8fbGqPNUpNk7qYQh/Dph7P8Kc/9WN3ehU1ej0aeQHBm6Zi51uO5PwU3vnsNPlmW6Tu7np6dl+CZ3wWpRZiEqScnfgBs9wkKB7RtEuftpsBz73JbA8xBKjpLrduYGuOjSXHGf3Sh9iF/khSN2WuJ2lHf55/bzG+JUrEYKeazMOMefU1Jh1Pr5+1NFcRfRp7Pn6BDrPdKbwvREmg6OhoXnz1c87myxvaI7VRbBw+mwt1miGoAxbx9osj2J9useGKhRrIdZnHoGmHiS1XNWt6UUYd5IsP+zLn5G3SC4tI/P5LBs89RUyl+qeTuli8NhL77v/Hk20+4XC22TRhrCTUfgTDN1xFYooi0BO7pTfPPfkEfXckUa7QodfKkYRuYcZqH0rynJje8QtOZpmftxDm5WgyTIQpoKuRUVkhRSq1/laYImOMQq2cGDL2MqzXylpTi1l0BOl5vugyG9fcWg3bUO7D4k7P8OQLoziSWGGyhQs1iewa/CJt2nRi5Y0SLIq3LmE3I4evIyhPjk6fzfGJg/jWbGIwFp/ls/admbjxLOESlTmkT0nIqmFMP51MVVNhC83JSGuvK+9xeMpH9Jt9nPBUCUWJR5kxdC4n74kECfq4rfR74c882Wc78aXV6PRa5JJQts1cg3dRLk4zOzP5RAYVJluAjqjN/floiTtRaSXoDEYEXQ2yyopGGEupqJCjFknLJG4Z7B/Rh5UBknpTi1CB05TuzHHJuk+rrG2agYo4D/Y7bGTr7gMcOXqEg7s3s9Z+Lz/El6FuMpZST9y2/rz41JP02RpHSbUoMwokYduYtdabwhwnZnWZzPF0aUPCvHSX1GIxBLslmVFTLzMHGNVvJf51NnUQKi4w9T9zcc6sQCUYKPdbQtdnn+KFEYeJKxdNbQKqpD0Me/kvPN1xBYGSarMPR0ei4xhGrbtKdpUWwZDDyS8+YaVvnskpaiw5z+QOXZm4/hSh+TVmX6WS0DUjmXkqAanF9iiCJlThPO113px2hqwKVYPBrwVSj2Nzz2f40xMD2JstR2Mo57Z3MOlZcaSU6tBmfM/Y/t8SWCaGZxkpPjWOdh9MZLPTbYo0Zi+34gbLB00zOVMtFpnWymdz6Qy5Bxn6wjss9Ba9z02n0pUlcsPTBScXbyJyquu0eWN1DhG+bpw7c54rkfkozIOPsfQEYyykrsoj0qSNuHP5WhzZCTfw8ryMu7s7/jElaPQGSmIC8LzsgYdvFBK1hvR9g3mx03KulSlrIwRkrkx+5QWGH0hDVjdD0VGWeIMrLk64eEeQU12rXWPI5dCwF3l3oTclivsbJD0znrbtp+NS1CiaRZ/IzgmL8CixkL2RohOf8tI/J3OuTqMFdEkcGNmJ8Yfjm7eLCpX4Le3CC69O4FSGzKQ91+THEpUkQS6GgjUNM+gz8DjhTaaiBW1ed5f1H4qkPpwjuZawOAHFrbX0776QS9nVaAU98dv68tyTT9J/dxpVKgPlEb6EpGUSm1KKVpPJMTNhik5RY/EZJr3WiYmbRMJU0xoLnLEkFq9d43mz3UCWnQkj12zmUoas5JPpZ0ipNDuWhQr8v+7E3//cltHfJ2JyzBqzOTi8LW3++BTd1gabZwc6EhxHM3J9MBK5OFXWke/xDZPXX0MimrhUGQR7BnArPotytRgOB0J1MJtm2OFpwrg5UMXrejIun8QnQ476/uCGZh4UqPRfzn9e+icTTqSa+lqoKSDuXpJpKi9aAvUJ2xnwwlM82W8nyVIl+vJI/ELTyIhNoVSjIfP4ZwxZ6U+B6BQ1FnPu8zfoMnEDZ8IKUDU3w7OujbGEWO/dTOjwKgOXniI0u6Z2MFOGYDd8BqeTrGfx9Q+qUy/isNYRJ79wohJSycrOIi0xhoiAs2z6dhte6bImcNCTsGMgbds8ST9REVDqkUb6EZqWYZIZjSaTE58PYaVfPjKtgLHkHF+8KRLmaRNhtqo5pXH47JnI2/8cyDcnQ8gWTVmAMnQ1I2eeJqGs1lciVATwTdfnaNN2FIfjyk02bmPOYUa/8hf+31NdsAuSUC2+2rpE9o8bw4agPFOdQEfBD8uZtiGAPJkGoyqTkCtXuRWXSbnKUDsrrw5h8+w1XE6taISBhoAl7/OPwVuJqrO/12L6YFL/03t8efA058468MXnO4mSmTU3XR6XFk3CPkhcTGFElXGdy/63SMiWmkgBBKqv2zN11RWyFc0E6xvKSA4L4lpAAAENvkHESDRNLlxqmdRF27ErDvb78bx1lzBXe6ZMXMeVLAWa6miOLV7Ooev3uBd8gqXjZ3AgotykBTQgdW0RXqsmMnnVYTxuZ1OWF8GpxYPp0ncBZ+LL0RiMyDPdWfXZXPb4JFCu1ZJhTeqaMqKOTeXDj2ZzOq4Sk2lOUJB40QH7/Z7cuhuGq/1UJq3zJEuhxfiQpH65jtR1JGzrw/Md5uBhNQAYso8y+rV+bIy436FS91opA1n6wTM83WMj9ypqBdVYay+oS9LkgeY6qyZv47ZIEE0mEAW5KVIHQfYDc97pzNdeRch1FlL/E+9O3ceJs2fZPGUyOyOlZjNSPWEWiphZEabUFE7bXOH11wVlKZmxYQT4BXMntZBqkwmlmuCN07HzzLCaURmR+i2m49+fou2YoyRVqtEXujD9o7d45Yk/8HS3dYSUKNHrEnAcPYL1N8SBr3bYM8oScd22Ddf4StR6IzrRGVw3IqpJubSD3W6xlDwwLl7DjdVT2X6rDPn943x9oxocKQla3pln/9KD9ZFltc5BoxhZXf+xkPoT70xhz7EznN08lSk7IigzTz10BZdZOmUDVwvEiDYVmcFXCLgVR1a5usl3sT5n85GgpDQzjvAAf4LvpFAoE+35AtUhDsxc/QNploHT+kFjKX6blrLLK44i0YxodU/QlJPotpbFjqFUKixuSksCC6k/wTuT93DszFk2T53CjtulZsx0FFxextQNV8mvFmVGJMwAbsVl1RGmJafmfk0yExfOVf9g7qSIseqi2a2akM2zWP1D7cBpetZYgf+SLjzXpi2jv4unvEZPoessev+7HU/84Wm6rg4yDay6xP2MG72BwDxLJA0YZUlc3LEd11jRSmBE30hmUj12scctxoRNnSiZCtUSvvo/vNR9mcmJay0mrSD1Tsw/484VDwdGf7KBiEpLuJORqvgLbN5ykaRq0f6jQ2e9ykidzMWtO3EzEWHD6tSBqE4n8NxJjn5/hCNHrL5HnQjJVTWpgbVI6oYcLsztxZBFR/EOvUWY1waGtGvPl075VBdeZev8bfgVKtDUZHJwZHsGbLmFVBwR6zT1YnLCT7N563H8Y7IpU2gx6BTk+3xDt3/0xD60DJUetAkH+Xq1OwlF4oIBA5n7BvNC+09Yum0DS6YMY8DYZRy7mkS5pjbCw5DjzNzeQ1j0vRcht8Lw2jiUdq9/iZMYS67/6Zr6jgmLsCb1iLXdeO69RfiKMbVmkNXXvubf7SZwrIXFI7q4rfR9rg2vzbps1jrreqjJA72inJLCQiQ5LswZvoIr8enkSYoorlLVTVvrHmyG1NHfY1PPVxi2J4Eqldasqf+JjnNP4vaDB5vHjmBDeFmdqUMkTBcLYRoaE2Zdac0fCIKpboIgYPoC6pRL7NjtRlypyhTHbHnYKPVl4Qd/589tx3A0qZws5xl8PHsziz78P576W3fWhZRQFePIqBGiLVpetyhENHVVZUcSHNOEZqsvJDb0LtktLcbTKygvKaJQkoPLvJF86xlLWq6EouJKDA8aZPXxbB/wIn9pP4NLVqRhaZP4W0fqHWdzzPUyHlvGMXJDGCWWkcMoI9F1O9td46gQZzI6MVKqmXfXOuO6YwFBNMNYMBYfVafgvnM3brHF1DRlRlFeZdXcfUSJcfCmsOwwzh+7xJ1CpUnLN1YEsGb+Xqoq5HWl1B7Uk/oHs47hctmDLeNGsiG0uFYrFmM3ZIlc3L4d1zhpE4TZKLumTpuSmVQPdu1xI6bIegGZkQq/r+nybBvajv6O+NJMXGYPYY7DQno8/zTPdFtNUKGUmH1jGb0+kFzR9FJXngFZ9h1CYvKpMdvx627pi4gLizINqvfPRrVEOfThH+/NwyOvqoFp9sGk/kRftsVJKJUm88MZX7JEz7+lVEMVWRE3iJZo7iNgfWEMIXezqdS2MH03KCgryCcvN5dc629eYe300VKO1W9LpC5UXuLL1//F8DWncHFzx93tJHvs13EiXIq6pozMu8F4XzzDiZPfs6jXc7y1wMu0MsxE6m3fZvSir/j044k4BOYhtxJAQR7NjsGv02OFL4WKKiIOrOVIZIlZSGtJ/cV3ZnAiJBTvHWPo8OZYDkVbpksClZem88Zrw1l90hk3d3fcTu7Bft0JwsvV6B9A6pXnJvDSa9O4UNjI/KKLZvOk5fiWWq5rCV7+Ps92Wc51ixkIqHGfTrt/TeW8tUnGCk8wUnzuM9r9tR0TTltrrA0SWZ3UkODuyNaN67FfM5Ue/+7PzOWrWGu/gT0/JFPTeKFZc6RuSGHXwJfos/42FUqNmdSfoM+We+QVS0m+cg7fTEXtTMdUuoGqrFrCVLdm7mxV46YP9RTGhnI3u+L+/XGMUny++sBkgvn0+xsc+WIA810TCN8ymJfa/I3u9kEEOQxjxPob9w+Cgha5vInBzahCLvpM6t/m+6qlSvBg37ZNrLdfw9Qe79B/xjJWrbVnw54fkCtbXgFpLHHii389Q7txJ+pNSY1KsJD6k70duJNdhDTFi/N+GcitbDwGWTZ3QmIoUInbBDz8R18US1hUFlKzgtM4R9GnMXfhBQoqakxEVxO+g7EDp3Ag0uzs1yfhOGcd1VJZo0ctpP4kvTZFklUkJcXLCb8GJqsWCLNRbq071VMUF0ZUltQ0E7N+RlQEFnd+ljZtR3M48DumDFqAc2woWz9px1+f6caagKs4jBzF+sBcqhr7Fy0y0zhaziwzxsbXTQXrSdg5iJc7TMe5LnqmtkatIHWzTV3QI6+UodGkEuAba5pailkIWjnVKnFaYt1EMNbITY7MRpcbJqqJwXnLGlauWM7y5VbfFQ64xstN4XsNH4CmSV1LVlQMhXnfMeKFfzB6byTpuRIKCwspLCxCqtRjKL7B7vkL2HIxlHsJsRwe346353maQjJrNfXOzDt6lMX93uOjOSe4Z61RCRqyTk3i3+9N59ztizhsdCOt0rL9gZnUzTZ1VclNdoz6gB4LXUgVp30iaR4bzYsvj8YxMp0ciVinQgqLRNOCEaEZ84s26x4xxVqqL8+hfdtRHGzg/BTZOoBlU3bVmUtAS8iKD3i28zKCSuvNIZqAhbz1ykRONLvwQ8X1ZR/w/PuzcM2oX0jWGPf6cz0VOQnExUQTHXGILwbM51RQOHeiY0jMq0JvPVsTH2qO1HXROPT8B0N3JVBZp6lbbOpG9IpKZBoNadf8ia3Q1GrSFuFvLGz1lfsRR0ZUcnHpdVMSakTq/RXv//3PtB0wlv7dZuCSUYksbjdDXmrD37p/zuc9+rPGyvTyIwpuNqm+IofEuBiioyM4PPlj5p8MJOxONDGJuejEDcJa+KiCV9C57QfMvJBKZTMOrDpSN9vUjXoFlTI16rRAAuKkqE1mJAGtXI7K2IIy1kI9Gt8yqkQeuJ8fLOmEmhusmbWD2+LaAQEM5ancDr1DhrnPhUpvls3ZT2VFfThf7bP1pF5rUzfLjFpNWmAAcVJV7QzKIjNNEqOlFq39NcuMxSls/ZioCCzqxLNt2tL/0/50m3GBlPJK4h2H0+6vz9Bt0iR6DFhDYG7VI1rprSd+x8f8460ZuORYInxqK9QCqcc2dJSaZN9IWeAaPv3GA3mNZYJv3bIfeawvJ+3ubW6GhxNu/b15h3Rp0wuBDBmOfPxcB+Z5ltQt09UVXDHtj5BQHMCyLs/z5tiD3DHtRyM6p3LJKlZR6DKLdzvPxjlDtHNX4jL5Vd6c7W7aFMdYeJRRbbuxMjCHZP+tjOnUhfHbAimoqV/WbCi+woLOr/PR8Cms9co1CWptaw2k7hrAC+8txk+MfjFtuGPPkA8G8K1nlildTchKuj7/JmMPRFJq3lJBnptFscaA0ZDB3kEv8NbcH+o36NEVcMVuGUcTZSgzTzHh9bcYd/iueTsGsVQ9xf5rmbstpH6dAFoi1nbn2XcX4mNlfhHjbIe/2p/Nd+sX7lj3kiALYXWfbkw+eIuiB9p6rZ8ENP4sGbuJ8HJFCzb1O2ZH6TC+q3OUAgpv5r3diYWeEuRaMZLB2lFaOxc0lgdhP2EZ7s3OMhrV5xGeGsu9mP/+Mzz59F958/OzpFWpMaoT2DXkJdr85Rme77aMawXWphdL4ea9Qkz7y1iu/dhfDQFLx7MprKR1NnWhmtC1/fnP5P2EFdSu5m6qRH38dvpbOUpNKBvLub5hEssv5SNr7eYuTWX+U68ZKwjdtRC7CzGUNpY/oYp7Rxbz9fd3qGwQoysWJpKataPUIjM32PjZcsCxo3EAAAqgSURBVC7VLQL7qRX7sc+JisBCOj37FE//9Q0+O51sMu2qEx0Z1u6v/OVvz9P1G/9GphexDBVFSbe4ERROYnGtyal1Jeu4u6k3L703n8u5teYXfWks/p7XaJrUDfncdFrLsFee4A9/eJVBi+xYu34961cvZmL3V+m07Doy0bj8i34ElGmBnF49hFeefJp/j7dj+5597N2xlrkjO9FhwnFyZGXEn/uawZ06M3jaCjbt3M3u/ecIzVcivbqSD9/4iOlbDnDk++9ZN/o1nusyhc2n3XHdPJY327xA76WXSSrPI8xxLO926MUXdke5nmO27RurCFn5H/45wJ4gy4sjyEm5eoqVg17mz3/rxLTtF7hdqEZXk8fV7V8wcNh8HL0TKS+P5dySIXTuPIhpKzaxY/du9p8LJbc0kaunVzO03Z95+t/jsNu+h317d7B27kg6dZjAsexqNNpibn63mPEjxzNn9S6+O3GCI/u247DlMD4p1qYDPYk7+vH8m7MbOEoFdSK7h77LxKPJtZEcpj7TUp6ZREp6Itf2LmDG6nPckVjbCM0da9TfZ1Zr0OX6VDydrlNQ08yCJlGOLqxl6Ct/5n//+Abjt7gQWVq7UEp9ex29us7FLbOSrJvOrB/xT5763z/Q7uMFrFxjz/r1q1k86UP+1fkbrlqtAG1Q/s95YizHd3Fn/q9NeyafT6NK1H4FNQn7R9Du6b/R7dtrFJiiXiyVEPeXMe8VMmNv3f4ylrs/7ldP2pUL3MhXtrhaVlueRVJKOonX9rFw5mrORhSYZn8NyzKi1xsxFNzGZcNI/tXmj/yh3UDmf7sa+/XrWb14Ej1e68ISq2X9DZ//uc8EavJucNLBjvUH3bgek0VpRSk5iaG47V3DCoeThIgRUnU2X7E+RiQRrmwc1Z6n//gH2g2Yx4rVFpnpwWtdluBntQL0526BJX9juR/fdHuOv7T/nDPJtZttie/fwTGv8bdnurLcL9cc9WJ+QlAQd2E/x/3CCQ84zIr5W/BuPCO3ZH7fr5Ywu24mR+lVicwUSl16ZQWDu3drhtQFBUVp9wj198bLy5/rN28TERFJxK0Qrvn6EJpSef80+75CH/UFAV1F7da+Pt5e+AdHEh0TS0zUba77euJzJ99k59LLcokOvsJFZzeuBN7krujtVhswyDK57eOOu28wkbFJJNz24ryLL7eT00i5G4K/jzeBkVlU6Axoy+K57uNL4M148mTm0EOMFHz/JZ8fiqFMZV7iLmiRZscTEeSDt5c/oXeTKZSL6QXUJUmE+XoTGF2AQq9FlhtDsNdFnN2uEHjzLnFZ5ajU5WQnRBLk4423fzCR0THExtzj9nVfPH3ukKcWowEEtBXZxIZfw8/fD/9rwdyMvEt0SlFdSGYt0gJlp8fz0iufcybfskhHtI9pyDw3jb5TTpIsapvokVzbyayR/ejdqwfvt+9ArylrOODkTUhUIpkSKQpFJXkJN3D5/hLRYvRHUxYKsVBBg6xSURcyel+PCzIkSXe44evFlSt+BN9JpbhGxKeGmxtGMGbTNXIVWuTF6USHBuDj5YX/9XBuR0QQGXGLkGu++IQkU9GMPfa+8h7pBSOy1Ks4O/sRZx6IxIguTWEkHucvEpJRhbaBqUlAXZpNos9yevVY8ZDbDghoZJW1Zotm2qQvDGT3nNEM6N2LHh+8Todek7Hbdx7vkLskZEqQyhVU5icS7HKUS9EV1FQVkR4TRoCPN15+1wm/HUFEZAS3Qq/h6xNCstRs4mqmvJ/1sqCmLO0OgZfPc/zwXnbu2MneQ0c54+bPrVQxOKEBo4uCh6I4nZiwAEQu8LOSmVCzzEgtYdU/a8UbZW6UkXbNBWc/McLJ7HsUNBTd+QGni8Gki2s+rN8lQw6u367k5L0CyktDsR/Uk28u18asN8q5iVM1fovf4R9DtxNTKm4pIKDKv4v3kQXNkHoTWfwmLgnG2giGRpURzNfrLgsGNDVKasSOtQJRMGioqRHjmsUd6HSoVaKJxxINYfkVc2nkwRcvGSV4bNiAR4a4CrKuJATR9igWIuZjOra6p9egEonZXAdT+coaNOLWsaZkAqITxLqOtZfN1+uzMmknek0NSqWqEZnUJ1KJkS4vfsLelMoGW3IayiM5OG8mu0KLqdHKyY0OJcjfCw/nExzYsoZlC6bz2eih9OvZk979BjJo8ED69h3KbMcgcpsLR60vtoUjCya1eIphkmK79blurJi9Ec8GM40Wsvm1bokyoqldZl9fBQMaVe1W0/XXao+Mer1pfxlxK4JrdZuGNU71KM4FFHnRhAX5433ZmZMHtrJ22QJmfDaGof160rN3XwYOGszAvn0ZOnsPgdktr2x+FDV6+DyMaGTF5GYkExcbR1JaDkVVFr/Vw+f+S+Ug6NRoGvsQDBrT/u33v+cK8pNSKVLq0MuusWzAaHaEiiG+raitIMVpcnve+vJs3eIjQZHBdeezvzNSb0VbH30SNaneh9i8ahFLD4VRaGVnf/RlPVyOhpzDDH+5OytvlDbcJkDQUhbjzoE9rtyTKlCIWoQYrqVTUV1eSHZKHHfDb+Dnfp7jB3axedNW9p2+wp3catPy5oerVaOnDRKuH9/P2esZVGrNM55GSX7Pp4asA4zs+/OTul4t/vGEOGjqUMnLKcxOJe7uTW74eeB0/AC7tjiwde8pPO/kmPYCt9Jtfs/wPp5115cSfnQNdnu9SRLXBLSms/SJ7Br0Lz7edJNi8xaPgqaSgpxCG6k/WErUJHvswsHxPGE58kdPcg+uQKtTCKowVnVtz+SzuY029BJ3C5BTkJZtip23mmg0yNs0k6iuRFouRaZqOfyuwYM/5kRQIEnPQVw01Fw9fkx2v7W0vwypt9BqwYC2Rk6VtBypTIwAaQ1DtJCf7dbPi4CxjIjz+znkFEC06Hi3TOsfVKrSl0UfdOGrS1lUWjbvE60FYCP1B2EnOmWUZXnkFovOmt/4CyJIuTL3A/rYh5n+kODBbbOleNQIGMTtfXuv5GpRU5Exj7o0W36/bwS0ZF3ewZZj/kQXKFCl+eJzT0ZN4zj2JhqpT3Hkk87TOJlo3gzMKk3T0S9WCWyHvycEjJS4Taf7xKOkmpyiv6e6//7raiyJw2f3BDpY9pcRncK//2bZWvBzIaCJZMcn3en96XTmL1rMrFFjsL9W1DBCpsmyBcouzqTnlGPEPso/yWiyLNvFXx0BY8kVvho8j4s5ln29f/Uq/ddUQFCWkBETip/vdSLFvUIsu5X+1yBga+iPQsBQTNz1QIJuBBMcEkyQ3w0Syyw7eraQkyDDf+kI5p2z+hs8q+Q2Td0KjMfi0FhJuMM0ll7Koto6TOexaNxvvBFN7BXyG6+xrXq/KgKNIt/MNvEHVUmQBrBm5np80ysbROJZnrORugWJx+ZXQJXlwcZ154h/2L3QHxtMbA2xIfC4IKAj230zDs4xFDfzR942Un9c+tq6HeIiiIATnLtRwKPZ/Mo6c9uxDQEbAr8WAsaK27ie9iHWvItlU/WwkXpTqDwG14yKArKLlKZ/eHkMmmNrgg0BGwLiskh1MTmSarTNLvO2hTTaBMWGgA0BGwKPFQI2Tf2x6k5bY2wI2BD4b0fARur/7RJga78NARsCjxUC/x+3ILT73sRliQAAAABJRU5ErkJggg==)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Co9D_oNXUlgj" + }, + "source": [ + "The components of the formula are as follows,\n", + "\n", + "- **E⁽ˡ⁾**: the embedding table after l steps of embedding propagation, where E⁽⁰⁾ is the initial embedding table,\n", + "- **LeakyReLU**: the rectified linear unit used as activation function,\n", + "- **W**: the weights trained by the network,\n", + "- **I**: an identity matrix,\n", + "- **L**: the Laplacian matrix for the user-item graph, which is formulated as" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SLrIvZowUyyI" + }, + "source": [ + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOkAAAA6CAYAAABVnX5iAAAVDklEQVR4Ae2dd1hU176Gz1/35mhEjLElGsWSeKPx5CRGb2JNMWJLjp4cjSZqComJxxKjRlEsUQRF7EFRYjkWUETpKggoUqSpiBVp0oQBBAZmmGGG2e99hjqUGRgFIbnb59mPM7utb31rvXu13x7+gvhPdEB0oE078Jc2rU4UJzogOoAIqVgJRAfauAMipG28gER5ogMipGIdEB1o4w6IkLbxAhLliQ6IkIp1QHSgjTsgQtrGC0iUJzogQirWAdGBNu6ACGkbL6D/F/IU0RxatQLrfW74+gUSlVSIqkz4w2VdKE7hWshlArycsF62HMfgHGSlT58PEdI/XFX4EwqWnsZi8EgWHI/kblIa2YWlaJ6+bj9zowSVFElmOqnx7vw8YhBzDiXzuOTpMyJC+syLUkywngNSF2b3G8WqoCyKVPWO/vF2qKLZMHoAM50SRUj/eKUnKm7QgSeEVC2Xo9QIPH1b1aCqmp3qRPwPbMdmrRWrVqxi3aYt2G3ZjN1vJ/C/lU2JWlNzrvaTCGltP8RvfwIHjIRUkD3AZ8dqLNdasWShJQ7+CRSWlrWcEZpCUq8fY+GI/vTsac4azyDCIv3YN38iYz7+ip1XHiFT6TwqREhbrizEO7eSA8ZAKhQQtmM2Iz+wYO/5AJyXT2ToB//mP7fyUbQgp5RGsn5Edzp0nIRDYh5yjYL0kxYM7tKVt5b6kiktrWnRRUifsCIJctJjYnmoUNGSZWm0uraqy+iMPMUFRkCqyfZgwdsv0Xf679x7XEJh0C8Mf6kPE+2jyJG1YMmqYtg4qjsmVZAKUHrFkmHdOmL2tSupBUoR0qeoAkAJ6TFn2Pi1JR7ZxSh1eiZPd9+nvVqr6yzWX6/CI7uoDel62nwZeb0RkCqClvN2144MXRlMVpGaslRH/tGzEy9PP0xSvqIGFCMlNHp6XUhLJYRYm9P7xVeZ6RRHXonOA0JsSRu1s4ET1BRlRWI71YLjmYUo2gyklbqmWXA8oy3pasDCltzVZEgF8o7Poo9pB0bbxJIrK0PIPcrM3qaYvL+FuBwZdaZwmk91FaTtBjJ5/nwsvpjCuwMGYf7LYcLTZKh0ExYhfULfNens/2Iezo/aGAxaXV/Ow7lNPTye0OMnvcwISAtc5mDWyYRR1tfLu7eajAP8s5cppuN2cC9P3vKQdhjBspOn2DbnTTo/34Px2yKR1O1mi5A+YU3QpOHYJiFNEyFtMqSgjFrPqO6mDPnJj0ypitLoXxnVvSMDf/AkvVBn8uYJq4ney6paUu2YNCGb9IjtTB3QmZdHreJ8ahG1AotESPXaaOCAhsf33Fg08mNW+twmt7Ss5cYuBlTUP6Th8X03Fo8azwqfW89IlwalNIukm+FcSypqG+F3RkAqSK+yaXxveo63IyqniMTfP6PvS0NZeDaRQqVun7O+20+1Rwte5cTRb9rZXWUm51eM5uUuA5i2M5xHcnVNnWpTkKoLSb93g+u3ksnRimyWsZ6a1LCTHNhpx2ZbW2y32LNtmz3b9zjh7BtBcoFuyJgGyQ0/3N1cOXXqlM7mytnQJJSqisF8ZmYGweec2WOzGQcXH2JuxqFWqxsuM8Vdzmzdwsnrj1GodTNkQJdPBEkFpdSEm1bo8tCjS1FLl0ulLm9i4m7p19WwWuP3Co+55W3PDxPG8MORJAoVjVRsrR/2dpy89piSWn4Yn7TeK4yAFKGEh4Hb+Xr8JL5bu5bvJnzI9NXOXM8uoaXkoU4j7IQVk/qY8NxzA/ls4wmuZskpiD/L6smDMHtjIt9v9qSgWFmRxbYBqYAs3pttq9ay+5Q3Pi52LPp+Ax7xxZQ2UuZ6C6r6gIaizFucWjKG18zMmGDlQXBYOEEeh7BZMIOpXy5n36UUZOUjdQ05Ny/i7XGWM2fO6Gxn8QxPQVk5mg8MDGTZsmUsXLSQhQsXsmPHDuRyeXWKNR8E8gNW88GAPgxb4kVasW73qVLXzzW6Lmt1eeroCkqhuFpXgB5dydUPj4CAgFq6du3apUdXjcKn/iQoyEs8zfy/9WDKrgc8lus+iOreXSA/0IoPXzNj2E8epBbp+lH33Kf4bgykgKDM4X7EJfz8zuPrd4Wb6VJULRnsqyki/W4Ul3y98PQ8T3BMPFkyNZqyYtJuXMb7lCveYQ9QVAVUtAlIZdfYO3sE73//O2HxGWTGOTL343kcji1AqTMT/eTFVkrk2v/lxfYdmLQ3mUJFGcqiLB5EOrPso0EMHjmXPWHZ5S2dSlbA47xccnN1tzzypAo0lU37o0cZ+Ls4st1uE1uc3Im5qafF0mTg9u+PeaPXc7Qb8A0nkwprjzWo0NWlfQcmOiRRUKkrQatrnFbXHHaHZpWHiTVNV3ptXXG3W74l1RaKKoYNI3sxdU8C+YYCwLV+LBjPkFe0fnyNS2IBLdKjNBLSinoloFYqy+E09Jh58jpY+0pBbzdRQKMqLR82VOtofUgF8i8sZVgPM2YcTigfBwjyVK5H3iK9SNVMby+oiF7/Ll3amzBlfyqFVWsmgoxE528Z3LkLA2cdIK5Q0bTZPFkov20/w+27l9j9sw1+WbI68FUUiDreiW/m/MqOb4fwgokZnx++T0GtWqnV9V6FLseH9XS9odU1cz838xVNC5iQheFQpWupDRceFTeoq6q6aIoeEnXhDM7HXPCJTKVIpUEbuVoiiScq8CxeERkUZd8nzNeVYye8iEwr1lkaUJN7Jwh3l2Mcd9nBnMHd+GS3YUjV8b/z7dxf2f7t3+jc0YwZh+6S31j3uEqsMf8/EaTGJPCMz219SGUELv07nTuOYIN2nKJtOTUag7CU5d4j7FIAF/398dfZLgbFkqFUN3CtHki1SeW48tWrprTrMoW9dwtoUp1RpRMbm4FCFsnG6YtxT5c2EDggJ8rucywcr5McZMXobib0/Mc+4vJ1HwR6IK3U9fVrnWjfZQoOt/MpaUq3X6vrZoUu6+k/6dFVUcGE4lgOL7Nkb0AUUUEHWT7rexyuSpCryyiM98Nu7ije/8oKuy27+f3QXlbN/ADzFWdIkSoRBBl33WxYZXMUv8hoQs6swbx3ZybtMgSpnOits7DYd42kwDWM7dGRXp86cPNxif4HkCaX++GXCbxYu5z9LwYRm6HQeWDUgUaEtI4htb8a/6qaOo7N73fDpL8F7pnF6MYV1751zTfFgwCOH3Ziv6MjjjrbgROXSZGrjIIU1XU2jupC+78OYfnlHGR65n9qUtd+EhA0Um5772P3sVDSZPXTFPIvsuKjCSw95MPFcw7MGWRKu27j2RZT+SAqv6F+SFFrdXWl/V/fYFmQhGIjdN3xdizXlVpcX1dVPjS5AWxZYItvmpQSWQIOUwcy3iYUiUyNqigd1+9ep9c4S9yCb5CYlkr0ZnN6j1lHiKSY0rTTLPpgKlY+d5Eo1CiyfVj81kt8aqAl1fqxctxElh70xl/rxxudaN/tY+yjc5HrG9IoEghyPozT/trl7HjgOJeTZVQN2aryVP2/CGm1FQ19MBpSTfYxZvU2pff0wyRIlQ0AVj+ZsiIJaQ9TSElJJjm5ZktJz6OkTNtlq/tPf0uqhdRaC2m74ayPytNfYWrdUkGCvzOufpHcu5NAWnGZzkys9kQNGad/wPyTRWzcuRfH/XvY8MXfebHdi4zZFF4+c11xOwOQqm5gPborz7cbzrrIHOqub9eSU/1FQcJFFwO6qk8snyxJiLqMr9sxDh9x4qexPRg035OM8smcUoJ/eRuzf+0jIb+k3E+Z61f0HbaU85mF5Hj+wKBXZ3MkuaAi2ko7Jh1laEyqIcNtPhM+XcSGnQ6VfrxFl3YvMtq64sFQo0znU1kxkrRUUlJSapVzckp6edic3rkdEVIdE+t/NBrSkkvL+Hv3v2Fx8gHSJk7lltx0xW69FZYrV7JSZ1tl40qcVNlA90k/pJosZ2b360jHNxbhm9G0llyT7c3KWbNZsm4L1kvt8Ko7JlXHc+CLqaxwi+L2gySSkpJIDLdn8ismdB5uRVC2jIqGUT+kWl1z+pvScfBCfNKKmtTD0Ei8sZw1hyVrN1fr0hdXrJEEs3vxYmxPXiIq9hp7P+/L4B/dySh/+6IC0n6zDpJaUBG/WuL2Df3eWYJvZiGZR2bwSp+ZHExpIqTqeJy+nMaK05Hciq/xY0rvjnQevprALD2+l8ThZv8rayxrl/NKSxtOxRag0Ne7eGJI1TwMPMhOO1s2bdpUZ7Nlt9ddZC36akx9oMr3tOqYVJByZfVYhn2xh7BHTV+XUufcJzIshODg4Frblav3yWkwsEBF1LqKiaPJjjUTR2XSBHw3TKRfz7eYuzeC7JKmBSUI8mQigsOJjo4hIuQGqSW642AB6VUbPpm2gcsZxTVrbapEnKb3peMLb7LEN53i8n69iqjKiaPJ+2omjjTluibRX6vL4SpZugvbespRu1urK7JKV2iFroZ7khqy3X7krWHfcfxeLoqyAly/6s/r37tWQxq09E3MPncipRLS4lNzMXtrEV4ZUgrD1jKi11AWeCRTWCqgyfVh4Zvd+dD6WgNvjmj9sOUf/9zApXSdh40qkd9n9Mf0hTf5ySeVolohNpWZVOcQHxVGSJ1yDr5ylXsSJXXfja625okh1SBNiSXqahihoaEE2M1hrq03F4JCCQ0NI/pBHiq9iVan3vwfDEGqlnDTz4uAhl4W16OkSS1paW4Sd+8ncCdgNwu+seQ/ERnI660ca1A3iyFq0kJPYDXhFZ7/r/9mwKeW2Npvx36rLWt+XsD8BcvYdMSfOzmKOl1WPTk0uFtDRtgRVk8dhNno+ewLSEJeuc4pueaO9dR+dHiuPWYTlrLTPYbYoKNYTazRZVNX12F/bkuaQ1dd0QLSAEtGvj6ab21/44CTE+v/+So9hs/BxjmCm0GH+GlMd0wHzcD+/AMKU0JxshjKC12HY7E3gMTsaxxe8DHvmX/LKtsd7Nm/A4t3XmLIVEuORjzSCVKo8MNq2mD6jv6RvRcTq9ejJdfd2TStPyZ/bY+Z+c/sOHOdXKVOlE1dycZ8bwBSddJFft+5mfXaX0NY8yu2W+3ZamfPXueAco+rq5ogoF0e0W7Sk/P44Vg6eTJN5T5jRBhzrhq5XFm9xFfvSgOQaiRe/GL+Lu/O2MFVSVUPrd4dau1oFFJ1ZgDb5k1l3PvvM/rt13h9zFesdXDhXMg17iRl8rhYRn7abYJPOXFWu05aD95a6TXhiwZpxl2ignzwdHfHJzCcyOhooiIjCAkOIUob3aRdSK4/kG3CveueIlCccYswf298gyK5kyZFXX5jAXl2AjdDL+Dl6YGnXwjXHmQhSb2joyusWldocAiRzaqrrk7QFCYQ5uXKaZ8gwq/FERvqwVFnb0LvZpKdcoNQP088fAKJSc5HWZDKjSvn8fI4x+XYhxSolDxOjOSi51m8AsKIjrtLuO9pfC9HcC9LVplnbZo1fpwLiuR2WmFlkECVH37VfsTEZ+uZT6ivvdE9DUCqKUgh5ugC3uvfk57ma/AMCuXqeQfmTxqL+Te7Cc2S1xtSFJVDmtFIgEajagycICBP8GXXmtWsX7uUxVZ7uZhQWH9CzACkQslDIjwc+OHbzYRJimjKTzo1AqlA8cMYLvv74HH6GAd2bGTVz/P5ZtY0Jn00lrEfjMN80iQmfPQRk+ftIPCh7rqcgbz+IQ7VfgroXct+hnkRypTI5dqlDAEEFYoSJWXlrUhVy1HRomjjM8sDOapamUqN2uu1UTHaVket0q5p186j4azUPteoSw3fGBqAVHtJaeQ63uv2PCaT95KUX4JGkYbLN6/TpevbLDv3qF6Xu6UhFQrC2TV3NB9a/Iav/wmWTXyHjxYc5VZ+nWUpA5AiFJMQ5MJ/vK6TXaJqYNK0vlmNQqoqKamI6tCUIpfmkJ54hxsRwVz0duWI4062bFiP9baDuEekIC1fXK+fiLhHdMCgA3ogVcVsYGT3DnSshFSglCsr36GriRlfn06jsM4sW8tCqiHbcyFDX+7LdKc75MkLCfplOC/3mYR9pKT2bL5BSJU8Tk0iPV9JWVPW0uEp/j6pUIZSJiU/N4fcQm3Xo/aT1mChiAdFB3QdaCKkpZIQrM178+KrM3GK012/rrhZy0KqIGj523QzHcqKy48oUpeR6jiVXp16Mv1wEvlVUXFaKYYg1a7ZG4lKIy2prpPiZ9GBFnKgEUjbDZzCggXfMfvTdxkwaDzLDoY1GJCizrjBjbSS+mPE5pAt5HH8iz50MhnNphs5FJcJ5B6dRe9OJozdEkeuTKdZNAip8WJESI33TLyiuR1oBNLnRyzj5MmtfDnkBdr3MGd7VE4Tg1iaUahQgMscM14wGcXGaxKKyzRkOH3GK6amjNt+l1y5CGkzui3eqs050Aik5WNSSRpXt35C/849GWN1gbTipk26NF9elUSuH0kP0yEs9stAqiolZsNoepgOZJ5HWvn6c3VaYktabYX44c/igD5Io3+tmDiapJ3dlaPI8GX5qJfo8upn7L6a1cBafUsaIiANt2Z8n16Y20UhkSZxcHo/Xhr6b9wS6rzCJ0LakgUh3rtVHGgAUnVaOM5rJtHH5Dme+5/p2LhEIJEXcN/NkkmvmzFk8o/Yed1H1jwvMDcp24I8hYv2X2E+eR7r1s9j4of/wvJYDFkldYI6REib5Kd40h/JgQYg1UjTuRMRiLeHO+6+l4i+r205NZQVpXI9yJOTJz0Jjc+ltOY3a55BjgWUOfcID7zAhXM+nA+OJa1Q92dzKiWIkD6DshCTeLYONABp+euF+pYqBA2qUlV5IMezFVqZmqBGqTQQDCJC2irFIibakg40CGlLJtjC9xYhbWGDxds/ewdESA16Lq6TGrRHPPhMHNBCavY60zae4nxQKLGpbeT3gI3MvCBL51bUVUIubGfGwF78a7/4R4SNtFA8vc06oHpIqPtZzofe4Pa9B6TmKShrntecnmmWhdLHpCc/IP72VS6ccePyg+Jm+fVMsSV9psUoJtawAwJlahUqlXZT67w61/DZbXavoKnJh1rd5AD6xvIjQtqYQ+Jx0YFWdkCEtJULQExedKAxB0RIG3NIPC460MoOiJC2cgGIyYsONOaACGljDonHRQda2QER0lYuADF50YHGHBAhbcwh8bjoQCs7IELaygUgJi860JgDIqSNOSQeFx1oZQdESFu5AMTkRQcac0CEtDGHxOOiA63swP8B060KAVwSYHwAAAAASUVORK5CYII=)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TcNXtDMFVLii" + }, + "source": [ + "#### Architecture" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kyKDfxNfBWl-" + }, + "source": [ + "![](https://github.com/recohut/reco-static/raw/master/media/images/120222_ncf.png)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "4i1YYbJB4oGQ" + }, + "source": [ + "class NGCF(nn.Module):\n", + " def __init__(self, n_users, n_items, emb_dim, layers, reg, node_dropout, mess_dropout,\n", + " adj_mtx):\n", + " super().__init__()\n", + "\n", + " # initialize Class attributes\n", + " self.n_users = n_users\n", + " self.n_items = n_items\n", + " self.emb_dim = emb_dim\n", + " self.adj_mtx = adj_mtx\n", + " self.laplacian = adj_mtx - sp.eye(adj_mtx.shape[0])\n", + " self.reg = reg\n", + " self.layers = layers\n", + " self.n_layers = len(self.layers)\n", + " self.node_dropout = node_dropout\n", + " self.mess_dropout = mess_dropout\n", + "\n", + " #self.u_g_embeddings = nn.Parameter(torch.empty(n_users, emb_dim+np.sum(self.layers)))\n", + " #self.i_g_embeddings = nn.Parameter(torch.empty(n_items, emb_dim+np.sum(self.layers)))\n", + "\n", + " # Initialize weights\n", + " self.weight_dict = self._init_weights()\n", + " print(\"Weights initialized.\")\n", + "\n", + " # Create Matrix 'A', PyTorch sparse tensor of SP adjacency_mtx\n", + " self.A = self._convert_sp_mat_to_sp_tensor(self.adj_mtx)\n", + " self.L = self._convert_sp_mat_to_sp_tensor(self.laplacian)\n", + "\n", + " # initialize weights\n", + " def _init_weights(self):\n", + " print(\"Initializing weights...\")\n", + " weight_dict = nn.ParameterDict()\n", + "\n", + " initializer = torch.nn.init.xavier_uniform_\n", + " \n", + " weight_dict['user_embedding'] = nn.Parameter(initializer(torch.empty(self.n_users, self.emb_dim).to(device)))\n", + " weight_dict['item_embedding'] = nn.Parameter(initializer(torch.empty(self.n_items, self.emb_dim).to(device)))\n", + "\n", + " weight_size_list = [self.emb_dim] + self.layers\n", + "\n", + " for k in range(self.n_layers):\n", + " weight_dict['W_gc_%d' %k] = nn.Parameter(initializer(torch.empty(weight_size_list[k], weight_size_list[k+1]).to(device)))\n", + " weight_dict['b_gc_%d' %k] = nn.Parameter(initializer(torch.empty(1, weight_size_list[k+1]).to(device)))\n", + " \n", + " weight_dict['W_bi_%d' %k] = nn.Parameter(initializer(torch.empty(weight_size_list[k], weight_size_list[k+1]).to(device)))\n", + " weight_dict['b_bi_%d' %k] = nn.Parameter(initializer(torch.empty(1, weight_size_list[k+1]).to(device)))\n", + " \n", + " return weight_dict\n", + "\n", + " # convert sparse matrix into sparse PyTorch tensor\n", + " def _convert_sp_mat_to_sp_tensor(self, X):\n", + " \"\"\"\n", + " Convert scipy sparse matrix to PyTorch sparse matrix\n", + " Arguments:\n", + " ----------\n", + " X = Adjacency matrix, scipy sparse matrix\n", + " \"\"\"\n", + " coo = X.tocoo().astype(np.float32)\n", + " i = torch.LongTensor(np.mat([coo.row, coo.col]))\n", + " v = torch.FloatTensor(coo.data)\n", + " res = torch.sparse.FloatTensor(i, v, coo.shape).to(device)\n", + " return res\n", + "\n", + " # apply node_dropout\n", + " def _droupout_sparse(self, X):\n", + " \"\"\"\n", + " Drop individual locations in X\n", + " \n", + " Arguments:\n", + " ---------\n", + " X = adjacency matrix (PyTorch sparse tensor)\n", + " dropout = fraction of nodes to drop\n", + " noise_shape = number of non non-zero entries of X\n", + " \"\"\"\n", + " \n", + " node_dropout_mask = ((self.node_dropout) + torch.rand(X._nnz())).floor().bool().to(device)\n", + " i = X.coalesce().indices()\n", + " v = X.coalesce()._values()\n", + " i[:,node_dropout_mask] = 0\n", + " v[node_dropout_mask] = 0\n", + " X_dropout = torch.sparse.FloatTensor(i, v, X.shape).to(X.device)\n", + "\n", + " return X_dropout.mul(1/(1-self.node_dropout))\n", + "\n", + " def forward(self, u, i, j):\n", + " \"\"\"\n", + " Computes the forward pass\n", + " \n", + " Arguments:\n", + " ---------\n", + " u = user\n", + " i = positive item (user interacted with item)\n", + " j = negative item (user did not interact with item)\n", + " \"\"\"\n", + " # apply drop-out mask\n", + " A_hat = self._droupout_sparse(self.A) if self.node_dropout > 0 else self.A\n", + " L_hat = self._droupout_sparse(self.L) if self.node_dropout > 0 else self.L\n", + "\n", + " ego_embeddings = torch.cat([self.weight_dict['user_embedding'], self.weight_dict['item_embedding']], 0)\n", + "\n", + " all_embeddings = [ego_embeddings]\n", + "\n", + " # forward pass for 'n' propagation layers\n", + " for k in range(self.n_layers):\n", + "\n", + " # weighted sum messages of neighbours\n", + " side_embeddings = torch.sparse.mm(A_hat, ego_embeddings)\n", + " side_L_embeddings = torch.sparse.mm(L_hat, ego_embeddings)\n", + "\n", + " # transformed sum weighted sum messages of neighbours\n", + " sum_embeddings = torch.matmul(side_embeddings, self.weight_dict['W_gc_%d' % k]) + self.weight_dict['b_gc_%d' % k]\n", + "\n", + " # bi messages of neighbours\n", + " bi_embeddings = torch.mul(ego_embeddings, side_L_embeddings)\n", + " # transformed bi messages of neighbours\n", + " bi_embeddings = torch.matmul(bi_embeddings, self.weight_dict['W_bi_%d' % k]) + self.weight_dict['b_bi_%d' % k]\n", + "\n", + " # non-linear activation \n", + " ego_embeddings = F.leaky_relu(sum_embeddings + bi_embeddings)\n", + " # + message dropout\n", + " mess_dropout_mask = nn.Dropout(self.mess_dropout)\n", + " ego_embeddings = mess_dropout_mask(ego_embeddings)\n", + "\n", + " # normalize activation\n", + " norm_embeddings = F.normalize(ego_embeddings, p=2, dim=1)\n", + "\n", + " all_embeddings.append(norm_embeddings)\n", + "\n", + " all_embeddings = torch.cat(all_embeddings, 1)\n", + " \n", + " # back to user/item dimension\n", + " u_g_embeddings, i_g_embeddings = all_embeddings.split([self.n_users, self.n_items], 0)\n", + "\n", + " self.u_g_embeddings = nn.Parameter(u_g_embeddings)\n", + " self.i_g_embeddings = nn.Parameter(i_g_embeddings)\n", + " \n", + " u_emb = u_g_embeddings[u] # user embeddings\n", + " p_emb = i_g_embeddings[i] # positive item embeddings\n", + " n_emb = i_g_embeddings[j] # negative item embeddings\n", + "\n", + " y_ui = torch.mul(u_emb, p_emb).sum(dim=1)\n", + " y_uj = torch.mul(u_emb, n_emb).sum(dim=1)\n", + " log_prob = (torch.log(torch.sigmoid(y_ui-y_uj))).mean()\n", + "\n", + " # compute bpr-loss\n", + " bpr_loss = -log_prob\n", + " if self.reg > 0.:\n", + " l2norm = (torch.sum(u_emb**2)/2. + torch.sum(p_emb**2)/2. + torch.sum(n_emb**2)/2.) / u_emb.shape[0]\n", + " l2reg = self.reg*l2norm\n", + " bpr_loss = -log_prob + l2reg\n", + "\n", + " return bpr_loss" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5xbcqHLUSowG" + }, + "source": [ + "### Training and Evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N6S7uZU3Tpht" + }, + "source": [ + "Training is done using the standard PyTorch method. If you are already familiar with PyTorch, the following code should look familiar.\n", + "\n", + "One of the most useful functions of PyTorch is the torch.nn.Sequential() function, that takes existing and custom torch.nn modules. This makes it very easy to build and train complete networks. However, due to the nature of NCGF model structure, usage of torch.nn.Sequential() is not possible and the forward pass of the network has to be implemented ‘manually’. Using the Bayesian personalized ranking (BPR) pairwise loss, the forward pass is implemented as follows:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "LEMcstCz4vSm", + "outputId": "4e06cd7c-8e69-4f6b-d4b8-054d373f01cb" + }, + "source": [ + "# read parsed arguments\n", + "args = parse_args()\n", + "data_dir = args.data_dir\n", + "dataset = args.dataset\n", + "batch_size = args.batch_size\n", + "layers = eval(args.layers)\n", + "emb_dim = args.emb_dim\n", + "lr = args.lr\n", + "reg = args.reg\n", + "mess_dropout = args.mess_dropout\n", + "node_dropout = args.node_dropout\n", + "k = args.k\n", + "\n", + "# generate the NGCF-adjacency matrix\n", + "data_generator = Data(path=data_dir + dataset, batch_size=batch_size)\n", + "adj_mtx = data_generator.get_adj_mat()\n", + "\n", + "# create model name and save\n", + "modelname = \"NGCF\" + \\\n", + " \"_bs_\" + str(batch_size) + \\\n", + " \"_nemb_\" + str(emb_dim) + \\\n", + " \"_layers_\" + str(layers) + \\\n", + " \"_nodedr_\" + str(node_dropout) + \\\n", + " \"_messdr_\" + str(mess_dropout) + \\\n", + " \"_reg_\" + str(reg) + \\\n", + " \"_lr_\" + str(lr)\n", + "\n", + "# create NGCF model\n", + "model = NGCF(data_generator.n_users, \n", + " data_generator.n_items,\n", + " emb_dim,\n", + " layers,\n", + " reg,\n", + " node_dropout,\n", + " mess_dropout,\n", + " adj_mtx)\n", + "if use_cuda:\n", + " model = model.cuda()\n", + "\n", + "# current best metric\n", + "cur_best_metric = 0\n", + "\n", + "# Adam optimizer\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)\n", + "\n", + "# Set values for early stopping\n", + "cur_best_loss, stopping_step, should_stop = 1e3, 0, False\n", + "today = datetime.now()\n", + "\n", + "print(\"Start at \" + str(today))\n", + "print(\"Using \" + str(device) + \" for computations\")\n", + "print(\"Params on CUDA: \" + str(next(model.parameters()).is_cuda))\n", + "\n", + "results = {\"Epoch\": [],\n", + " \"Loss\": [],\n", + " \"Recall\": [],\n", + " \"NDCG\": [],\n", + " \"Training Time\": []}\n", + "\n", + "for epoch in range(args.n_epochs):\n", + "\n", + " t1 = time()\n", + " loss = train(model, data_generator, optimizer)\n", + " training_time = time()-t1\n", + " print(\"Epoch: {}, Training time: {:.2f}s, Loss: {:.4f}\".\n", + " format(epoch, training_time, loss))\n", + "\n", + " # print test evaluation metrics every N epochs (provided by args.eval_N)\n", + " if epoch % args.eval_N == (args.eval_N - 1):\n", + " with torch.no_grad():\n", + " t2 = time()\n", + " recall, ndcg = eval_model(model.u_g_embeddings.detach(),\n", + " model.i_g_embeddings.detach(),\n", + " data_generator.R_train,\n", + " data_generator.R_test,\n", + " k)\n", + " print(\n", + " \"Evaluate current model:\\n\",\n", + " \"Epoch: {}, Validation time: {:.2f}s\".format(epoch, time()-t2),\"\\n\",\n", + " \"Loss: {:.4f}:\".format(loss), \"\\n\",\n", + " \"Recall@{}: {:.4f}\".format(k, recall), \"\\n\",\n", + " \"NDCG@{}: {:.4f}\".format(k, ndcg)\n", + " )\n", + "\n", + " cur_best_metric, stopping_step, should_stop = \\\n", + " early_stopping(recall, cur_best_metric, stopping_step, flag_step=5)\n", + "\n", + " # save results in dict\n", + " results['Epoch'].append(epoch)\n", + " results['Loss'].append(loss)\n", + " results['Recall'].append(recall.item())\n", + " results['NDCG'].append(ndcg.item())\n", + " results['Training Time'].append(training_time)\n", + " else:\n", + " # save results in dict\n", + " results['Epoch'].append(epoch)\n", + " results['Loss'].append(loss)\n", + " results['Recall'].append(None)\n", + " results['NDCG'].append(None)\n", + " results['Training Time'].append(training_time)\n", + "\n", + " if should_stop == True: break\n", + "\n", + "# save\n", + "if args.save_results:\n", + " date = today.strftime(\"%d%m%Y_%H%M\")\n", + "\n", + " # save model as .pt file\n", + " if os.path.isdir(\"./models\"):\n", + " torch.save(model.state_dict(), \"./models/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".pt\")\n", + " else:\n", + " os.mkdir(\"./models\")\n", + " torch.save(model.state_dict(), \"./models/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".pt\")\n", + "\n", + " # save results as pandas dataframe\n", + " results_df = pd.DataFrame(results)\n", + " results_df.set_index('Epoch', inplace=True)\n", + " if os.path.isdir(\"./results\"):\n", + " results_df.to_csv(\"./results/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".csv\")\n", + " else:\n", + " os.mkdir(\"./results\")\n", + " results_df.to_csv(\"./results/\" + str(date) + \"_\" + modelname + \"_\" + dataset + \".csv\")\n", + " # plot loss\n", + " results_df['Loss'].plot(figsize=(12,8), title='Loss')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "n_users=943, n_items=1682\n", + "n_interactions=100000\n", + "n_train=80000, n_test=20000, sparsity=0.06305\n", + "Creating interaction matrices R_train and R_test...\n", + "Complete. Interaction matrices R_train and R_test created in 1.4850668907165527 sec\n", + "Loaded adjacency-matrix (shape: (2625, 2625) ) in 0.018111467361450195 sec.\n", + "Initializing weights...\n", + "Weights initialized.\n", + "Start at 2021-07-12 09:57:58.311285\n", + "Using cuda for computations\n", + "Params on CUDA: True\n", + "Epoch: 0, Training time: 9.11s, Loss: 107.9355\n", + "Epoch: 1, Training time: 8.88s, Loss: 101.6095\n", + "Epoch: 2, Training time: 8.75s, Loss: 80.7764\n", + "Epoch: 3, Training time: 8.76s, Loss: 76.1915\n", + "Epoch: 4, Training time: 8.58s, Loss: 73.0698\n", + "Evaluate current model:\n", + " Epoch: 4, Validation time: 1.51s \n", + " Loss: 73.0698: \n", + " Recall@20: 0.0623 \n", + " NDCG@20: 0.2352\n", + "Epoch: 5, Training time: 8.84s, Loss: 69.3378\n", + "Epoch: 6, Training time: 8.71s, Loss: 64.4498\n", + "Epoch: 7, Training time: 8.67s, Loss: 60.1440\n", + "Epoch: 8, Training time: 8.76s, Loss: 56.8538\n", + "Epoch: 9, Training time: 8.78s, Loss: 52.3951\n", + "Evaluate current model:\n", + " Epoch: 9, Validation time: 1.54s \n", + " Loss: 52.3951: \n", + " Recall@20: 0.0837 \n", + " NDCG@20: 0.2559\n", + "Epoch: 10, Training time: 8.72s, Loss: 50.5261\n", + "Epoch: 11, Training time: 8.73s, Loss: 49.2488\n", + "Epoch: 12, Training time: 8.72s, Loss: 48.5012\n", + "Epoch: 13, Training time: 8.75s, Loss: 47.5585\n", + "Epoch: 14, Training time: 8.82s, Loss: 47.0483\n", + "Evaluate current model:\n", + " Epoch: 14, Validation time: 1.51s \n", + " Loss: 47.0483: \n", + " Recall@20: 0.0926 \n", + " NDCG@20: 0.2676\n", + "Epoch: 15, Training time: 8.84s, Loss: 46.4847\n", + "Epoch: 16, Training time: 8.98s, Loss: 46.2644\n", + "Epoch: 17, Training time: 8.99s, Loss: 45.5963\n", + "Epoch: 18, Training time: 8.78s, Loss: 45.0955\n", + "Epoch: 19, Training time: 8.84s, Loss: 44.9321\n", + "Evaluate current model:\n", + " Epoch: 19, Validation time: 1.55s \n", + " Loss: 44.9321: \n", + " Recall@20: 0.1102 \n", + " NDCG@20: 0.2934\n", + "Epoch: 20, Training time: 8.61s, Loss: 44.4621\n", + "Epoch: 21, Training time: 9.02s, Loss: 44.1910\n", + "Epoch: 22, Training time: 8.94s, Loss: 43.7996\n", + "Epoch: 23, Training time: 8.83s, Loss: 43.1078\n", + "Epoch: 24, Training time: 9.01s, Loss: 43.1549\n", + "Evaluate current model:\n", + " Epoch: 24, Validation time: 1.54s \n", + " Loss: 43.1549: \n", + " Recall@20: 0.1217 \n", + " NDCG@20: 0.3255\n", + "Epoch: 25, Training time: 9.08s, Loss: 42.8759\n", + "Epoch: 26, Training time: 8.92s, Loss: 42.4126\n", + "Epoch: 27, Training time: 8.82s, Loss: 42.0810\n", + "Epoch: 28, Training time: 8.97s, Loss: 41.7865\n", + "Epoch: 29, Training time: 8.89s, Loss: 41.3096\n", + "Evaluate current model:\n", + " Epoch: 29, Validation time: 1.57s \n", + " Loss: 41.3096: \n", + " Recall@20: 0.1257 \n", + " NDCG@20: 0.3217\n", + "Epoch: 30, Training time: 9.15s, Loss: 40.9893\n", + "Epoch: 31, Training time: 9.11s, Loss: 40.8605\n", + "Epoch: 32, Training time: 9.06s, Loss: 40.3089\n", + "Epoch: 33, Training time: 8.87s, Loss: 40.1379\n", + "Epoch: 34, Training time: 8.89s, Loss: 39.6859\n", + "Evaluate current model:\n", + " Epoch: 34, Validation time: 1.51s \n", + " Loss: 39.6859: \n", + " Recall@20: 0.1293 \n", + " NDCG@20: 0.3432\n", + "Epoch: 35, Training time: 9.12s, Loss: 39.9238\n", + "Epoch: 36, Training time: 9.12s, Loss: 39.4329\n", + "Epoch: 37, Training time: 9.20s, Loss: 38.9671\n", + "Epoch: 38, Training time: 8.79s, Loss: 38.7849\n", + "Epoch: 39, Training time: 8.78s, Loss: 38.3410\n", + "Evaluate current model:\n", + " Epoch: 39, Validation time: 1.54s \n", + " Loss: 38.3410: \n", + " Recall@20: 0.1365 \n", + " NDCG@20: 0.3411\n", + "Epoch: 40, Training time: 8.85s, Loss: 38.6723\n", + "Epoch: 41, Training time: 8.78s, Loss: 37.9243\n", + "Epoch: 42, Training time: 9.07s, Loss: 37.8358\n", + "Epoch: 43, Training time: 8.85s, Loss: 37.2368\n", + "Epoch: 44, Training time: 8.97s, Loss: 37.4086\n", + "Evaluate current model:\n", + " Epoch: 44, Validation time: 1.51s \n", + " Loss: 37.4086: \n", + " Recall@20: 0.1383 \n", + " NDCG@20: 0.3554\n", + "Epoch: 45, Training time: 8.94s, Loss: 37.1695\n", + "Epoch: 46, Training time: 9.05s, Loss: 36.9502\n", + "Epoch: 47, Training time: 8.75s, Loss: 36.5551\n", + "Epoch: 48, Training time: 9.08s, Loss: 36.4953\n", + "Epoch: 49, Training time: 9.13s, Loss: 35.9976\n", + "Evaluate current model:\n", + " Epoch: 49, Validation time: 1.54s \n", + " Loss: 35.9976: \n", + " Recall@20: 0.1397 \n", + " NDCG@20: 0.3541\n", + "Epoch: 50, Training time: 8.79s, Loss: 35.8774\n", + "Epoch: 51, Training time: 9.03s, Loss: 36.0130\n", + "Epoch: 52, Training time: 9.00s, Loss: 35.4460\n", + "Epoch: 53, Training time: 8.76s, Loss: 35.2867\n", + "Epoch: 54, Training time: 9.11s, Loss: 35.4907\n", + "Evaluate current model:\n", + " Epoch: 54, Validation time: 1.53s \n", + " Loss: 35.4907: \n", + " Recall@20: 0.1435 \n", + " NDCG@20: 0.3563\n", + "Epoch: 55, Training time: 8.97s, Loss: 35.1628\n", + "Epoch: 56, Training time: 8.86s, Loss: 34.5842\n", + "Epoch: 57, Training time: 8.83s, Loss: 34.1935\n", + "Epoch: 58, Training time: 8.88s, Loss: 34.3039\n", + "Epoch: 59, Training time: 8.79s, Loss: 34.2499\n", + "Evaluate current model:\n", + " Epoch: 59, Validation time: 1.49s \n", + " Loss: 34.2499: \n", + " Recall@20: 0.1495 \n", + " NDCG@20: 0.3704\n", + "Epoch: 60, Training time: 8.84s, Loss: 33.9897\n", + "Epoch: 61, Training time: 8.66s, Loss: 33.2779\n", + "Epoch: 62, Training time: 8.86s, Loss: 33.2062\n", + "Epoch: 63, Training time: 8.78s, Loss: 32.9654\n", + "Epoch: 64, Training time: 9.03s, Loss: 32.2721\n", + "Evaluate current model:\n", + " Epoch: 64, Validation time: 1.51s \n", + " Loss: 32.2721: \n", + " Recall@20: 0.1497 \n", + " NDCG@20: 0.3725\n", + "Epoch: 65, Training time: 8.90s, Loss: 32.5445\n", + "Epoch: 66, Training time: 8.85s, Loss: 32.1805\n", + "Epoch: 67, Training time: 8.81s, Loss: 32.1525\n", + "Epoch: 68, Training time: 8.80s, Loss: 31.7560\n", + "Epoch: 69, Training time: 8.81s, Loss: 31.3688\n", + "Evaluate current model:\n", + " Epoch: 69, Validation time: 1.51s \n", + " Loss: 31.3688: \n", + " Recall@20: 0.1536 \n", + " NDCG@20: 0.3816\n", + "Epoch: 70, Training time: 8.55s, Loss: 31.3098\n", + "Epoch: 71, Training time: 8.87s, Loss: 31.3700\n", + "Epoch: 72, Training time: 8.72s, Loss: 31.1579\n", + "Epoch: 73, Training time: 8.76s, Loss: 30.1733\n", + "Epoch: 74, Training time: 8.76s, Loss: 30.5201\n", + "Evaluate current model:\n", + " Epoch: 74, Validation time: 1.50s \n", + " Loss: 30.5201: \n", + " Recall@20: 0.1581 \n", + " NDCG@20: 0.3809\n", + "Epoch: 75, Training time: 8.70s, Loss: 30.2994\n", + "Epoch: 76, Training time: 8.76s, Loss: 29.8949\n", + "Epoch: 77, Training time: 8.77s, Loss: 29.7122\n", + "Epoch: 78, Training time: 8.74s, Loss: 29.7030\n", + "Epoch: 79, Training time: 8.64s, Loss: 29.6655\n", + "Evaluate current model:\n", + " Epoch: 79, Validation time: 1.49s \n", + " Loss: 29.6655: \n", + " Recall@20: 0.1609 \n", + " NDCG@20: 0.3873\n", + "Epoch: 80, Training time: 8.94s, Loss: 29.6567\n", + "Epoch: 81, Training time: 8.87s, Loss: 29.5109\n", + "Epoch: 82, Training time: 8.91s, Loss: 29.1704\n", + "Epoch: 83, Training time: 8.82s, Loss: 28.6625\n", + "Epoch: 84, Training time: 8.79s, Loss: 28.7304\n", + "Evaluate current model:\n", + " Epoch: 84, Validation time: 1.48s \n", + " Loss: 28.7304: \n", + " Recall@20: 0.1613 \n", + " NDCG@20: 0.3908\n", + "Epoch: 85, Training time: 8.85s, Loss: 29.0495\n", + "Epoch: 86, Training time: 8.76s, Loss: 28.4390\n", + "Epoch: 87, Training time: 8.81s, Loss: 28.5633\n", + "Epoch: 88, Training time: 8.83s, Loss: 28.3275\n", + "Epoch: 89, Training time: 8.96s, Loss: 27.8343\n", + "Evaluate current model:\n", + " Epoch: 89, Validation time: 1.52s \n", + " Loss: 27.8343: \n", + " Recall@20: 0.1591 \n", + " NDCG@20: 0.3895\n", + "Epoch: 90, Training time: 8.92s, Loss: 28.3271\n", + "Epoch: 91, Training time: 8.85s, Loss: 28.0346\n", + "Epoch: 92, Training time: 8.69s, Loss: 27.7937\n", + "Epoch: 93, Training time: 8.93s, Loss: 27.5649\n", + "Epoch: 94, Training time: 9.08s, Loss: 27.9189\n", + "Evaluate current model:\n", + " Epoch: 94, Validation time: 1.50s \n", + " Loss: 27.9189: \n", + " Recall@20: 0.1611 \n", + " NDCG@20: 0.3912\n", + "Epoch: 95, Training time: 8.86s, Loss: 27.9343\n", + "Epoch: 96, Training time: 8.83s, Loss: 27.2735\n", + "Epoch: 97, Training time: 8.92s, Loss: 27.3794\n", + "Epoch: 98, Training time: 8.84s, Loss: 27.2788\n", + "Epoch: 99, Training time: 8.86s, Loss: 27.4216\n", + "Evaluate current model:\n", + " Epoch: 99, Validation time: 1.50s \n", + " Loss: 27.4216: \n", + " Recall@20: 0.1656 \n", + " NDCG@20: 0.3922\n", + "Epoch: 100, Training time: 8.71s, Loss: 26.6066\n", + "Epoch: 101, Training time: 8.88s, Loss: 27.1389\n", + "Epoch: 102, Training time: 9.04s, Loss: 26.6459\n", + "Epoch: 103, Training time: 8.71s, Loss: 26.8171\n", + "Epoch: 104, Training time: 8.91s, Loss: 26.7730\n", + "Evaluate current model:\n", + " Epoch: 104, Validation time: 1.49s \n", + " Loss: 26.7730: \n", + " Recall@20: 0.1627 \n", + " NDCG@20: 0.3926\n", + "Epoch: 105, Training time: 9.06s, Loss: 26.4580\n", + "Epoch: 106, Training time: 9.12s, Loss: 25.9192\n", + "Epoch: 107, Training time: 8.93s, Loss: 26.4427\n", + "Epoch: 108, Training time: 8.77s, Loss: 26.3804\n", + "Epoch: 109, Training time: 8.86s, Loss: 26.1349\n", + "Evaluate current model:\n", + " Epoch: 109, Validation time: 1.52s \n", + " Loss: 26.1349: \n", + " Recall@20: 0.1691 \n", + " NDCG@20: 0.3950\n", + "Epoch: 110, Training time: 8.81s, Loss: 25.8410\n", + "Epoch: 111, Training time: 8.84s, Loss: 25.9275\n", + "Epoch: 112, Training time: 8.77s, Loss: 25.9278\n", + "Epoch: 113, Training time: 8.92s, Loss: 26.2235\n", + "Epoch: 114, Training time: 8.90s, Loss: 25.4737\n", + "Evaluate current model:\n", + " Epoch: 114, Validation time: 1.50s \n", + " Loss: 25.4737: \n", + " Recall@20: 0.1673 \n", + " NDCG@20: 0.3995\n", + "Epoch: 115, Training time: 8.78s, Loss: 25.7582\n", + "Epoch: 116, Training time: 8.77s, Loss: 25.3173\n", + "Epoch: 117, Training time: 8.63s, Loss: 25.4568\n", + "Epoch: 118, Training time: 8.63s, Loss: 25.3934\n", + "Epoch: 119, Training time: 8.63s, Loss: 25.2544\n", + "Evaluate current model:\n", + " Epoch: 119, Validation time: 1.50s \n", + " Loss: 25.2544: \n", + " Recall@20: 0.1689 \n", + " NDCG@20: 0.4028\n", + "Epoch: 120, Training time: 8.77s, Loss: 24.9747\n", + "Epoch: 121, Training time: 8.93s, Loss: 24.7825\n", + "Epoch: 122, Training time: 8.92s, Loss: 25.2147\n", + "Epoch: 123, Training time: 8.79s, Loss: 24.5176\n", + "Epoch: 124, Training time: 8.72s, Loss: 24.7453\n", + "Evaluate current model:\n", + " Epoch: 124, Validation time: 1.48s \n", + " Loss: 24.7453: \n", + " Recall@20: 0.1682 \n", + " NDCG@20: 0.3954\n", + "Epoch: 125, Training time: 8.78s, Loss: 24.9444\n", + "Epoch: 126, Training time: 8.81s, Loss: 24.9258\n", + "Epoch: 127, Training time: 8.77s, Loss: 24.5360\n", + "Epoch: 128, Training time: 8.70s, Loss: 24.4527\n", + "Epoch: 129, Training time: 8.65s, Loss: 24.5864\n", + "Evaluate current model:\n", + " Epoch: 129, Validation time: 1.48s \n", + " Loss: 24.5864: \n", + " Recall@20: 0.1689 \n", + " NDCG@20: 0.3977\n", + "Epoch: 130, Training time: 8.66s, Loss: 24.2351\n", + "Epoch: 131, Training time: 8.84s, Loss: 24.4298\n", + "Epoch: 132, Training time: 8.57s, Loss: 24.3624\n", + "Epoch: 133, Training time: 8.74s, Loss: 24.1980\n", + "Epoch: 134, Training time: 8.84s, Loss: 24.0672\n", + "Evaluate current model:\n", + " Epoch: 134, Validation time: 1.47s \n", + " Loss: 24.0672: \n", + " Recall@20: 0.1735 \n", + " NDCG@20: 0.4069\n", + "Epoch: 135, Training time: 8.75s, Loss: 24.4691\n", + "Epoch: 136, Training time: 8.67s, Loss: 23.9019\n", + "Epoch: 137, Training time: 8.77s, Loss: 24.1378\n", + "Epoch: 138, Training time: 8.68s, Loss: 23.8090\n", + "Epoch: 139, Training time: 8.81s, Loss: 23.9487\n", + "Evaluate current model:\n", + " Epoch: 139, Validation time: 1.48s \n", + " Loss: 23.9487: \n", + " Recall@20: 0.1687 \n", + " NDCG@20: 0.4037\n", + "Epoch: 140, Training time: 8.64s, Loss: 23.8015\n", + "Epoch: 141, Training time: 8.57s, Loss: 24.0985\n", + "Epoch: 142, Training time: 8.70s, Loss: 23.8640\n", + "Epoch: 143, Training time: 8.77s, Loss: 23.5799\n", + "Epoch: 144, Training time: 8.77s, Loss: 23.7568\n", + "Evaluate current model:\n", + " Epoch: 144, Validation time: 1.48s \n", + " Loss: 23.7568: \n", + " Recall@20: 0.1708 \n", + " NDCG@20: 0.4068\n", + "Epoch: 145, Training time: 8.75s, Loss: 23.6537\n", + "Epoch: 146, Training time: 8.77s, Loss: 23.8114\n", + "Epoch: 147, Training time: 8.64s, Loss: 23.5442\n", + "Epoch: 148, Training time: 8.51s, Loss: 23.2413\n", + "Epoch: 149, Training time: 8.77s, Loss: 23.5159\n", + "Evaluate current model:\n", + " Epoch: 149, Validation time: 1.49s \n", + " Loss: 23.5159: \n", + " Recall@20: 0.1698 \n", + " NDCG@20: 0.4052\n", + "Epoch: 150, Training time: 8.67s, Loss: 23.4435\n", + "Epoch: 151, Training time: 8.54s, Loss: 23.5388\n", + "Epoch: 152, Training time: 8.54s, Loss: 23.2494\n", + "Epoch: 153, Training time: 8.60s, Loss: 23.1259\n", + "Epoch: 154, Training time: 8.68s, Loss: 23.1326\n", + "Evaluate current model:\n", + " Epoch: 154, Validation time: 1.49s \n", + " Loss: 23.1326: \n", + " Recall@20: 0.1709 \n", + " NDCG@20: 0.4059\n", + "Epoch: 155, Training time: 8.69s, Loss: 22.8828\n", + "Epoch: 156, Training time: 8.60s, Loss: 23.0292\n", + "Epoch: 157, Training time: 8.54s, Loss: 22.9355\n", + "Epoch: 158, Training time: 8.61s, Loss: 22.7000\n", + "Epoch: 159, Training time: 8.83s, Loss: 22.9723\n", + "Evaluate current model:\n", + " Epoch: 159, Validation time: 1.47s \n", + " Loss: 22.9723: \n", + " Recall@20: 0.1731 \n", + " NDCG@20: 0.4118\n", + "Early stopping at step: 5 log:0.1731223464012146\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsUAAAHwCAYAAABOlBKbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3zd5X33//d1ls7R3tOSlzzkiY0xZhMgCZvQ7EFIQ0Lvlmb3kdk2d9M0bXqnd0hzN00hJCG/JCQpGSYJGZhA2QYbG+8hW7b23jo60hnX7w/JRhgZrPk90vf1fDz8sM/3DH30fQjz5uJzfS5jrRUAAADgZh6nCwAAAACcRigGAACA6xGKAQAA4HqEYgAAALgeoRgAAACuRygGAACA6xGKAQAA4HqEYgBIEsaYE8aYa5yuAwDciFAMAAAA1yMUA0ASM8akGGPuNsY0jv662xiTMvpcvjHmN8aYbmNMpzHmSWOMZ/S5zxhjGowxfcaYw8aYq539TgAgufmcLgAA8Jq+IGmLpPMkWUlbJf2tpL+T9ClJ9ZIKRl+7RZI1xqyQ9NeSLrDWNhpjFknyzm7ZADC3sFIMAMntvZK+ZK1ttda2SfoHSbeNPheVVCJpobU2aq190lprJcUlpUhaZYzxW2tPWGuPOVI9AMwRhGIASG6lkk6OeXxy9Jok/R9J1ZL+aIw5boz5rCRZa6slfVzS/5bUaoz5iTGmVACAsyIUA0Bya5S0cMzjitFrstb2WWs/Za1dIulmSZ881Ttsrf2xtfbS0fdaSV+d3bIBYG4hFANAcvEbY4Knfkl6QNLfGmMKjDH5kv5e0g8lyRhzozGm0hhjJPVopG0iYYxZYYy5anRDXkTSoKSEM98OAMwNhGIASC4PayTEnvoVlLRD0h5JeyW9KOnLo69dJmmbpH5Jz0r6lrX2MY30E/+LpHZJzZIKJX1u9r4FAJh7zMieDAAAAMC9WCkGAACA6xGKAQAA4HqEYgAAALgeoRgAAACuRygGAACA6/mcLkCS8vPz7aJFi5wuAwAAAPPczp072621BWdeT4pQvGjRIu3YscPpMgAAADDPGWNOjned9gkAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6hGIAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6hGIAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6hGIAAAC4HqEYAAAArkcoBgAAgOsRigEAAOB6rg3F1lp1h4c1FIs7XQoAAAAc5tpQ/HR1h8770iPaXdvtdCkAAABwmGtD8YKckCSprmvQ4UoAAADgNNeG4tLskIyR6jrDTpcCAAAAh7k2FAd8HpVmhQjFAAAAcG8olkZaKOq6CMUAAABu5+pQXJ6bqlpWigEAAFzP1aG4IjdVLb1DikQZywYAAOBmrg7F5bkjEygauplAAQAA4GauDsUVuamSRAsFAACAy7k6FJfnjITiekIxAACAq7k6FBdkpCjF52GlGAAAwOVcHYqNMSrPTVVdJz3FAAAAbubqUCxJ5cwqBgAAcD1CMbOKAQAAXM/1obgiN1V9kZh6wlGnSwEAAIBDXB+KF+Qwlg0AAMDtXB+KT80qpq8YAADAvVwfik+dalfHSjEAAIBruT4UZwT9yk710z4BAADgYq4PxdJIC0VdF7OKAQAA3IpQrJHjnmmfAAAAcC9CsaQFuSE1dA0qkbBOlwIAAAAHEIo10j4xHE+opS/idCkAAABwAKFYI+0TklTbQQsFAACAGxGKNXZWMZvtAAAA3IhQLKk0OyRjONUOAADArQjFkgI+j0oyg6onFAMAALgSoXhUeW4qK8UAAAAuRSgeVZ6bqrouQjEAAIAbEYpHleekqqV3SJFo3OlSAAAAMMsIxaMKMlIkSV3hYYcrAQAAwGwjFI9KDXglSYPDrBQDAAC4DaF4VNA/GoppnwAAAHAdQvGo0OhKMT3FAAAA7vO6odgY811jTKsxZt+Ya7nGmEeMMUdHf88ZvW6MMf9ujKk2xuwxxmycyeKnU+jUSvFwwuFKAAAAMNvOZaX4+5KuPePaZyU9aq1dJunR0ceSdJ2kZaO/7pT0n9NT5swL0T4BAADgWq8biq21T0jqPOPyLZLuH/3z/ZLeMub6D+yI5yRlG2NKpqvYmRQKjNwKQjEAAID7TLanuMha2zT652ZJRaN/LpNUN+Z19aPXXsUYc6cxZocxZkdbW9sky5g+oYBPkjQ4HHO4EgAAAMy2KW+0s9ZaSXYS77vHWrvJWrupoKBgqmVM2cs9xawUAwAAuM1kQ3HLqbaI0d9bR683SCof87oFo9eS3ss9xWy0AwAAcJvJhuKHJN0++ufbJW0dc/39o1MotkjqGdNmkdRSfPQUAwAAuJXv9V5gjHlA0pWS8o0x9ZK+KOlfJP3MGHOHpJOS3jH68oclXS+pWlJY0p/PQM0zwuMxCvo9zCkGAABwodcNxdbad5/lqavHea2VdNdUi3JKyO+lpxgAAMCFONFujJDfS/sEAACACxGKxwgFWCkGAABwI0LxGKEAK8UAAABuRCgeg55iAAAAdyIUjxGkpxgAAMCVCMVjhPxeRrIBAAC4EKF4DHqKAQAA3IlQPAY9xQAAAO5EKB6DkWwAAADuRCgeg8M7AAAA3IlQPEbI71UsYRWNJ5wuBQAAALOIUDxGKOCVJFaLAQAAXIZQPEbQPxKKI/QVAwAAuAqheIyQn5ViAAAANyIUj0H7BAAAgDsRisc4HYppnwAAAHAVQvEYp9snCMUAAACuQigeg55iAAAAdyIUj0FPMQAAgDsRisegfQIAAMCdCMVjnJ5TzEoxAACAqxCKx6B9AgAAwJ0IxWO83D6RcLgSAAAAzCZC8Rhej1HA51E4GnO6FAAAAMwiQvEZQn6vImy0AwAAcBVC8RlCfi89xQAAAC5DKD5DKODVYJSeYgAAADchFJ8h6PcypxgAAMBlCMVnSA14mVMMAADgMoTiM9BTDAAA4D6E4jME/V6FaZ8AAABwFULxGUK0TwAAALgOofgMIb+HjXYAAAAuQyg+Az3FAAAA7kMoPkMwQCgGAABwG0LxGVL9Pg3HEoonrNOlAAAAYJYQis8QCozcEjbbAQAAuAeh+Awhv1eSaKEAAABwEULxGYKnQjETKAAAAFyDUHyGUICVYgAAALchFJ8hxEoxAACA6xCKz0BPMQAAgPsQis9A+wQAAID7EIrPcCoUR2ifAAAAcA1C8RlonwAAAHAfQvEZToXiMCvFAAAArkEoPkPwVPsEK8UAAACuQSg+AyPZAAAA3IdQfAa/1yOfx9BTDAAA4CKE4nGEAl5CMQAAgIsQiscR8nvpKQYAAHARQvE4QgEvPcUAAAAuQigeR8jvZSQbAACAixCKxxH001MMAADgJoTicdBTDAAA4C6E4nEwfQIAAMBdCMXjYKMdAACAuxCKxzHSPpFwugwAAADMEkLxOEJstAMAAHAVQvE4aJ8AAABwF0LxOE6NZEskrNOlAAAAYBYQiscR8nslSUMx+ooBAADcgFA8jpB/5LbQVwwAAOAOhOJxpAZ8kgjFAAAAbkEoHkcwMNI+wWY7AAAAdyAUj+NUTzFHPQMAALgDoXgcp0Ix7RMAAADuQCgeRygwclvCtE8AAAC4AqF4HEE/PcUAAABuQigeBz3FAAAA7kIoHgcj2QAAANyFUDyOEO0TAAAArkIoHkcwwIl2AAAAbkIoHkfA65HH0FMMAADgFoTicRhjFPJ7GckGAADgEoTiswgFvLRPAAAAuASh+CxCAa8irBQDAAC4AqH4LEJ+VooBAADcglB8FoRiAAAA9yAUn0XQ72VOMQAAgEsQis8iFPAykg0AAMAlCMVnwUg2AAAA9yAUnwU9xQAAAO5BKD6L1BSv+odiTpcBAACAWTClUGyM+YQxZr8xZp8x5gFjTNAYs9gYs90YU22M+akxJjBdxc6mRXlp6g5H1dE/5HQpAAAAmGGTDsXGmDJJH5W0yVq7RpJX0rskfVXS1621lZK6JN0xHYXOtpXFmZKkw819DlcCAACAmTbV9gmfpJAxxicpVVKTpKskPTj6/P2S3jLFr+GIlSUZkqSDhGIAAIB5b9Kh2FrbIOlrkmo1EoZ7JO2U1G2tPdWMWy+pbLz3G2PuNMbsMMbsaGtrm2wZMyY/PUX56Sk61NTrdCkAAACYYVNpn8iRdIukxZJKJaVJuvZc32+tvcdau8lau6mgoGCyZcyoqpIMHWwmFAMAAMx3U2mfuEZSjbW2zVoblfQLSZdIyh5tp5CkBZIaplijY6pKMnWkpV+xeMLpUgAAADCDphKKayVtMcakGmOMpKslHZD0mKS3jb7mdklbp1aic1YWZ2g4ltCJjgGnSwEAAMAMmkpP8XaNbKh7UdLe0c+6R9JnJH3SGFMtKU/SfdNQpyNOTaA42MRmOwAAgPnM9/ovOTtr7RclffGMy8clbZ7K5yaLpYVp8nmMDjX36qb1pU6XAwAAgBnCiXavIcXn1dKCdB1ipRgAAGBeIxS/jpUlGTrErGIAAIB5jVD8OlYWZ6qhe1A9g1GnSwEAAMAMIRS/jlMn23GIBwAAwPxFKH4dq0pGJlDQQgEAADB/EYpfR2FGinJS/TrEyXYAAADzFqH4dRhjtLI4k1nFAAAA8xih+BysLMnQ4eY+JRLW6VIAAAAwAwjF56CqOFOD0bhqO8NOlwIAAIAZQCg+B6cnUNBXDAAAMC8Ris/BssIMeYzoKwYAAJinCMXnIBTwalF+mg4wqxgAAGBeIhSfo/PKs7XjRCeb7QAAAOYhQvE5urQyX13hqA7SVwwAADDvEIrP0SWV+ZKkp6vbHa4EAAAA041QfI6KMoOqLEzXU9UdTpcCAACAaUYonoBLK/P1Qk2nhmJxp0sBAADANCIUT8AllfkajMa1q7bb6VIAAAAwjQjFE3Dhklx5DH3FAAAA8w2heAIyg36tL88mFAMAAMwzhOIJurQyXy/V96g3EnW6FAAAAEwTQvEEXbw0X/GE1fbjnU6XAgAAgGlCKJ6gjQuzFfR7aKEAAACYRwjFE5Ti82rz4jxCMQAAwDxCKJ6ES5bm6Whrv1p6I06XAgAAgGlAKJ6EU0c+P3OM1WIAAID5gFA8CatKMpWd6tezxzjyGQAAYD4gFE+Cx2O0sSJHL3KyHQAAwLxAKJ6kjRXZqm7tV0+YecUAAABzHaF4kjZU5EiSdtezWgwAADDXEYonaX15tjxGevFkl9OlAAAAYIoIxZOUnuLT8qIM7apjpRgAAGCuIxRPwYaKHO2q7VIiYZ0uBQAAAFNAKJ6CjRXZ6ovEdKyt3+lSAAAAMAWE4ik4tdluF6PZAAAA5jRC8RQsyU9TVsivF2vZbAcAADCXEYqnwOMx2lCRzUoxAADAHEconqIN5Tk60tqn3giHeAAAAMxVhOIp2rgwW9ZKLzGaDQAAYM4iFE/R+vJsGcNmOwAAgLmMUDxFmUG/lhWms9kOAABgDiMUT4ONFTnaVdvNIR4AAABzFKF4GmyoyFbPYFQ1HQNOlwIAAIBJIBRPg40c4gEAADCnEYqnwZKCdKX4PDrc3Ot0KQAAAJgEQvE08HqMlhak60hLv9OlAAAAYBIIxdNkeVG6jrb0OV0GAAAAJoFQPE2WF2eosSfCyXYAAABzEKF4miwvzJAkHaWFAgAAYM4hFE+T5UWnQjEtFAAAAHMNoXiaLMgJKeT3stkOAABgDiIUTxOPx6iyMF1HW1kpBgAAmGsIxdNoeVGGDjcTigEAAOYaQvE0Wl6Urta+IfWEmUABAAAwlxCKp9GpzXZHaKEAAACYUwjF02hZUbok6QgTKAAAAOYUQvE0KssOKS3g1RH6igEAAOYUQvE0MsaosiiDsWwAAABzDKF4mq0oYiwbAADAXEMonmbLizLU3j+szoFhp0sBAADAOSIUT7NlpyZQsNkOAABgziAUT7PlTKAAAACYcwjF06w4M6iMFB+hGAAAYA4hFE8zY4yWFzOBAgAAYC4hFM+A5UXpOtrSJ2ut06UAAADgHBCKZ8Cywgx1haNq72cCBQAAwFxAKJ4BK4pHJlAcau51uBIAAACcC0LxDFhVkilJOtBIKAYAAJgLCMUzICctoNKsoPYTigEAAOYEQvEMWVWapf2NPU6XAQAAgHNAKJ4hq0szdbx9QOHhmNOlAAAA4HUQimfImrIsWSsdbKKFAgAAINkRimfI6tKRzXb0FQMAACQ/QvEMKckKKifVr/0NhGIAAIBkRyieIcYYrS7N0v4mNtsBAAAkO0LxDFpdmqkjzf2KxhNOlwIAAIDXQCieQatKMzUcT+hoS7/TpQAAAOA1EIpn0OrSLEliXjEAAECSIxTPoMX5aQr5vUygAAAASHKE4hnk9RhVlWToAKEYAAAgqRGKZ9jq0iwdaOpVImGdLgUAAABnQSieYWvKMtU/FFNtZ9jpUgAAAHAWhOIZdmqz3T422wEAACStKYViY0y2MeZBY8whY8xBY8xFxphcY8wjxpijo7/nTFexc9GyonT5PIbNdgAAAElsqivF35D0e2vtSknrJR2U9FlJj1prl0l6dPSxa6X4vFpWlEEoBgAASGKTDsXGmCxJl0u6T5KstcPW2m5Jt0i6f/Rl90t6y1SLnOtWl2bqQGOPrGWzHQAAQDKaykrxYkltkr5njNlljPmOMSZNUpG1tmn0Nc2SiqZa5Fy3tixL7f3DaugedLoUAAAAjGMqodgnaaOk/7TWbpA0oDNaJezI0ui4y6PGmDuNMTuMMTva2tqmUEby27w4V5K0/Xinw5UAAABgPFMJxfWS6q2120cfP6iRkNxijCmRpNHfW8d7s7X2HmvtJmvtpoKCgimUkfxWFGUoJ9WvZ493OF0KAAAAxjHpUGytbZZUZ4xZMXrpakkHJD0k6fbRa7dL2jqlCucBj8fowsV5eo5QDAAAkJR8U3z/RyT9yBgTkHRc0p9rJGj/zBhzh6STkt4xxa8xL2xZkqvf729WXWdY5bmpTpcDAACAMaYUiq21uyVtGuepq6fyufPRlqV5kqTtNZ2EYgAAgCTDiXazZHlhhnLTArRQAAAAJCFC8SwZ6SvO1bPHCMUAAADJhlA8i7YsyVND96DqOsNOlwIAAIAxCMWzaMuSkb5iWigAAACSC6F4Fi0rTB/tK+YQDwAAgGRCKJ5FHo/RliW5eu54h0YO+wMAAEAyIBTPslN9xfVdg06XAgAAgFGE4ll2qq+YI58BAACSB6F4li0rTFce84oBAACSCqF4lhljdOGSXD1fw2Y7AACAZEEodsDasmzVdw2qJxx1uhQAAACIUOyIVaWZkqQDTb0OVwIAAACJUOyIqpIMSdJBQjEAAEBSIBQ7oDAjqPz0FEIxAABAkiAUO6SqJIP2CQAAgCRBKHbIqpJMHW3pVzSecLoUAAAA1yMUO2RVaaaG4wkda+t3uhQAAADXIxQ7pKpkZAIFfcUAAADOIxQ7ZEl+mgI+jw429TldCgAAgOsRih3i83q0oihDBxpZKQYAAHAaodhBVSUZOtjUK2ut06UAAAC4GqHYQatKMtUxMKy2viGnSwEAAHA1QrGDTm22289mOwAAAEcRih1UVcoECgAAgGRAKHZQZtCvBTkhNtsBAAA4jFDssKqSTFaKAQAAHEYodtiqkkzVtA9ocDjudCkAAACuRSh2WFVJphJWOtzCIR4AAABOIRQ7bDWb7QAAABxHKHbYgpyQMlJ8bLYDAABwEKHYYcYYrS/P1uNHWhVPcLIdAACAEwjFSeC9F1aornNQjxxocboUAAAAVyIUJ4E3rS7WgpyQvvtUjdOlAAAAuBKhOAl4PUYfuHiRnj/Rqb31PU6XAwAA4DqE4iTxzgvKlZ7i031PHXe6FAAAANchFCeJjKBf79hUrt/saVJzT8TpcgAAAFyFUJxE/vySRUpYqx88e8LpUgAAAFyFUJxEynNT9aZVxfrR9lqFh2NOlwMAAOAahOIkc8dli9UzGNXPXqhzuhQAAADXIBQnmU0Lc3TRkjz92yNH1NpLbzEAAMBsIBQnGWOMvvJnazUcS+jvt+53uhwAAABXIBQnocX5afrEG5fr9/ub9bu9TU6XAwAAMO8RipPUhy5drDVlmfq7rfvVE446XQ4AAMC8RihOUj6vR1996zp1hYf15d8ecLocAACAeY1QnMRWl2bpLy5fov/eWa+tuxucLgcAAGDeIhQnuY9evUybF+Xq4z/drfufOeF0OQAAAPMSoTjJBf1e/eCOzbqmqkhffGi/vvaHw7LWOl0WAADAvEIongOCfq/+870b9e7N5fp/j1Xrsz/fq3iCYAwAADBdfE4XgHPj83r0lVvXKj89Rd/8U7VCAa++eNMqGWOcLg0AAGDOIxTPIcYYfepNKxSJxnXvkzVakBPShy5b4nRZAAAAcx6heA763HVVauge1Jd/e1AlWSHdsK7E6ZIAAADmNHqK5yCPx+j/vuM8bVqYo0/8bLdeONHpdEkAAABzGqF4jgr6vbr3/Zu0IDukD37vBf1o+0kl2HwHAAAwKYTiOSwnLaAf3LFZq0oz9YVf7tPbvv2MDjb1Ol0WAADAnEMonuMW5KTqJ3du0b+9fb1OdIR14zef0j8/fFCRaNzp0gAAAOYMQvE8YIzRW89foEc/eYXetnGB/uuJ47rpm09pb32P06UBAADMCYTieSQnLaCvvm2d7v/gZvVGorr1W0/r7m1HFI0nnC4NAAAgqZlkODJ406ZNdseOHU6XMa/0hKP64kP79KvdjcpLC+iSynxdWpmvS5flqzQ75HR5AAAAjjDG7LTWbjrzOnOK56msVL/uftcG3XxeqX79UpOeqm7XQy81SpL+bGOZvnLrWgX9XoerBAAASA6E4nnuqpVFumplkay1OtLSr1/uatC3/+eYjrb0679uO59VYwAAANFT7BrGGK0oztBnr1upe9+/STXtA7r5/z3FwR8AAAAiFLvSG1cV6Vd3XayMoF/vufc53b3tCCPcAACAqxGKXaqyMEO/uusSXbemRHdvO6o33/2EHjvc6nRZAAAAjmD6BPR0dbv+bus+HW8b0DVVhbpqZZHWlmVpeXG6UnxsxgMAAPPH2aZPEIohSRqOJXTvk8d175PH1R2OSpL8XqMN5Tn61JuW68IleQ5XCAAAMHWEYpwTa63qOge1r7FHext6tHVXgxp7IrpuTbE+d12VKvJSnS4RAABg0gjFmJTB4bi+8+RxfevxY4onrN61uVw3ry/VxooceTzG6fIAAAAmhFCMKWnuiehrfzysh3Y3ajieUFFmiq5dXay3nV+utQuynC4PAADgnBCKMS36IlH96VCrHt7bpMcPt2koltBly/L1l1cu1UVL8mQMq8cAACB5EYox7XojUf3ouVrd91SN2vuHdF55tr50y2qtW5DtdGkAAADjOlsoZk4xJi0z6NdfXrlUT33mDfrHt6xRc09Eb//2s/r1S41OlwYAADAhhGJMWdDv1W1bFuq3H71U6xZk6SMP7NLd244oGf4vBAAAwLnwOV0A5o+89BT98EMX6vO/2Ke7tx3V4eY+XbAoV/1DMfUPxWSt1RXLC7VlSa58Xv57DAAAJA96ijHtrLW654nj+pffH9KpH6+Q36u4tRqOJZSXFtC1a4r1ZxsX6PyFOc4WCwAAXIWNdph13eFhWSulB33yez2KRON6/HCrfr2nSX862KrBaFzXVBXpc9ev1NKCdKfLBQAALkAoRlIJD8d0/zMn9R+PVSsSjet9Wxbqo1cvU25awOnSAADAPEYoRlJq7x/S1x85ogeer5Xf69EN60r03gsrtLEih5nHAABg2hGKkdSqW/v0vadPaOvuRvUPxbSiKEPvuKBcN60vUWFG0OnyAADAPEEoxpwwMBTTQy816oHna7WnvkceI11Sma9bzivTjetKFPR7nS4RAADMYYRizDnVrX3aurtRv9rdoLrOQZVlh/SFG6p03ZpiWisAAMCkEIoxZ1lr9VR1u/7ptwd1qLlPFy7O1RdvWq1VpZlOlwYAAOYYQjHmvFg8oZ+8UKd/++NhdQ9GddmyAr3rgnJdU1WkgI/DQAAAwOsjFGPe6AlHdd/TNfrvHXVq6okoNy2gG9eVaE1plpYWpmlpQbqyUxntBgAAXo1QjHknnrB68mibfvpCnR491KrhWOL0c5WF6frKrWu1eXGugxUCAIBkM2Oh2BjjlbRDUoO19kZjzGJJP5GUJ2mnpNustcOv9RmEYkxVPGFV3xXWsbZ+Vbf264fP1aquK6wPXLxIn37zSoUCTK0AAABnD8XT0Yj5MUkHxzz+qqSvW2srJXVJumMavgbwmrweo4V5abpqZZHuvHypfvexy/T+LQv1vadP6NpvPKHf7GlUc0/E6TIBAECSmtJKsTFmgaT7Jf2TpE9KuklSm6Ria23MGHORpP9trX3za30OK8WYKc8e69Cnf/6S6joHJUmFGSlatyBbN60v0U3rSuXxMNoNAAA3OdtKsW+Kn3u3pE9Lyhh9nCep21obG31cL6lsil8DmLSLluZp2yev0L6GXu2p79ae+h7tONmpbT9p0XeerNHnrl+pi5fmS5K6Bob1xNE2HWzq01s3lmlZUcbrfDoAAJgvJh2KjTE3Smq11u40xlw5ifffKelOSaqoqJhsGcDrSvF5df7CHJ2/MEeSlEhY/Wp3g772h8N6z73bdWllvsLDMe2u61Zi9H+c3Pvkcd22ZaE+cc1yZaX6HaweAADMhkm3Txhj/lnSbZJikoKSMiX9UtKbRfsE5oBINK77nzmh7zxVo9KsoK5cUagrVxSoPDdVd287oh9vr1VWyK9PvmmF3n1BuXxeZiEDADDXzehIttGV4r8ZnT7x35J+bq39iTHm25L2WGu/9VrvJxQjGR1s6tU//Hq/njveqZXFGfr7m1adbrUAAABz00xOnzjTZyR90hhTrZEe4/tm4GsAM66qJFMPfHiL/vO9G9UXiek9927XX/5wp4639SsZ5nsDAIDpw+EdwDmIROO694nj+tbjxzQYjasoM0UbK3K0oSJbV60sUmVhutMlAgCAc8CJdsA0aO6J6A/7m/VibZd21XartjMsj5HeunGBPvHG5SrNDkmS+odiemh3o363r0m3X7RI16wqcrhyAAAgEYqBGdHSG9G9T2+gTucAAB9TSURBVBzXD549KRnp9osWqn8orod2N2hgOK70FJ/CwzF95da1etdmpqwAAOC0mZpTDLhaUWZQf3vjKn3gkkX6+iNH9Z2napTi8+imdaV694UVWlGUob/60Yv67C/2qr1/SHe9oVLGcGAIAADJhpViYBo190SUmuJVZvDl2cbReEKfeXCPfrGrQe+9sEK3bihTcVZQhRlBBXyMeQMAYDaxUgzMguKs4Kuu+b0efe3t65WfkaJ7njiuH22vPf3ckoI0feXWtdqyJG82ywQAAGdgpRiYRcfb+lXbGVZzT0TNvRFt3d2okx0D+us3VOqjVy/jgBAAAGYYK8VAElhSkK4lBS+Pb/vwZUv0xYf269//VK2nj3XoI1dVqrq1X7vrurW3oUchv1eXLcvXFcsLtWlRjoJ+r4PVAwAwf7FSDCSBrbsb9IVf7lP/UEySVJYd0tqyLPVGotpxokvD8YSCfo+2LMnT5csKdPnyAi0tSGPTHgAAE8RKMZDEbjmvTJsX5+pwc59Wl2apICPl9HPh4ZieO96hJ46064kjbfrS4QOSRoLzuzeX67aLFikr5D/bRwMAgHPASjEwx9R1hvXE0Tb9fl+znjzarowUn95/8UJ98JLFyktPef0PAADAxTi8A5iH9jX06FuPV+t3+5rlMUY5qX5lhfzKSQ1oUX6aPnb1MpXnpjpdJgAASYNQDMxj1a39emh3g9oHhtUdHlbXQFQv1XcrnrD6qysr9RdXLGGTHgAAIhQDrtPUM6gv//agfrunSQvzUnXXlZVaWZKhRflpygz6Za1Vc29ER1v6dbJjQBdX5mvpmMkYAADMR4RiwKWeOtquv39on463DZy+lpcW0HAsob7RaReSlOLz6DPXrtQHLl4kj4epFgCA+YlQDLhYLJ7QsbYB1bQP6ETHgGraBhTwebS8KF2VhRkqyAjonx8+pEcPteqiJXn62jvWqyw7NO5nWWtlrQjOAIA5iVAM4DVZa/WzHXX60q8PyBijSyrztLo0S2vKMlWSFdJLdd167niHttd0qmcwqpvXl+rdmyu0bkEW85IBAHMGoRjAOanrDOvrjxzRrrpu1bQPvOK5/PSALlycp6Dfq4f3NmkwGldVSaY+fNli3bqhjHAMAEh6hGIAE9YXiepgU58auwe1pizrFafo9UWi2rq7UT/aXquDTb26pqpI//LWtcpnVjIAIIkRigHMiETC6rtP1+hf/3BYmUGf/vVt63TVyiKnywIAYFxnC8UeJ4oBMH94PEYfumyJfv3Xlyo/PUUf/P4OffSBXTrc3Od0aQAAnDNWigFMm6FYXN98tFrffbpG4eG4rqkq1F9euVR5aSmq7QyrtjOspp5BBX1eZaX6lRn0qzAjRRsX5nC4CABgVtA+AWDWdA0M6/5nT+j7z5xQdzj6iuc8Rkqc8ddOWsCrK1cU6k2ri3TVykJlBP2zVywAwFUIxQBm3cBQTL/d0yRjpIrcVFXkpaooI6hoIqHewZh6I1HVdoT1yMEWPXKgRW19Q8pO9evb7ztfW5bkOV0+AGAeIhQDSGqJhNXO2i597hd7dbJjQP/6tnW6dcMCp8sCAMwzbLQDkNQ8HqMLFuXq5//rYm1amKtP/PQl3b3tiJLhP9wBAPOfz+kCAGCsrFS/7v/gZn3uF3t197aj+tOhVpXnpio/LaC89BStKsnUxZV5Sg3w1xcAYPrwbxUASSfg8+hrb1+nqpIM/XF/iw429qq9f0i9kdjp5y9akqc3rCjQW89fwMY8AMCU0VMMYM6IROPacaJLjx1u1WOHWnW8fUCVhem67/ZNWpiX5nR5AIA5gI12AOadZ6rb9Vc/flFG0n/dtkmbF+eefu5kx4COtvTr0mX5zEAGAJxGKAYwL51oH9AH739BdZ1hff76KoWH43p4b5P2N/ZKknLTAnrfhRV630ULVZgRdLhaAIDTCMUA5q2ecFR3/fhFPVXdLknaWJGt69eWaElBmn68vU6PHmqR3+PRJZV5ygj6FfR7lOLzamVJht66cQEryQDgIoRiAPNaNJ7Q09XtWl6UodLs0Cueq2kf0PefrtH2mk4NxRKKROMKD8fVMxhVfnqK7rh0sd63pYINewDgAoRiABjDWqvtNZ36j8eq9eTRdmUEfbr9okX680sWKS89xenyAAAzhFAMAGext75H33q8Wr/f36wUn0fvuqBCd16+5FUrzgCAuY9QDACvo7q1X9/+n2P61a4GWUklWUFlp/qVkxpQeopPfZGYugeH1TUQVcJanb8wRxcvzdfFS/O0MC9VxhinvwUAwOsgFAPAOWroHtQD22vV0D2o7vCwugej6o/ElB70KSc1oOxUv2Jxq+01HWrpHZIkrS/P1g/v2ExfMgAkubOFYk60A4AzlGWH9DdvXvG6r7PWqqZ9QI8dbtM/P3xQH3lgl77z/k3yeT2zUCUAYDrxNzcATJIxRksK0nXHpYv1pVvW6PHDbfqnhw86XRYAYBJYKQaAafCeCytU3dqv7z5do8rCdL33woVq6Y3ox9tr9d876pSa4tOllfm6tDJfFy7Jpc0CAJIMoRgApskXbqjS8fZ+/f3W/XrsUKseP9ymuLW6fFmBrKSfvFCr7z9zQj6P0U3rS3XXGypVWZjudNkAALHRDgCmVV8kqrd/+1k1dg/qnReU631bFmphXpokKRKN68XaLv1xf4t++kKdIrG4blhbor++qlIrizMdrhwA3IHpEwAwS4ZicUlSiu/sx0e39w/pO0/W6P979oQGhuMqzQpqw8IcbSjPVlVJpgI+jzzGyOsx8nuN0gI+paZ4lRbwnX7OY8QYOACYIEIxACShroFh/Wp3g3ae7NKu2m41dA9O6P0eI5VkhbSyOEMrSzK0sjhTV64ooGcZAM6CUAwAc0BLb0THWvsVt1bxhFXCWg3HEgoPx0d/xTQcSyhhpXjCKpZIqLZzUIeaenW8fUDxhFVqwKu3bCjT+y5cqFWltGUAwFjMKQaAOaAoM6iizOCk3jsUi2tfQ49+8nydfr6zXj/eXqtNC3P0+RuqtLEi53Xf3z8UU4rPIz9zlgG4ECvFADAPdYeH9eDOen3nyRq19EX0rgsq9JlrVyg7NfCK18UTVk8cbdPPXqjTtoMtKsoM6pvv3qAN5xCiAWAuon0CAFyofyimux85ou89c0LZIb/uuGyxEgmrjoFhdQ4M6/maTjX1RJSbFtDN60v1yIEWtfRG9OlrV+hDly6Rx8NGPgDzC6EYAFzsQGOvvvCrvdpV2y1JSk/xKTctoGWF6Xrb+Qt0dVWRAj6PesJRfebne/T7/c26ckWBtizJU03bgI6396uxO6LVpZm6ckWhrlxRoNLskMPfFQBMHKEYAFzOWqu2/iFlBv0K+s8+Ls5aqx8+d1L/+JuDGo4nlJ8e0JL8dBVlBfXiya7TEzKqSjL1qTcu1zWril7x/s6BYX1j2xE1dEd0w7pivWlVsdJS2MICIDkQigEAE9IdHpYxRlmhl8e7WWtV3dqvxw+36Scv1OpY24CuWlmoL960SgtyUvXj52v1tT8cVv9QTIUZKWrqiSjo9+iNq4p1/ZpiXbIsX5mMiwPgIEIxAGBaDccS+v4zNfrGtqOKJqwqclNV3dqvLUty9aVb1qiyIF07a7u0dXeDfrunSV3hqHweo40Lc3TligJdsbxAq0oyX3EAyf7GHt3/zAk9erBVkWhc0YRVLJ5QWU5I33z3Rp1Xnu3gdwxgPiAUAwBmRHNPRP/8u4M61NSnu66q1E3rSl510l40ntCu2m49frhVjx9u04GmXklSYUaKrlheoLULsvSbl5r0/IlOhfxeXbumWLlpAfm8Rj6P0dbdjWrtHdKXb12jd2wqd+LbBDBPEIoBAEmjtTei/znSpsePtOnJI23qjcS0ICek2y9apHdsKldW6itbLLoGhvWRB3bpqep2vf+ihfr89VVq6xtSXVdY9V2DCng9KssJqSw7pKLMoFr7Ijrc3KcjLX3q6B/WBy5ZpJIsNgYCIBQDAJJULJ7QiY6wFuenyfsaI+Bi8YT+9Q+Hdc8Tx1/z84yRxv6rzWOk8txUPfDhLa+amNE5MKzmnggn/wEuQigGAMwLjxxo0Z76bi3ICWlBTqoW5IQUjSdU3zWo+q5BNfdEVJiZohVFGVpelKGajgHdft/zyk7z64EPb9GCnFRZa/WLFxv0j789oO5wVDevL9UXbqia9GmCAOYOQjEAwLVequvWbfdtV2bIr397+3r9x+PH9MSRNm2syNbmxXn67tM18nuMPnbNMl1TVaS9DT3aU9+jvQ09Go4llJ7iU3qKTxlBny5dlq83ry5+zbF2AJIXoRgA4Gp763v0vvu2q2cwqtSAV59+8wrddtEieT1GJzsG9KVfH9Cjh1pPvz7F59Gq0kylp/g0MBRT/1BM7f0jJwFmBn26+bxSvXNThdYuyBr361lrFR6OKzXgfdXGQwDOIRQDAFzvQGOvfrajTh+6bLEW5KS+6vknj7apoWtQaxdkaXlRhvxezyueTySsnjveoZ/tqNPv9jVrKJbQLeeV6os3rVZuWuAVX+dzv9ijl+p75PcaZYUCyk3zq7IwXW9YUag3rCxUfnrKpL8Pay1BG5gkQjEAANOoZzCq7z5Vo289Xq3MoF//cMtqXVNVpLu3HdW9Tx5XTqpft21ZpKFYXF3hkRXm3XXdaukdkjHSeeXZunZ1sW4+r/ScJ2McaenTr3Y1aOvuRqX4PLrn/eersjBjhr9TYH4hFAMAMAMONffq0w/u0Z76HmUGfeqNxPTOTeX63PUrlZ0aeMVrrbXa39irPx1q1baDLdpT3yNjpC2L83Tj+hL5PR4190bU3BtRR/+QEmP+FV3fNaiDTb3yeowuW5avfQ29iiUS+t4HLtCGipxZ/q6BuYtQDADADInFE7rvqRo9erBVn3jjcl20NO+c3lfTPqCtuxv0q10NOtERPn09Ny2g/PSAvJ6X2zcygj5dv6ZYN64vVX56ik52DOi2+55XW9+QvvW+jXrDikJVt/bpod2N2nawVatKM/Xxa5aN2yYCuBmhGACAJGWt1bG2fqX4vCrMTFGK79wmW7T1DekD33teh5v7tKQgTUda+k+3Zuxv7JWsdNtFC3XXGyqVGvCqurVfR1v7VN85qPSgT7lpAeWkBlSYmaJFeWlM1IArEIoBAJiH+iJRffYXe9XaG9ENa0t0/doSFWYG1dA9qG9sO6IHd9bL5/UoFk+8oh3jTB4jLcpL07KidK0vz9YNa0u0MC/tNb92JBrXk0fbdUllnlIDvmn+zoCZQSgGAMCFjrb06Ufba5UZ8o8eaJKu8txUhYfj6hwYVnd4WE09ER1t6dORln4daenT8fYBSdLasizduK5EV1cVaWlB2umJF7F4Qg/urNc3Hj2qpp6I1pRl6jvvv0DFWa99+EkkGld7/5ACXo98Xo/8XqP0FB+TNDCrCMUAAOCc1HeF9bu9zfrNnka9VN8jScpJ9ev8hblaVZqpX7/UqJr2AZ1Xnq0b15Xo648cUXrQp/tuv0Bryl6e29w/FNMLNZ3aXtOp52s6tLehR9H4K3PHssJ0/a8rlurm80pfNQIPmAmEYgAAMGF1nWE9e6xDL5zo1I6TXappH9CKogz9zZtX6JqqQhljdKi5V3d8f4c6B4b1tzdWqWtgWE8cbdeLJ7sUS1j5vUZry7K0eXGeFuenKpawisYSGowmtHV3gw4196ksO6Q7L1+iSyrzVJgZVMboCvJQLK4T7WEdbe1TdziqN68uVkHG5Gc8A4RiAAAwZb2RqNIDPnk8r2x5aO2L6M4f7NTuum5J0urSTF2+vECXVuZrY0WOQoHxN/FZa/XY4VZ967Fj2nGy6/T1kN+r7FS/WvuGFB/TDB3wenTDuhLdfvEirSvL0rG2fj1/olM7TnSpOzysnLSA8tICyk1L0eL8VK0py1JZduisLRp1nWH95/8c0976Hn38mmW6uqpoqrcISY5QDAAAZlQkGtfOk11aUZwxqRP79jX06Hj7gFp7I2rpjahjYFhl2SFVFqarsjBdXo/RA9tr9eDOeg0Mx5UW8GpgOC5Jyk9PUXFWiroGouoYGFIkmjj9uTmpfq0py9KKogxVFqZraWG6UgNefe/pE/rlrgZ5jVFxVlC1nWG9c1O5/vbGKmUE/ae/pxdOdCrF59UFi3Im3P88FIurqTuihXmp9E4nCUIxAACYF/oiUf18Z72OtvbrvPJsXbAo91Whc2AopqOt/dpb3629DT3a19CrY239Goq9HJZTfB6958IK/cXlS5WT5tc3th3Vt//nmEqzQ3rvhQv1wolOPXusQ4PRkeC9JD9N79pcrrduXKCskF8nOsI60tKnmvYBFWakqKokU5WF6Qp4Pdpxsku/3NWg3+5pVG8kpoKMFF2xvEBXrijQZcsKlBXyz/p9wwhCMQAAcLV4wqqxe1DVrf1q6Y3o6qqiV/Un7zzZqU/+7CWd7AirIjdVV64YCbLd4ageeL5WL5zokt9rZGQ0HE+86mt4jJQZ8qs7HFXI79W1a4q1sSJb22s69eTRdvUMRuX3Gl2xvEA3rS/VNVVFSg14daIjrN11XdpT36OA16MFOSEtyElVYWaK6jrD2t/Yq/2NvWroGtTmxbm6bk2xNi/OlY/NiRNGKAYAADgHQ7G4OvqHVZIVfFXLw9GWPj34Yr0kaXlhhlYUZ2hxfppaeiM63Nyng819auga1GXL8vXGVUVKS3l5fnMsntBL9d36w/4W/fqlRjX1RBT0exT0e9Udjkoa6aWOW6vh2CsDt9djVFmQrsLMFL1wolORaEK5aQG9aVWRbt1QpgsW5b6iz7tzYFiPHGhWVziqvLSA8jNSlJcWUH8kpsaeiBq7B9XRP6TzF+XqmqpCV82ZJhQDAAAkiUTC6sXaLv1mT5Mi0bjOK8/W+vJsLS/KkJHUPjCk+q5BNfdEVJod0srijNMnDoaHY3riSJt+t69Z2w60aGA4rgU5If3ZhjIVZQX1u73NevZ4xys2KI4n5PdqMBpXasCrN60q0g3rSrWqNFMlmcFXbaScTwjFAAAA80x4OKY/7m/Rz1+s19PV7UpYaVFeqq5fW6Ib1o2cStjRP6T2/mF19A8pPehTaVZIxVlBBbwePX+iU1t3N+rhvU3qGRxZrQ76PVqUl6bS7JAkKWGtElbKTw/oqpWFunx5gTKDL/dEDwzFdKytX0G/VyVZwdObFF9LXyR6Tq+bCYRiAACAeaylN6KewaiWFaZPeNLFcCyhF2u7dKytXzVtAzrePqCW3og8xshjJGOMTnYMqCs80hN94eI8pQa8OtzSp5Md4Vd8VkaKT2U5IV2wKFdXrijQRUtHjgE/dSjMw/uadLIjrO2fv9qRA1sIxQAAAJi0+GjLx7aDLfrTwVbFrVVVcaZWFI8cHz4ct2rqHlRj96BOdIT1fE2nBqNxBXweVeSmqrq1X5K0pixT160p0Z9fssiRXmZCMQAAAGbNqRnPjx9u05GWPl1Sma/r1hRrYV6ao3WdLRS7Z6shAAAAZk3Q79Vly0bmMs8FDLcDAACA6xGKAQAA4HqEYgAAALgeoRgAAACuRygGAACA6xGKAQAA4HqEYgAAALjepEOxMabcGPOYMeaAMWa/MeZjo9dzjTGPGGOOjv6eM33lAgAAANNvKivFMUmfstaukrRF0l3GmFWSPivpUWvtMkmPjj4GAAAAktakQ7G1tsla++Lon/skHZRUJukWSfePvux+SW+ZapEAAADATJqWnmJjzCJJGyRtl1RkrW0afapZUtF0fA0AAABgpkw5FBtj0iX9XNLHrbW9Y5+z1lpJ9izvu9MYs8MYs6OtrW2qZQAAAACTNqVQbIzxayQQ/8ha+4vRyy3GmJLR50sktY73XmvtPdbaTdbaTQUFBVMpAwAAAJiSqUyfMJLuk3TQWvt/xzz1kKTbR/98u6Stky8PAAAAmHm+Kbz3Ekm3SdprjNk9eu3zkv5F0s+MMXdIOinpHVMrEQAAAJhZkw7F1tqnJJmzPH31ZD8XAAAAmG2caAcAAADXIxQDAADA9czI1DSHizCmTSP9x07Il9Tu0Neei7hfE8c9mxju18RxzyaG+zVx3LOJ4X5N3Gzes4XW2leNPkuKUOwkY8wOa+0mp+uYK7hfE8c9mxju18RxzyaG+zVx3LOJ4X5NXDLcM9onAAAA4HqEYgAAALgeoVi6x+kC5hju18RxzyaG+zVx3LOJ4X5NHPdsYrhfE+f4PXN9TzEAAADASjEAAABcz7Wh2BhzrTHmsDGm2hjzWafrSUbGmHJjzGPGmAPGmP3GmI+NXs81xjxijDk6+nuO07UmE2OM1xizyxjzm9HHi40x20d/1n5qjAk4XWMyMcZkG2MeNMYcMsYcNMZcxM/Y2RljPjH6z+M+Y8wDxpggP2OvZIz5rjGm1Rizb8y1cX+mzIh/H713e4wxG52r3DlnuWf/Z/Sfyz3GmF8aY7LHPPe50Xt22BjzZmeqds5492vMc58yxlhjTP7oY37GdPZ7Zoz5yOjP2X5jzL+OuT7rP2OuDMXGGK+k/5B0naRVkt5tjFnlbFVJKSbpU9baVZK2SLpr9D59VtKj1tplkh4dfYyXfUzSwTGPvyrp69baSkldku5wpKrk9Q1Jv7fWrpS0Xv9/e/caYsdZx3H8+yObhKSF3oKxZitbNfVFai+hSvGGjSJtLV3BQiMBqxaEvPDypmoNCIIvRERLvVS0pYkaLFpjDYLSmpYqaBttyKX1mrYh3bAxCZJ4JY3154vniZ2e3bObgMnMZn4fGHbmmTmH5/z3f878z8wzc0rskmPTkLQM+Ahwle1LgXnAapJjg9YD1w60Dcup64DldfoQcNdp6mPXrGdqzB4CLrV9GfBH4HaAuh9YDayoj/la3a/2yXqmxgtJFwHvBPY2mpNjxXoGYibpGmAcuNz2CuALtb2VHOtlUQy8Adht+xnbzwP3Uf4p0WB70va2Ov83SrGyjBKrDXWzDcC72+lh90gaBd4F3F2XBawC7q+bJF4Nks4B3grcA2D7eduHSY7NZARYJGkEWAxMkhx7Cds/B/4y0Dwsp8aBb7l4DDhX0oWnp6fdMV3MbD9o+9918TFgtM6PA/fZPmr7WWA3Zb/aG0NyDOBLwMeB5gVbyTGGxmwt8DnbR+s2B2p7KznW16J4GfBcY3mitsUQksaAK4HHgaW2J+uq/cDSlrrVRXdQPhD/U5cvAA43dizJtZe6GDgI3FuHnNwt6SySY9OyvY9yJGUvpRg+AjxBcuxEDMup7A9OzAeBn9T5xGwaksaBfbZ3DKxKvIa7BHhLHf71qKTX1/ZWYtbXojhOgqSzgR8AH7P91+Y6l9uX5BYmgKQbgAO2n2i7L3PICLASuMv2lcA/GBgqkRx7UR0HO075MvEK4CymOYUbM0tOnRxJ6yjD6Ta23ZeukrQY+BTw6bb7MseMAOdThmjeBnyvnmFtRV+L4n3ARY3l0doWAyTNpxTEG21vqs1/Pn7qp/49MOzxPfMm4EZJeyhDclZRxsueW091Q3Jt0AQwYfvxunw/pUhOjk3vHcCztg/aPgZsouRdcmx2w3Iq+4MZSHo/cAOwxi/ewzUxm+rVlC+rO+o+YBTYJunlJF4zmQA21aElWylnWZfQUsz6WhT/Glher9heQBnMvbnlPnVO/bZ2D/A7219srNoM3FLnbwF+dLr71kW2b7c9anuMklMP214DPALcVDdLvBps7week/Ta2vR24Lckx4bZC1wtaXF9fx6PV3JsdsNyajPwvnqHgKuBI41hFr0m6VrKcLAbbf+zsWozsFrSQkkXUy4g29pGH7vC9i7bL7M9VvcBE8DK+hmXHBvuAeAaAEmXAAuAQ7SVY7Z7OQHXU66mfRpY13Z/ujgBb6acYtwJbK/T9ZRxsluAPwE/A85vu69dm4C3AT+u86+qb+bdwPeBhW33r0sTcAXwm5pnDwDnJcdmjNdngN8DTwLfBhYmx6bE6LuUMdfHKMXJrcNyChDlbkRPA7sod/Zo/TV0JGa7KeM6j3/+f72x/boasz8A17Xd/y7Ea2D9HmBJcmzWHFsAfKd+nm0DVrWZY/lFu4iIiIjovb4On4iIiIiI+J8UxRERERHReymKIyIiIqL3UhRHRERERO+lKI6IiIiI3ktRHBHRMkkvSNremD45+6NO+LnHJD35/3q+iIgz1cjsm0RExCn2L9tXtN2JiIg+y5HiiIiOkrRH0ucl7ZK0VdJravuYpIcl7ZS0RdIra/tSST+UtKNOb6xPNU/SNyU9JelBSYtae1ERER2Vojgion2LBoZP3NxYd8T264CvAHfUti8DG2xfBmwE7qztdwKP2r4cWAk8VduXA1+1vQI4DLznFL+eiIg5J79oFxHRMkl/t332NO17KD97+oyk+cB+2xdIOgRcaPtYbZ+0vUTSQWDU9tHGc4wBD9leXpc/Acy3/dlT/8oiIuaOHCmOiOg2D5k/GUcb8y+Q60kiIqZIURwR0W03N/7+qs7/Elhd59cAv6jzW4C1AJLmSTrndHUyImKuy9GCiIj2LZK0vbH8U9vHb8t2nqSdlKO9761tHwbulXQbcBD4QG3/KPANSbdSjgivBSZPee8jIs4AGVMcEdFRdUzxVbYPtd2XiIgzXYZPRERERETv5UhxRERERPRejhRHRERERO+lKI6IiIiI3ktRHBERERG9l6I4IiIiInovRXFERERE9F6K4oiIiIjovf8CyBMWCmhaCugAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wBD0wM3JVXol" + }, + "source": [ + "### Appendix" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Iz0QI1P7VaDP" + }, + "source": [ + "#### References\n", + "1. [https://medium.com/@yusufnoor_88274/implementing-neural-graph-collaborative-filtering-in-pytorch-4d021dff25f3](https://medium.com/@yusufnoor_88274/implementing-neural-graph-collaborative-filtering-in-pytorch-4d021dff25f3)\n", + "2. [https://github.com/xiangwang1223/neural_graph_collaborative_filtering](https://github.com/xiangwang1223/neural_graph_collaborative_filtering)\n", + "3. [https://arxiv.org/pdf/1905.08108.pdf](https://arxiv.org/pdf/1905.08108.pdf)\n", + "4. [https://github.com/metahexane/ngcf_pytorch_g61](https://github.com/metahexane/ngcf_pytorch_g61)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_JywmGguVbNU" + }, + "source": [ + "#### Next\n", + "\n", + "Try out this notebook on the following datasets:\n", + "\n", + "![](https://github.com/recohut/reco-static/raw/master/media/images/120222_data.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MppKYNXJVswT" + }, + "source": [ + "Compare out the performance with these baselines:\n", + "\n", + "1. MF: This is matrix factorization optimized by the Bayesian\n", + "personalized ranking (BPR) loss, which exploits the user-item\n", + "direct interactions only as the target value of interaction function.\n", + "2. NeuMF: The method is a state-of-the-art neural CF model\n", + "which uses multiple hidden layers above the element-wise and\n", + "concatenation of user and item embeddings to capture their nonlinear feature interactions. Especially, we employ two-layered\n", + "plain architecture, where the dimension of each hidden layer\n", + "keeps the same.\n", + "3. CMN: It is a state-of-the-art memory-based model, where\n", + "the user representation attentively combines the memory slots\n", + "of neighboring users via the memory layers. Note that the firstorder connections are used to find similar users who interacted\n", + "with the same items.\n", + "4. HOP-Rec: This is a state-of-the-art graph-based model,\n", + "where the high-order neighbors derived from random walks\n", + "are exploited to enrich the user-item interaction data.\n", + "5. PinSage: PinSage is designed to employ GraphSAGE\n", + "on item-item graph. In this work, we apply it on user-item interaction graph. Especially, we employ two graph convolution\n", + "layers, and the hidden dimension is set equal\n", + "to the embedding size.\n", + "6. GC-MC: This model adopts GCN encoder to generate\n", + "the representations for users and items, where only the first-order\n", + "neighbors are considered. Hence one graph convolution layer,\n", + "where the hidden dimension is set as the embedding size, is used." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e7RRc2UQBuc9" + }, + "source": [ + "## A simple recommender with tensorflow\n", + "> A tutorial on how to build a simple deep learning based movie recommender using tensorflow library." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hLtJPt_5idKN" + }, + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "from tensorflow import keras\n", + "from tensorflow.keras import layers\n", + "from tensorflow.keras import models\n", + "\n", + "tf.random.set_seed(343)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "DNLlAwKUihC1" + }, + "source": [ + "# Clean up the logdir if it exists\n", + "import shutil\n", + "shutil.rmtree('logs', ignore_errors=True)\n", + "\n", + "# Load TensorBoard extension for notebooks\n", + "%load_ext tensorboard" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "8IRTF0EVjQuX", + "outputId": "932eaa43-725c-4fb8-e9d4-dca92ced4cf0" + }, + "source": [ + "movielens_ratings_file = 'https://github.com/sparsh-ai/reco-data/blob/master/MovieLens_100K_ratings.csv?raw=true'\n", + "df_raw = pd.read_csv(movielens_ratings_file)\n", + "df_raw.head()" + ], + "execution_count": null, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
UserIdMovieIdRatingTimestamp
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n", + "
" + ], + "text/plain": [ + " UserId MovieId Rating Timestamp\n", + "0 196 242 3.0 881250949\n", + "1 186 302 3.0 891717742\n", + "2 22 377 1.0 878887116\n", + "3 244 51 2.0 880606923\n", + "4 166 346 1.0 886397596" + ] + }, + "execution_count": 22, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "El1C8OwWjhxk", + "outputId": "f8ed06e7-8554-45f2-982d-987f153a5cc7" + }, + "source": [ + "df = df_raw.copy()\n", + "df.columns = ['userId', 'movieId', 'rating', 'timestamp']\n", + "user_ids = df['userId'].unique()\n", + "user_encoding = {x: i for i, x in enumerate(user_ids)} # {user_id: index}\n", + "movie_ids = df['movieId'].unique()\n", + "movie_encoding = {x: i for i, x in enumerate(movie_ids)} # {movie_id: index}\n", + "\n", + "df['user'] = df['userId'].map(user_encoding) # Map from IDs to indices\n", + "df['movie'] = df['movieId'].map(movie_encoding)\n", + "\n", + "n_users = len(user_ids)\n", + "n_movies = len(movie_ids)\n", + "\n", + "min_rating = min(df['rating'])\n", + "max_rating = max(df['rating'])\n", + "\n", + "print(f'Number of users: {n_users}\\nNumber of movies: {n_movies}\\nMin rating: {min_rating}\\nMax rating: {max_rating}')\n", + "\n", + "# Shuffle the data\n", + "df = df.sample(frac=1, random_state=42)" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of users: 943\n", + "Number of movies: 1682\n", + "Min rating: 1.0\n", + "Max rating: 5.0\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1W5V8T-C8Gpv" + }, + "source": [ + "### Scheme of the model\n", + "\n", + "![](https://github.com/recohut/reco-static/raw/master/media/images/120222_scheme.png)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "G-iv9rijkaBf" + }, + "source": [ + "class MatrixFactorization(models.Model):\n", + " def __init__(self, n_users, n_movies, n_factors, **kwargs):\n", + " super(MatrixFactorization, self).__init__(**kwargs)\n", + " self.n_users = n_users\n", + " self.n_movies = n_movies\n", + " self.n_factors = n_factors\n", + " \n", + " # We specify the size of the matrix,\n", + " # the initializer (truncated normal distribution)\n", + " # and the regularization type and strength (L2 with lambda = 1e-6)\n", + " self.user_emb = layers.Embedding(n_users, \n", + " n_factors, \n", + " embeddings_initializer='he_normal',\n", + " embeddings_regularizer=keras.regularizers.l2(1e-6),\n", + " name='user_embedding')\n", + " self.movie_emb = layers.Embedding(n_movies, \n", + " n_factors, \n", + " embeddings_initializer='he_normal',\n", + " embeddings_regularizer=keras.regularizers.l2(1e-6),\n", + " name='movie_embedding')\n", + " \n", + " # Embedding returns a 3D tensor with one dimension = 1, so we reshape it to a 2D tensor\n", + " self.reshape = layers.Reshape((self.n_factors,))\n", + " \n", + " # Dot product of the latent vectors\n", + " self.dot = layers.Dot(axes=1)\n", + "\n", + " def call(self, inputs):\n", + " # Two inputs\n", + " user, movie = inputs\n", + " u = self.user_emb(user)\n", + " u = self.reshape(u)\n", + " \n", + " m = self.movie_emb(movie)\n", + " m = self.reshape(m)\n", + " \n", + " return self.dot([u, m])\n", + "\n", + "n_factors = 50\n", + "model = MatrixFactorization(n_users, n_movies, n_factors)\n", + "model.compile(\n", + " optimizer=keras.optimizers.Adam(learning_rate=0.001),\n", + " loss=keras.losses.MeanSquaredError()\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Bac1w7u49Ddx", + "outputId": "bb733033-9aba-446b-a56d-f971897221d0" + }, + "source": [ + "try:\n", + " model.summary()\n", + "except ValueError as e:\n", + " print(e, type(e))" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This model has not yet been built. Build the model first by calling `build()` or calling `fit()` with some data, or specify an `input_shape` argument in the first layer(s) for automatic build. \n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o-JSFnJA-1dz" + }, + "source": [ + "This is why building models via subclassing is a bit annoying - you can run into errors such as this. We'll fix it by calling the model with some fake data so it knows the shapes of the inputs." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7wkIhqmO92Ca", + "outputId": "6825be87-3d5e-4d25-e276-5043ff3a3bb9" + }, + "source": [ + "_ = model([np.array([1, 2, 3]), np.array([2, 88, 5])])\n", + "model.summary()" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"matrix_factorization_1\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "user_embedding (Embedding) multiple 47150 \n", + "_________________________________________________________________\n", + "movie_embedding (Embedding) multiple 84100 \n", + "_________________________________________________________________\n", + "reshape_1 (Reshape) multiple 0 \n", + "_________________________________________________________________\n", + "dot_1 (Dot) multiple 0 \n", + "=================================================================\n", + "Total params: 131,250\n", + "Trainable params: 131,250\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9Nxdrz7b_HOq" + }, + "source": [ + "We're going to expand our toolbox by introducing callbacks. Callbacks can be used to monitor our training progress, decay the learning rate, periodically save the weights or even stop early in case of detected overfitting. In Keras, they are really easy to use: you just create a list of desired callbacks and pass it to the model.fit method. It's also really easy to define your own by subclassing the Callback class. You can also specify when they will be triggered - the default is at the end of every epoch.\n", + "\n", + "We'll use two: an early stopping callback which will monitor our loss and stop the training early if needed and TensorBoard, a utility for visualizing models, monitoring the training progress and much more." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6N_Y7u5o-QpY", + "outputId": "b4f299bc-25e2-4e38-b349-dc07184fb488" + }, + "source": [ + "callbacks = [\n", + " keras.callbacks.EarlyStopping(\n", + " # Stop training when `val_loss` is no longer improving\n", + " monitor='val_loss',\n", + " # \"no longer improving\" being defined as \"no better than 1e-2 less\"\n", + " min_delta=1e-2,\n", + " # \"no longer improving\" being further defined as \"for at least 2 epochs\"\n", + " patience=2,\n", + " verbose=1,\n", + " ),\n", + " keras.callbacks.TensorBoard(log_dir='logs')\n", + "]\n", + "\n", + "history = model.fit(\n", + " x=(df['user'].values, df['movie'].values), # The model has two inputs!\n", + " y=df['rating'],\n", + " batch_size=128,\n", + " epochs=20,\n", + " verbose=1,\n", + " validation_split=0.1,\n", + " callbacks=callbacks\n", + ")" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/20\n", + "704/704 [==============================] - 3s 3ms/step - loss: 12.0905 - val_loss: 5.5121\n", + "Epoch 2/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 2.1751 - val_loss: 1.2149\n", + "Epoch 3/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 1.0271 - val_loss: 0.9839\n", + "Epoch 4/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.9003 - val_loss: 0.9266\n", + "Epoch 5/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.8470 - val_loss: 0.8996\n", + "Epoch 6/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.8046 - val_loss: 0.8786\n", + "Epoch 7/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.7667 - val_loss: 0.8680\n", + "Epoch 8/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.7329 - val_loss: 0.8618\n", + "Epoch 9/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.6999 - val_loss: 0.8558\n", + "Epoch 10/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.6688 - val_loss: 0.8558\n", + "Epoch 11/20\n", + "704/704 [==============================] - 2s 3ms/step - loss: 0.6381 - val_loss: 0.8560\n", + "Epoch 00011: early stopping\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5QUGLmtw_eWA" + }, + "source": [ + "We see that we stopped early because the validation loss was not improving. Now, we'll open TensorBoard (it's a separate program called via command-line) to read the written logs and visualize the loss over all epochs. We will also look at how to visualize the model as a computational graph." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "_J-v9Hua_SV8" + }, + "source": [ + "# Run TensorBoard and specify the log dir\n", + "%tensorboard --logdir logs" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ldq0DwgI_lWC" + }, + "source": [ + "We've seen how easy it is to implement a recommender system with Keras and use a few utilities to make it easier to experiment. Note that this model is still quite basic and we could easily improve it: we could try adding a bias for each user and movie or adding non-linearity by using a sigmoid function and then rescaling the output. It could also be extended to use other features of a user or movie." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-dpCn5hm_nUM" + }, + "source": [ + "Next, we'll try a bigger, more state-of-the-art model: a deep autoencoder." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zTGdZ0b4_4rl" + }, + "source": [ + "We'll apply a more advanced algorithm to the same dataset as before, taking a different approach. We'll use a deep autoencoder network, which attempts to reconstruct its input and with that gives us ratings for unseen user / movie pairs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yIf926SkCOEp" + }, + "source": [ + "![](https://github.com/recohut/reco-static/raw/master/media/images/120222_algo.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rSdY5NKQAYfI" + }, + "source": [ + "Preprocessing will be a bit different due to the difference in our model. Our autoencoder will take a vector of all ratings for a movie and attempt to reconstruct it. However, our input vector will have a lot of zeroes due to the sparsity of our data. We'll modify our loss so our model won't predict zeroes for those combinations - it will actually predict unseen ratings.\n", + "\n", + "To facilitate this, we'll use the sparse tensor that TF supports. Note: to make training easier, we'll transform it to dense form, which would not work in larger datasets - we would have to preprocess the data in a different way or stream it into the model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HBcKm55rCVkk" + }, + "source": [ + "### Sparse representation and autoencoder reconstruction\n", + "\n", + "![](https://github.com/recohut/reco-static/raw/master/media/images/120222_ae.png)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + }, + "id": "OWybs9LyE8bB", + "outputId": "1e108c11-a007-4942-bf0d-50409e4bbb1d" + }, + "source": [ + "df_raw.head()" + ], + "execution_count": null, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdmovieIdratingtimestamp
01962423.0881250949
11863023.0891717742
2223771.0878887116
3244512.0880606923
41663461.0886397596
\n", + "
" + ], + "text/plain": [ + " userId movieId rating timestamp\n", + "0 196 242 3.0 881250949\n", + "1 186 302 3.0 891717742\n", + "2 22 377 1.0 878887116\n", + "3 244 51 2.0 880606923\n", + "4 166 346 1.0 886397596" + ] + }, + "execution_count": 21, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "M9jASOsh_gvU" + }, + "source": [ + "# Create a sparse tensor: at each user, movie location, we have a value, the rest is 0\n", + "sparse_x = tf.sparse.SparseTensor(indices=df[['movie', 'user']].values, values=df['rating'], dense_shape=(n_movies, n_users))\n", + "\n", + "# Transform it to dense form and to float32 (good enough precision)\n", + "dense_x = tf.cast(tf.sparse.to_dense(tf.sparse.reorder(sparse_x)), tf.float32)\n", + "\n", + "# Shuffle the data\n", + "x = tf.random.shuffle(dense_x, seed=42)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1j2-lFANEp8t" + }, + "source": [ + "Now, let's create the model. We'll have to specify the input shape. Because we have 9724 movies and only 610 users, we'll prefer to predict ratings for movies instead of users - this way, our dataset is larger." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9s4qXdbuEpuX", + "outputId": "45ac7925-b8f3-44dd-8e35-53fa7192b538" + }, + "source": [ + "class Encoder(layers.Layer):\n", + " def __init__(self, **kwargs):\n", + " super(Encoder, self).__init__(**kwargs)\n", + " self.dense1 = layers.Dense(28, activation='selu', kernel_initializer='glorot_uniform')\n", + " self.dense2 = layers.Dense(56, activation='selu', kernel_initializer='glorot_uniform')\n", + " self.dense3 = layers.Dense(56, activation='selu', kernel_initializer='glorot_uniform')\n", + " self.dropout = layers.Dropout(0.3)\n", + " \n", + " def call(self, x):\n", + " d1 = self.dense1(x)\n", + " d2 = self.dense2(d1)\n", + " d3 = self.dense3(d2)\n", + " return self.dropout(d3)\n", + " \n", + " \n", + "class Decoder(layers.Layer):\n", + " def __init__(self, n, **kwargs):\n", + " super(Decoder, self).__init__(**kwargs)\n", + " self.dense1 = layers.Dense(56, activation='selu', kernel_initializer='glorot_uniform')\n", + " self.dense2 = layers.Dense(28, activation='selu', kernel_initializer='glorot_uniform')\n", + " self.dense3 = layers.Dense(n, activation='selu', kernel_initializer='glorot_uniform')\n", + "\n", + " def call(self, x):\n", + " d1 = self.dense1(x)\n", + " d2 = self.dense2(d1)\n", + " return self.dense3(d2)\n", + "\n", + "n = n_users\n", + "inputs = layers.Input(shape=(n,))\n", + "\n", + "encoder = Encoder()\n", + "decoder = Decoder(n)\n", + "\n", + "enc1 = encoder(inputs)\n", + "dec1 = decoder(enc1)\n", + "enc2 = encoder(dec1)\n", + "dec2 = decoder(enc2)\n", + "\n", + "model = models.Model(inputs=inputs, outputs=dec2, name='DeepAutoencoder')\n", + "model.summary()" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"DeepAutoencoder\"\n", + "__________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + "input_1 (InputLayer) [(None, 943)] 0 \n", + "__________________________________________________________________________________________________\n", + "encoder (Encoder) (None, 56) 31248 input_1[0][0] \n", + " decoder[0][0] \n", + "__________________________________________________________________________________________________\n", + "decoder (Decoder) (None, 943) 32135 encoder[0][0] \n", + " encoder[1][0] \n", + "==================================================================================================\n", + "Total params: 63,383\n", + "Trainable params: 63,383\n", + "Non-trainable params: 0\n", + "__________________________________________________________________________________________________\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aqXWA_TQGMDa" + }, + "source": [ + "Because our inputs are sparse, we'll need to create a modified mean squared error function. We have to look at which ratings are zero in the ground truth and remove them from our loss calculation (if we didn't, our model would quickly learn to predict zeros almost everywhere). We'll use masking - first get a boolean mask of non-zero values and then extract them from the result." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "G7AyGH8IFXAj" + }, + "source": [ + "def masked_mse(y_true, y_pred):\n", + " mask = tf.not_equal(y_true, 0)\n", + " se = tf.boolean_mask(tf.square(y_true - y_pred), mask)\n", + " return tf.reduce_mean(se)\n", + "\n", + "model.compile(\n", + " loss=masked_mse,\n", + " optimizer=keras.optimizers.Adam()\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6O-Hqm_FGTmz" + }, + "source": [ + "The model training will be similar as before - we'll use early stopping and TensorBoard. Our batch size will be smaller due to the lower number of examples. Note that we are passing the same array for both x and y, because the autoencoder reconstructs its input." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "OHoZ3IuJGSrL", + "outputId": "7923e0b0-7bc6-42ba-b3e6-c2bfe025291d" + }, + "source": [ + "callbacks = [\n", + " keras.callbacks.EarlyStopping(\n", + " monitor='val_loss',\n", + " min_delta=1e-2,\n", + " patience=5,\n", + " verbose=1,\n", + " ),\n", + " keras.callbacks.TensorBoard(log_dir='logs')\n", + "]\n", + "\n", + "model.fit(\n", + " x, \n", + " x, \n", + " batch_size=16, \n", + " epochs=100, \n", + " validation_split=0.1,\n", + " callbacks=callbacks\n", + ")" + ], + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:Model failed to serialize as JSON. Ignoring... Layer Decoder has arguments in `__init__` and therefore must override `get_config`.\n", + "Epoch 1/100\n", + "95/95 [==============================] - 2s 7ms/step - loss: 4.6136 - val_loss: 1.1074\n", + "Epoch 2/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 1.1491 - val_loss: 1.0088\n", + "Epoch 3/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 1.0577 - val_loss: 0.9768\n", + "Epoch 4/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 1.0257 - val_loss: 0.9758\n", + "Epoch 5/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 0.9971 - val_loss: 0.9774\n", + "Epoch 6/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 0.9812 - val_loss: 0.9604\n", + "Epoch 7/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 0.9598 - val_loss: 0.9275\n", + "Epoch 8/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 0.9501 - val_loss: 0.9253\n", + "Epoch 9/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 0.9177 - val_loss: 0.9159\n", + "Epoch 10/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 0.9193 - val_loss: 0.9189\n", + "Epoch 11/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 0.9016 - val_loss: 0.9040\n", + "Epoch 12/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 0.9119 - val_loss: 0.9108\n", + "Epoch 13/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 0.8917 - val_loss: 0.9192\n", + "Epoch 14/100\n", + "95/95 [==============================] - 0s 5ms/step - loss: 0.8855 - val_loss: 0.9166\n", + "Epoch 15/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 0.8843 - val_loss: 0.9067\n", + "Epoch 16/100\n", + "95/95 [==============================] - 0s 4ms/step - loss: 0.8851 - val_loss: 0.9034\n", + "Epoch 00016: early stopping\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kkkhIjHhGhP5" + }, + "source": [ + "Let's visualize our loss and the model itself with TensorBoard." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "MMVp_HbwGdGQ" + }, + "source": [ + "%tensorboard --logdir logs" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eSBFppW8Gkih" + }, + "source": [ + "That's it! We've seen how to use TensorFlow to implement recommender systems in a few different ways. I hope this short introduction has been informative and has prepared you to use TF on new problems. Thank you for your attention!" + ] + } + ] +} \ No newline at end of file From 1178d67c478088b3839330c73076a9e6bf8d6a5a Mon Sep 17 00:00:00 2001 From: sparsh Date: Fri, 11 Feb 2022 19:34:42 +0000 Subject: [PATCH 2/2] commit --- _notebooks/2022-01-09-lrgccf-gowalla.ipynb | 2 +- _notebooks/2022-01-10-fm-ml.ipynb | 1 + _notebooks/2022-01-11-deepfm-criteo.ipynb | 1 + _notebooks/2022-01-11-gmf-yelp.ipynb | 1 + _notebooks/2022-01-11-mbgmn-beibei.ipynb | 1 + _notebooks/2022-01-11-mf-ml.ipynb | 1 + _notebooks/2022-01-11-personalize-datalayer.ipynb | 1 + _notebooks/2022-01-12-olx-baselines.ipynb | 1 + _notebooks/2022-01-12-seq-mab-mushroom.ipynb | 1 + _notebooks/2022-01-12-sess-word2vec.ipynb | 1 + _notebooks/2022-01-12-slist-yoochoose.ipynb | 1 + 11 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 _notebooks/2022-01-10-fm-ml.ipynb create mode 100644 _notebooks/2022-01-11-deepfm-criteo.ipynb create mode 100644 _notebooks/2022-01-11-gmf-yelp.ipynb create mode 100644 _notebooks/2022-01-11-mbgmn-beibei.ipynb create mode 100644 _notebooks/2022-01-11-mf-ml.ipynb create mode 100644 _notebooks/2022-01-11-personalize-datalayer.ipynb create mode 100644 _notebooks/2022-01-12-olx-baselines.ipynb create mode 100644 _notebooks/2022-01-12-seq-mab-mushroom.ipynb create mode 100644 _notebooks/2022-01-12-sess-word2vec.ipynb create mode 100644 _notebooks/2022-01-12-slist-yoochoose.ipynb diff --git a/_notebooks/2022-01-09-lrgccf-gowalla.ipynb b/_notebooks/2022-01-09-lrgccf-gowalla.ipynb index 9a56b58..1d60e57 100644 --- a/_notebooks/2022-01-09-lrgccf-gowalla.ipynb +++ b/_notebooks/2022-01-09-lrgccf-gowalla.ipynb @@ -1 +1 @@ -{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-09-lrgccf-gowalla.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P174968%20%7C%20LR-GCCF%20on%20Gowalla.ipynb","timestamp":1644598450989}],"collapsed_sections":[],"authorship_tag":"ABX9TyO16nepUq2U+FqXoB29n/o1"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","source":["# LR-GCCF on Gowalla"],"metadata":{"id":"mVyRGyhdtRFw"}},{"cell_type":"markdown","source":["## Executive summary"],"metadata":{"id":"Y3oNohENVHAH"}},{"cell_type":"markdown","source":["| | |\n","| --- | --- |\n","| Problem | GCNs suffer from training difficulty due to non-linear activations, and over-smoothing problem. |\n","| Hypothesis | removing non-linearities would enhance recommendation performance. |\n","| Solution | Linear model with residual network structure |\n","| Dataset | Gowalla |\n","| Preprocessing | we remove users (items) that have less than 10 interaction records. After that, we randomly select 80% of the records for training, 10% for validation and the remaining 10% for test. |\n","| Metrics | HR, NDCG |\n","| Hyperparams | There are two important parameters: the dimension D of the user and item embedding matrix E, and the regularization parameter λ in the objective function. The embedding size is fixed to 64. We try the regularization parameter λ in the range [0.0001, 0.001, 0.01, 0.1], and find λ = 0.01 reaches the best performance. |\n","| Models | LR-GCCF |\n","| Cluster | PyTorch with GPU |"],"metadata":{"id":"DK7RWly8VKvU"}},{"cell_type":"markdown","source":["## Process flow\n","\n",""],"metadata":{"id":"s4weKl5kcU3v"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"paRynetXLa6r"}},{"cell_type":"code","source":["import random\n","import torch \n","import time\n","import pdb\n","import math\n","import os\n","import sys\n","from shutil import copyfile\n","from collections import defaultdict\n","import numpy as np\n","import pandas as pd \n","import scipy.sparse as sp \n","\n","import torch\n","import torch.nn as nn \n","from torch.utils.data import DataLoader\n","import torch.nn.functional as F\n","import torch.autograd as autograd\n","from torch.autograd import Variable\n","import torch.utils.data as data"],"metadata":{"id":"zfsN6pxILcQG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'"],"metadata":{"id":"CkWg0kI9LjgC"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Data"],"metadata":{"id":"VhlaApRdJJDh"}},{"cell_type":"code","source":["# download\n","dataset = 'gowalla'\n","!git clone --branch v1 https://github.com/RecoHut-Datasets/gowalla.git\n","!wget -q --show-progress -O gowalla/val.txt https://github.com/RecoHut-Datasets/gowalla/raw/main/silver/v1/val.txt\n","\n","# set paths\n","training_path='./gowalla/train.txt'\n","testing_path='./gowalla/test.txt'\n","val_path='./gowalla/val.txt'\n","\n","# meta\n","user_num=29858\n","item_num=40981 \n","factor_num=64\n","batch_size=2048*512\n","top_k=20 \n","num_negative_test_val=-1##all\n","\n","#testing\n","start_i_test=3\n","end_i_test=4\n","setp=1\n","\n","path_save_base = './datanpy'\n","if not os.path.exists(path_save_base):\n"," os.makedirs(path_save_base) \n","\n","run_id='0'\n","path_save_log_base='./log/'+dataset+'/newloss'+run_id\n","if not os.path.exists(path_save_log_base):\n"," os.makedirs(path_save_log_base) \n","\n","result_file=open(path_save_log_base+'/results.txt','w+')\n","\n","path_save_model_base='./newlossModel/'+dataset+'/s'+run_id\n","if not os.path.exists(path_save_model_base):\n"," os.makedirs(path_save_model_base)"],"metadata":{"id":"V5NS5Z5ULdnO"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"iYvFa6MOwiF0"},"source":["data2npy"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1v2H-ZUPwjUi","executionInfo":{"status":"ok","timestamp":1639038539256,"user_tz":-330,"elapsed":4316,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"6f97fdcd-fb87-481f-d314-dee370d6ae73"},"source":["train_data_user = defaultdict(set)\n","train_data_item = defaultdict(set) \n","links_file = open(training_path)\n","num_u=0\n","num_u_i=0\n","for _, line in enumerate(links_file):\n"," line=line.strip('\\n')\n"," tmp = line.split(' ')\n"," num_u_i+=len(tmp)-1\n"," num_u+=1\n"," u_id=int(tmp[0])\n"," for i_id in tmp[1:]: \n"," train_data_user[u_id].add(int(i_id))\n"," train_data_item[int(i_id)].add(u_id)\n","np.save(os.path.join(path_save_base,'training_set.npy'),[train_data_user,train_data_item,num_u_i]) \n","print(num_u,num_u_i)\n"," \n","test_data_user = defaultdict(set)\n","test_data_item = defaultdict(set) \n","links_file = open(testing_path)\n","num_u=0\n","num_u_i=0\n","for _, line in enumerate(links_file):\n"," line=line.strip('\\n')\n"," tmp = line.split(' ')\n"," num_u_i+=len(tmp)-1\n"," num_u+=1\n"," u_id=int(tmp[0])\n"," for i_id in tmp[1:]: \n"," test_data_user[u_id].add(int(i_id))\n"," test_data_item[int(i_id)].add(u_id)\n","np.save(os.path.join(path_save_base,'testing_set.npy'),[test_data_user,test_data_item,num_u_i]) \n","print(num_u,num_u_i)\n","\n","\n","val_data_user = defaultdict(set)\n","val_data_item = defaultdict(set) \n","links_file = open(val_path)\n","num_u=0\n","num_u_i=0\n","for _, line in enumerate(links_file):\n"," line=line.strip('\\n')\n"," tmp = line.split(' ')\n"," num_u_i+=len(tmp)-1\n"," num_u+=1\n"," u_id=int(tmp[0])\n"," for i_id in tmp[1:]: \n"," val_data_user[u_id].add(int(i_id))\n"," val_data_item[int(i_id)].add(u_id)\n","np.save(os.path.join(path_save_base,'val_set.npy'),[val_data_user,val_data_item,num_u_i]) \n","print(num_u,num_u_i)\n","\n","\n","user_rating_set_all = defaultdict(set)\n","for u in range(num_u):\n"," train_tmp = set()\n"," test_tmp = set() \n"," val_tmp = set() \n"," if u in train_data_user:\n"," train_tmp = train_data_user[u]\n"," if u in test_data_user:\n"," test_tmp = test_data_user[u] \n"," if u in val_data_user:\n"," val_tmp = val_data_user[u] \n"," user_rating_set_all[u]=train_tmp|test_tmp|val_tmp\n","np.save(os.path.join(path_save_base,'user_rating_set_all.npy'),user_rating_set_all) "],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["29858 810128\n","29858 217242\n","29857 108621\n"]}]},{"cell_type":"markdown","metadata":{"id":"0iTyipe9wzfU"},"source":["## Dataset"]},{"cell_type":"code","metadata":{"id":"LBJTWOBOxph7"},"source":["class BPRData(data.Dataset):\n"," def __init__(self,train_dict=None,num_item=0, num_ng=1, is_training=None, data_set_count=0,all_rating=None):\n"," super(BPRData, self).__init__()\n","\n"," self.num_item = num_item\n"," self.train_dict = train_dict\n"," self.num_ng = num_ng\n"," self.is_training = is_training\n"," self.data_set_count = data_set_count\n"," self.all_rating=all_rating\n"," self.set_all_item=set(range(num_item)) \n","\n"," def ng_sample(self):\n"," # assert self.is_training, 'no need to sampling when testing'\n"," # print('ng_sample----is----call-----') \n"," self.features_fill = []\n"," for user_id in self.train_dict:\n"," positive_list=self.train_dict[user_id]#self.train_dict[user_id]\n"," all_positive_list=self.all_rating[user_id]\n"," #item_i: positive item ,,item_j:negative item \n"," # temp_neg=list(self.set_all_item-all_positive_list)\n"," # random.shuffle(temp_neg)\n"," # count=0\n"," # for item_i in positive_list:\n"," # for t in range(self.num_ng): \n"," # self.features_fill.append([user_id,item_i,temp_neg[count]])\n"," # count+=1 \n"," for item_i in positive_list: \n"," for t in range(self.num_ng):\n"," item_j=np.random.randint(self.num_item)\n"," while item_j in all_positive_list:\n"," item_j=np.random.randint(self.num_item)\n"," self.features_fill.append([user_id,item_i,item_j]) \n"," \n"," def __len__(self): \n"," return self.num_ng*self.data_set_count#return self.num_ng*len(self.train_dict)\n"," \n","\n"," def __getitem__(self, idx):\n"," features = self.features_fill \n"," \n"," user = features[idx][0]\n"," item_i = features[idx][1]\n"," item_j = features[idx][2] \n"," return user, item_i, item_j "],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class resData(data.Dataset):\n"," def __init__(self,train_dict=None,batch_size=0,num_item=0,all_pos=None):\n"," super(resData, self).__init__() \n"," \n"," self.train_dict = train_dict \n"," self.batch_size = batch_size\n"," self.all_pos_train=all_pos \n","\n"," self.features_fill = []\n"," for user_id in self.train_dict:\n"," self.features_fill.append(user_id)\n"," self.set_all=set(range(num_item))\n"," \n"," def __len__(self): \n"," return math.ceil(len(self.train_dict)*1.0/self.batch_size)#self.data_set_count==batch_size\n"," \n","\n"," def __getitem__(self, idx): \n"," \n"," user_test=[]\n"," item_test=[]\n"," split_test=[]\n"," for i in range(self.batch_size):#self.data_set_count==batch_size \n"," index_my=self.batch_size*idx+i \n"," if index_my == len(self.train_dict):\n"," break \n"," user = self.features_fill[index_my]\n"," item_i_list = list(self.train_dict[user])\n"," item_j_list = list(self.set_all-self.all_pos_train[user])\n"," # pdb.set_trace() \n"," u_i=[user]*(len(item_i_list)+len(item_j_list))\n"," user_test.extend(u_i)\n"," item_test.extend(item_i_list)\n"," item_test.extend(item_j_list) \n"," split_test.append([(len(item_i_list)+len(item_j_list)),len(item_j_list)]) \n"," \n"," return torch.from_numpy(np.array(user_test)), torch.from_numpy(np.array(item_test)), split_test"],"metadata":{"id":"c3O3h1d8J6Yx"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"acpOF47Ix24z"},"source":["## Evaluate"]},{"cell_type":"code","metadata":{"id":"7p77Vhgxx9D9"},"source":["def metrics_loss(model, test_val_loader_loss, batch_size): \n"," start_time = time.time() \n"," loss_sum=[]\n"," loss_sum2=[]\n"," for user, item_i, item_j in test_val_loader_loss:\n"," user = user.cuda()\n"," item_i = item_i.cuda()\n"," item_j = item_j.cuda() \n"," \n"," prediction_i, prediction_j,loss,loss2 = model(user, item_i, item_j) \n"," loss_sum.append(loss.item()) \n"," loss_sum2.append(loss2.item())\n","\n"," # if np.isnan(loss2.item()).any():\n"," # pdb.set_trace()\n"," # pdb.set_trace()\n"," elapsed_time = time.time() - start_time\n"," test_val_loss1=round(np.mean(loss_sum),4)\n"," test_val_loss=round(np.mean(loss_sum2),4)\n"," str_print_val_loss=' val loss:'+str(test_val_loss)\n"," # print(round(elapsed_time,3))\n"," # print(test_val_loss1,test_val_loss)\n"," return test_val_loss"],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def hr_ndcg(indices_sort_top,index_end_i,top_k): \n"," hr_topK=0\n"," ndcg_topK=0\n","\n"," ndcg_max=[0]*top_k\n"," temp_max_ndcg=0\n"," for i_topK in range(top_k):\n"," temp_max_ndcg+=1.0/math.log(i_topK+2)\n"," ndcg_max[i_topK]=temp_max_ndcg\n","\n"," max_hr=top_k\n"," max_ndcg=ndcg_max[top_k-1]\n"," if index_end_i"],"metadata":{"id":"7OFu8P4HU2Dy"}},{"cell_type":"code","metadata":{"id":"KpquhpXeyKp3"},"source":["class BPR(nn.Module):\n"," def __init__(self, user_num, item_num, factor_num,user_item_matrix,item_user_matrix,d_i_train,d_j_train):\n"," super(BPR, self).__init__()\n"," \"\"\"\n"," user_num: number of users;\n"," item_num: number of items;\n"," factor_num: number of predictive factors.\n"," \"\"\" \n"," self.user_item_matrix = user_item_matrix\n"," self.item_user_matrix = item_user_matrix\n"," self.embed_user = nn.Embedding(user_num, factor_num)\n"," self.embed_item = nn.Embedding(item_num, factor_num) \n","\n"," for i in range(len(d_i_train)):\n"," d_i_train[i]=[d_i_train[i]]\n"," for i in range(len(d_j_train)):\n"," d_j_train[i]=[d_j_train[i]]\n","\n"," self.d_i_train=torch.cuda.FloatTensor(d_i_train)\n"," self.d_j_train=torch.cuda.FloatTensor(d_j_train)\n"," self.d_i_train=self.d_i_train.expand(-1,factor_num)\n"," self.d_j_train=self.d_j_train.expand(-1,factor_num)\n","\n"," nn.init.normal_(self.embed_user.weight, std=0.01)\n"," nn.init.normal_(self.embed_item.weight, std=0.01) \n","\n"," def forward(self, user, item_i, item_j): \n","\n"," users_embedding=self.embed_user.weight\n"," items_embedding=self.embed_item.weight \n","\n"," gcn1_users_embedding = (torch.sparse.mm(self.user_item_matrix, items_embedding) + users_embedding.mul(self.d_i_train))#*2. #+ users_embedding\n"," gcn1_items_embedding = (torch.sparse.mm(self.item_user_matrix, users_embedding) + items_embedding.mul(self.d_j_train))#*2. #+ items_embedding\n"," \n"," gcn2_users_embedding = (torch.sparse.mm(self.user_item_matrix, gcn1_items_embedding) + gcn1_users_embedding.mul(self.d_i_train))#*2. + users_embedding\n"," gcn2_items_embedding = (torch.sparse.mm(self.item_user_matrix, gcn1_users_embedding) + gcn1_items_embedding.mul(self.d_j_train))#*2. + items_embedding\n"," \n"," gcn3_users_embedding = (torch.sparse.mm(self.user_item_matrix, gcn2_items_embedding) + gcn2_users_embedding.mul(self.d_i_train))#*2. + gcn1_users_embedding\n"," gcn3_items_embedding = (torch.sparse.mm(self.item_user_matrix, gcn2_users_embedding) + gcn2_items_embedding.mul(self.d_j_train))#*2. + gcn1_items_embedding\n"," \n"," # gcn4_users_embedding = (torch.sparse.mm(self.user_item_matrix, gcn3_items_embedding) + gcn3_users_embedding.mul(self.d_i_train))#*2. + gcn1_users_embedding\n"," # gcn4_items_embedding = (torch.sparse.mm(self.item_user_matrix, gcn3_users_embedding) + gcn3_items_embedding.mul(self.d_j_train))#*2. + gcn1_items_embedding\n"," \n"," gcn_users_embedding= torch.cat((users_embedding,gcn1_users_embedding,gcn2_users_embedding,gcn3_users_embedding),-1)#+gcn4_users_embedding\n"," gcn_items_embedding= torch.cat((items_embedding,gcn1_items_embedding,gcn2_items_embedding,gcn3_items_embedding),-1)#+gcn4_items_embedding#\n"," \n"," \n"," user = F.embedding(user,gcn_users_embedding)\n"," item_i = F.embedding(item_i,gcn_items_embedding)\n"," item_j = F.embedding(item_j,gcn_items_embedding) \n"," # # pdb.set_trace() \n"," prediction_i = (user * item_i).sum(dim=-1)\n"," prediction_j = (user * item_j).sum(dim=-1) \n"," # loss=-((rediction_i-prediction_j).sigmoid())**2#self.loss(prediction_i,prediction_j)#.sum()\n"," l2_regulization = 0.01*(user**2+item_i**2+item_j**2).sum(dim=-1)\n"," # l2_regulization = 0.01*((gcn1_users_embedding**2).sum(dim=-1).mean()+(gcn1_items_embedding**2).sum(dim=-1).mean())\n"," \n"," loss2= -((prediction_i - prediction_j).sigmoid().log().mean())\n"," # loss= loss2 + l2_regulization\n"," loss= -((prediction_i - prediction_j)).sigmoid().log().mean() +l2_regulization.mean()\n"," # pdb.set_trace()\n"," return prediction_i, prediction_j,loss,loss2"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wAwTA05SyNpP"},"source":["train_dataset = BPRData(\n"," train_dict=training_user_set, num_item=item_num, num_ng=5, is_training=True,\\\n"," data_set_count=training_set_count,all_rating=user_rating_set_all)\n","train_loader = DataLoader(train_dataset,\n"," batch_size=batch_size, shuffle=True, num_workers=2)\n"," \n","testing_dataset_loss = BPRData(\n"," train_dict=testing_user_set, num_item=item_num, num_ng=5, is_training=True,\\\n"," data_set_count=testing_set_count,all_rating=user_rating_set_all)\n","testing_loader_loss = DataLoader(testing_dataset_loss,\n"," batch_size=batch_size, shuffle=False, num_workers=0)\n","\n","val_dataset_loss = BPRData(\n"," train_dict=val_user_set, num_item=item_num, num_ng=5, is_training=True,\\\n"," data_set_count=val_set_count,all_rating=user_rating_set_all)\n","val_loader_loss = DataLoader(val_dataset_loss,\n"," batch_size=batch_size, shuffle=False, num_workers=0)\n"," \n"," \n","model = BPR(user_num, item_num, factor_num,sparse_u_i,sparse_i_u,d_i_train,d_j_train)\n","model=model.to('cuda') \n","\n","optimizer_bpr = torch.optim.Adam(model.parameters(), lr=0.005)#, betas=(0.5, 0.99))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Training"],"metadata":{"id":"SO9SZNWfMuh5"}},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"D9ksZjDmyMBg","executionInfo":{"status":"ok","timestamp":1639038912797,"user_tz":-330,"elapsed":237974,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"7cbe674b-26d3-407f-ed5a-890bf00428c9"},"source":["########################### TRAINING #####################################\n"," \n","# testing_loader_loss.dataset.ng_sample() \n","\n","print('--------training processing-------')\n","count, best_hr = 0, 0\n","for epoch in range(5):\n"," model.train() \n"," start_time = time.time()\n"," train_loader.dataset.ng_sample()\n"," # pdb.set_trace()\n"," print('train data of ng_sample is end')\n"," # elapsed_time = time.time() - start_time\n"," # print(' time:'+str(round(elapsed_time,1)))\n"," # start_time = time.time()\n"," \n"," train_loss_sum=[]\n"," train_loss_sum2=[]\n"," for user, item_i, item_j in train_loader:\n"," user = user.cuda()\n"," item_i = item_i.cuda()\n"," item_j = item_j.cuda() \n","\n"," model.zero_grad()\n"," prediction_i, prediction_j,loss,loss2 = model(user, item_i, item_j) \n"," loss.backward()\n"," optimizer_bpr.step() \n"," count += 1 \n"," train_loss_sum.append(loss.item()) \n"," train_loss_sum2.append(loss2.item()) \n"," # print(count)\n","\n"," elapsed_time = time.time() - start_time\n"," train_loss=round(np.mean(train_loss_sum[:-1]),4)\n"," train_loss2=round(np.mean(train_loss_sum2[:-1]),4)\n"," str_print_train=\"epoch:\"+str(epoch)+' time:'+str(round(elapsed_time,1))+'\\t train loss:'+str(train_loss)+\"=\"+str(train_loss2)+\"+\" \n"," print('--train--',elapsed_time)\n","\n"," PATH_model=path_save_model_base+'/epoch'+str(epoch)+'.pt'\n"," torch.save(model.state_dict(), PATH_model)\n"," \n"," model.eval() \n"," # ######test and val########### \n"," val_loader_loss.dataset.ng_sample() \n"," val_loss=metrics_loss(model,val_loader_loss,batch_size) \n"," # str_print_train+=' val loss:'+str(val_loss)\n","\n"," testing_loader_loss.dataset.ng_sample() \n"," test_loss=metrics_loss(model,testing_loader_loss,batch_size) \n"," print(str_print_train+' val loss:'+str(val_loss)+' test loss:'+str(test_loss)) \n"," result_file.write(str_print_train+' val loss:'+str(val_loss)+' test loss:'+str(test_loss)) \n"," result_file.write('\\n') \n"," result_file.flush() "],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["--------training processing-------\n","train data of ng_sample is end\n","--train-- 36.8933470249176\n","epoch:0 time:36.9\t train loss:0.6929=0.6927+ val loss:0.6856 test loss:0.6863\n","train data of ng_sample is end\n","--train-- 35.34687113761902\n","epoch:1 time:35.3\t train loss:0.6765=0.6749+ val loss:0.6315 test loss:0.6367\n","train data of ng_sample is end\n","--train-- 33.52420949935913\n","epoch:2 time:33.5\t train loss:0.6113=0.6035+ val loss:0.5238 test loss:0.5349\n","train data of ng_sample is end\n","--train-- 35.61563539505005\n","epoch:3 time:35.6\t train loss:0.5038=0.4826+ val loss:0.4007 test loss:0.4129\n","train data of ng_sample is end\n","--train-- 34.377206325531006\n","epoch:4 time:34.4\t train loss:0.399=0.3562+ val loss:0.3048 test loss:0.3141\n"]}]},{"cell_type":"markdown","metadata":{"id":"jxYefqWOzUSX"},"source":["## Testing"]},{"cell_type":"code","source":["def readD(set_matrix,num_):\n"," user_d=[] \n"," for i in range(num_):\n"," len_set=1.0/(len(set_matrix[i])+1) \n"," user_d.append(len_set)\n"," return user_d\n","u_d=readD(training_user_set,user_num)\n","i_d=readD(training_item_set,item_num)\n","d_i_train=u_d\n","d_j_train=i_d"],"metadata":{"id":"kWWePLOTVxSg"},"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ONukrcRw2ngS"},"source":["#user-item to user-item matrix and item-user matrix\n","def readTrainSparseMatrix(set_matrix,is_user):\n"," user_items_matrix_i=[]\n"," user_items_matrix_v=[] \n"," if is_user:\n"," d_i=u_d\n"," d_j=i_d\n"," else:\n"," d_i=i_d\n"," d_j=u_d\n"," for i in set_matrix:\n"," len_set=len(set_matrix[i]) \n"," for j in set_matrix[i]:\n"," user_items_matrix_i.append([i,j])\n"," d_i_j=np.sqrt(d_i[i]*d_j[j])\n"," #1/sqrt((d_i+1)(d_j+1)) \n"," user_items_matrix_v.append(d_i_j)#(1./len_set) \n"," user_items_matrix_i=torch.cuda.LongTensor(user_items_matrix_i)\n"," user_items_matrix_v=torch.cuda.FloatTensor(user_items_matrix_v)\n"," return torch.sparse.FloatTensor(user_items_matrix_i.t(), user_items_matrix_v)\n","\n","sparse_u_i=readTrainSparseMatrix(training_user_set,True)\n","sparse_i_u=readTrainSparseMatrix(training_item_set,False)\n","\n","#user-item to user-item matrix and item-user matrix\n","# pdb.set_trace()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"37odx_9d2xCS"},"source":["class BPR(nn.Module):\n"," def __init__(self, user_num, item_num, factor_num,user_item_matrix,item_user_matrix,d_i_train,d_j_train):\n"," super(BPR, self).__init__()\n"," \"\"\"\n"," user_num: number of users;\n"," item_num: number of items;\n"," factor_num: number of predictive factors.\n"," \"\"\" \n"," self.user_item_matrix = user_item_matrix\n"," self.item_user_matrix = item_user_matrix\n"," self.embed_user = nn.Embedding(user_num, factor_num)\n"," self.embed_item = nn.Embedding(item_num, factor_num) \n","\n"," for i in range(len(d_i_train)):\n"," d_i_train[i]=[d_i_train[i]]\n"," for i in range(len(d_j_train)):\n"," d_j_train[i]=[d_j_train[i]]\n","\n"," self.d_i_train=torch.cuda.FloatTensor(d_i_train)\n"," self.d_j_train=torch.cuda.FloatTensor(d_j_train)\n"," self.d_i_train=self.d_i_train.expand(-1,factor_num)\n"," self.d_j_train=self.d_j_train.expand(-1,factor_num)\n","\n"," nn.init.normal_(self.embed_user.weight, std=0.01)\n"," nn.init.normal_(self.embed_item.weight, std=0.01) \n","\n"," def forward(self, user, item_i, item_j): \n","\n"," users_embedding=self.embed_user.weight\n"," items_embedding=self.embed_item.weight \n","\n"," gcn1_users_embedding = (torch.sparse.mm(self.user_item_matrix, items_embedding) + users_embedding.mul(self.d_i_train))#*2. #+ users_embedding\n"," gcn1_items_embedding = (torch.sparse.mm(self.item_user_matrix, users_embedding) + items_embedding.mul(self.d_j_train))#*2. #+ items_embedding\n"," \n"," gcn2_users_embedding = (torch.sparse.mm(self.user_item_matrix, gcn1_items_embedding) + gcn1_users_embedding.mul(self.d_i_train))#*2. + users_embedding\n"," gcn2_items_embedding = (torch.sparse.mm(self.item_user_matrix, gcn1_users_embedding) + gcn1_items_embedding.mul(self.d_j_train))#*2. + items_embedding\n"," \n"," gcn3_users_embedding = (torch.sparse.mm(self.user_item_matrix, gcn2_items_embedding) + gcn2_users_embedding.mul(self.d_i_train))#*2. + gcn1_users_embedding\n"," gcn3_items_embedding = (torch.sparse.mm(self.item_user_matrix, gcn2_users_embedding) + gcn2_items_embedding.mul(self.d_j_train))#*2. + gcn1_items_embedding\n"," \n"," gcn_users_embedding= torch.cat((users_embedding,gcn1_users_embedding,gcn2_users_embedding,gcn3_users_embedding),-1)#+gcn4_users_embedding\n"," gcn_items_embedding= torch.cat((items_embedding,gcn1_items_embedding,gcn2_items_embedding,gcn3_items_embedding),-1)#+gcn4_items_embedding#\n"," \n"," \n"," g0_mean=torch.mean(users_embedding)\n"," g0_var=torch.var(users_embedding)\n"," g1_mean=torch.mean(gcn1_users_embedding)\n"," g1_var=torch.var(gcn1_users_embedding) \n"," g2_mean=torch.mean(gcn2_users_embedding)\n"," g2_var=torch.var(gcn2_users_embedding)\n"," g3_mean=torch.mean(gcn3_users_embedding)\n"," g3_var=torch.var(gcn3_users_embedding)\n"," # g4_mean=torch.mean(gcn4_users_embedding)\n"," # g4_var=torch.var(gcn4_users_embedding)\n"," # g5_mean=torch.mean(gcn5_users_embedding)\n"," # g5_var=torch.var(gcn5_users_embedding)\n"," # g6_mean=torch.mean(gcn6_users_embedding)\n"," # g6_var=torch.var(gcn6_users_embedding)\n"," g_mean=torch.mean(gcn_users_embedding)\n"," g_var=torch.var(gcn_users_embedding)\n","\n"," i0_mean=torch.mean(items_embedding)\n"," i0_var=torch.var(items_embedding)\n"," i1_mean=torch.mean(gcn1_items_embedding)\n"," i1_var=torch.var(gcn1_items_embedding)\n"," i2_mean=torch.mean(gcn2_items_embedding)\n"," i2_var=torch.var(gcn2_items_embedding)\n"," i3_mean=torch.mean(gcn3_items_embedding)\n"," i3_var=torch.var(gcn3_items_embedding)\n"," # i4_mean=torch.mean(gcn4_items_embedding)\n"," # i4_var=torch.var(gcn4_items_embedding) \n"," # i5_mean=torch.mean(gcn5_items_embedding)\n"," # i5_var=torch.var(gcn5_items_embedding)\n"," # i6_mean=torch.mean(gcn6_items_embedding)\n"," # i6_var=torch.var(gcn6_items_embedding)\n"," i_mean=torch.mean(gcn_items_embedding)\n"," i_var=torch.var(gcn_items_embedding)\n","\n"," # pdb.set_trace() \n","\n"," str_user=str(round(g0_mean.item(),7))+' '\n"," str_user+=str(round(g0_var.item(),7))+' '\n"," str_user+=str(round(g1_mean.item(),7))+' '\n"," str_user+=str(round(g1_var.item(),7))+' '\n"," str_user+=str(round(g2_mean.item(),7))+' '\n"," str_user+=str(round(g2_var.item(),7))+' '\n"," str_user+=str(round(g3_mean.item(),7))+' '\n"," str_user+=str(round(g3_var.item(),7))+' '\n"," # str_user+=str(round(g4_mean.item(),7))+' '\n"," # str_user+=str(round(g4_var.item(),7))+' '\n"," # str_user+=str(round(g5_mean.item(),7))+' '\n"," # str_user+=str(round(g5_var.item(),7))+' '\n"," # str_user+=str(round(g6_mean.item(),7))+' '\n"," # str_user+=str(round(g6_var.item(),7))+' '\n"," str_user+=str(round(g_mean.item(),7))+' '\n"," str_user+=str(round(g_var.item(),7))+' '\n","\n"," str_item=str(round(i0_mean.item(),7))+' '\n"," str_item+=str(round(i0_var.item(),7))+' '\n"," str_item+=str(round(i1_mean.item(),7))+' '\n"," str_item+=str(round(i1_var.item(),7))+' '\n"," str_item+=str(round(i2_mean.item(),7))+' '\n"," str_item+=str(round(i2_var.item(),7))+' '\n"," str_item+=str(round(i3_mean.item(),7))+' '\n"," str_item+=str(round(i3_var.item(),7))+' '\n"," # str_item+=str(round(i4_mean.item(),7))+' '\n"," # str_item+=str(round(i4_var.item(),7))+' '\n"," # str_item+=str(round(i5_mean.item(),7))+' '\n"," # str_item+=str(round(i5_var.item(),7))+' '\n"," # str_item+=str(round(i6_mean.item(),7))+' '\n"," # str_item+=str(round(i6_var.item(),7))+' '\n"," str_item+=str(round(i_mean.item(),7))+' '\n"," str_item+=str(round(i_var.item(),7))+' '\n"," print(str_user)\n"," print(str_item)\n"," return gcn_users_embedding, gcn_items_embedding,str_user,str_item \n","\n","test_batch=52#int(batch_size/32) \n","testing_dataset = resData(train_dict=testing_user_set, batch_size=test_batch,num_item=item_num,all_pos=training_user_set)\n","testing_loader = DataLoader(testing_dataset,batch_size=1, shuffle=False, num_workers=0) \n"," \n","model = BPR(user_num, item_num, factor_num,sparse_u_i,sparse_i_u,d_i_train,d_j_train)\n","model=model.to('cuda')\n"," \n","optimizer_bpr = torch.optim.Adam(model.parameters(), lr=0.001)#, betas=(0.5, 0.99))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"LbN03HqY2scu","executionInfo":{"status":"ok","timestamp":1639039232397,"user_tz":-330,"elapsed":164076,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"c20f2291-2639-45ea-a4a9-ad4f148f724a"},"source":["########################### TESTING ##################################### \n","# testing_loader_loss.dataset.ng_sample() \n","\n","def largest_indices(ary, n):\n"," \"\"\"Returns the n largest indices from a numpy array.\"\"\"\n"," flat = ary.flatten()\n"," indices = np.argpartition(flat, -n)[-n:]\n"," indices = indices[np.argsort(-flat[indices])]\n"," return np.unravel_index(indices, ary.shape)\n","\n","print('--------test processing-------')\n","count, best_hr = 0, 0\n","for epoch in range(start_i_test,end_i_test,setp):\n"," model.train() \n","\n"," PATH_model=path_save_model_base+'/epoch'+str(epoch)+'.pt'\n"," #torch.save(model.state_dict(), PATH_model) \n"," model.load_state_dict(torch.load(PATH_model)) \n"," model.eval() \n"," # ######test and val########### \n"," gcn_users_embedding, gcn_items_embedding,gcn_user_emb,gcn_item_emb= model(torch.cuda.LongTensor([0]), torch.cuda.LongTensor([0]), torch.cuda.LongTensor([0])) \n"," user_e=gcn_users_embedding.cpu().detach().numpy()\n"," item_e=gcn_items_embedding.cpu().detach().numpy()\n"," all_pre=np.matmul(user_e,item_e.T) \n"," HR, NDCG = [], [] \n"," set_all=set(range(item_num)) \n"," #spend 461s \n"," test_start_time = time.time()\n"," for u_i in testing_user_set: \n"," item_i_list = list(testing_user_set[u_i])\n"," index_end_i=len(item_i_list)\n"," item_j_list = list(set_all-training_user_set[u_i]-testing_user_set[u_i])\n"," item_i_list.extend(item_j_list) \n","\n"," pre_one=all_pre[u_i][item_i_list] \n"," indices=largest_indices(pre_one, top_k)\n"," indices=list(indices[0]) \n","\n"," hr_t,ndcg_t=hr_ndcg(indices,index_end_i,top_k) \n"," elapsed_time = time.time() - test_start_time \n"," HR.append(hr_t)\n"," NDCG.append(ndcg_t) \n"," hr_test=round(np.mean(HR),4)\n"," ndcg_test=round(np.mean(NDCG),4) \n"," \n"," # test_loss,hr_test,ndcg_test = metrics(model,testing_loader,top_k,num_negative_test_val,batch_size) \n"," str_print_evl=\"epoch:\"+str(epoch)+'time:'+str(round(elapsed_time,2))+\"\\t test\"+\" hit:\"+str(hr_test)+' ndcg:'+str(ndcg_test) \n"," print(str_print_evl) \n"," result_file.write(gcn_user_emb)\n"," result_file.write('\\n')\n"," result_file.write(gcn_item_emb)\n"," result_file.write('\\n') \n","\n"," result_file.write(str_print_evl)\n"," result_file.write('\\n')\n"," result_file.flush()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["--------test processing-------\n","0.0005065 0.0039031 0.0008974 0.0035089 0.0004212 0.0024023 0.0008231 0.0026235 0.000662 0.0031095 \n","0.0012799 0.003691 0.0006682 0.0020425 0.0010818 0.0021988 0.0006408 0.0016047 0.0009177 0.0023843 \n","epoch:3time:151.82\t test hit:0.1021 ndcg:0.0841\n"]}]},{"cell_type":"markdown","metadata":{"id":"TLHw8weh7OQB"},"source":["---"]},{"cell_type":"code","metadata":{"id":"ZYGZlyeY7OQD"},"source":["!apt-get -qq install tree\n","!rm -r sample_data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oVruV5yt7OQD","executionInfo":{"status":"ok","timestamp":1638797225389,"user_tz":-330,"elapsed":24,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"8cc165b8-6167-4329-c3f1-5cd2ae01d647"},"source":["!tree -h --du ."],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[".\n","├── [ 15M] datanpy\n","│   ├── [2.9M] testing_set.npy\n","│   ├── [6.3M] training_set.npy\n","│   ├── [3.6M] user_rating_set_all.npy\n","│   └── [2.1M] val_set.npy\n","├── [7.3M] gowalla\n","│   ├── [495K] item_list.txt\n","│   ├── [1.0K] README.md\n","│   ├── [1.3M] test.txt\n","│   ├── [4.4M] train.txt\n","│   ├── [343K] user_list.txt\n","│   └── [752K] val.txt\n","├── [ 12K] log\n","│   └── [8.2K] gowalla\n","│   └── [4.2K] newlosss0\n","│   ├── [ 0] results_hdcg_hr.txt\n","│   └── [ 245] results.txt\n","└── [ 86M] newlossModel\n"," └── [ 86M] gowalla\n"," └── [ 86M] ss0\n"," ├── [ 17M] epoch0.pt\n"," ├── [ 17M] epoch1.pt\n"," ├── [ 17M] epoch2.pt\n"," ├── [ 17M] epoch3.pt\n"," └── [ 17M] epoch4.pt\n","\n"," 109M used in 8 directories, 17 files\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MrbNlTOW7OQE","executionInfo":{"status":"ok","timestamp":1638797236830,"user_tz":-330,"elapsed":3692,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b4f06826-9df0-476d-ecb4-ff603183437b"},"source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2021-12-06 13:27:22\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.104+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","pandas : 1.1.5\n","IPython : 5.5.0\n","scipy : 1.4.1\n","numpy : 1.19.5\n","torchvision: 0.11.1+cu111\n","sys : 3.7.12 (default, Sep 10 2021, 00:21:48) \n","[GCC 7.5.0]\n","argparse : 1.1\n","torch : 1.10.0+cu111\n","\n"]}]},{"cell_type":"code","metadata":{"id":"jzgrKrkC7Y-N","executionInfo":{"status":"ok","timestamp":1638797255976,"user_tz":-330,"elapsed":448,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"672fccb0-588b-49ca-8a77-5ab549796055","colab":{"base_uri":"https://localhost:8080/"}},"source":["!nvidia-smi"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mon Dec 6 13:27:41 2021 \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 495.44 Driver Version: 460.32.03 CUDA Version: 11.2 |\n","|-------------------------------+----------------------+----------------------+\n","| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n","| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n","| | | MIG M. |\n","|===============================+======================+======================|\n","| 0 Tesla K80 Off | 00000000:00:04.0 Off | 0 |\n","| N/A 73C P0 74W / 149W | 761MiB / 11441MiB | 0% Default |\n","| | | N/A |\n","+-------------------------------+----------------------+----------------------+\n"," \n","+-----------------------------------------------------------------------------+\n","| Processes: |\n","| GPU GI CI PID Type Process name GPU Memory |\n","| ID ID Usage |\n","|=============================================================================|\n","| No running processes found |\n","+-----------------------------------------------------------------------------+\n"]}]},{"cell_type":"markdown","metadata":{"id":"9Kw61_pe7OQE"},"source":["---"]},{"cell_type":"markdown","metadata":{"id":"OCasCymq7OQG"},"source":["**END**"]}]} \ No newline at end of file +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-09-lrgccf-gowalla.ipynb","provenance":[{"file_id":"https://github.com/recohut/notebook/blob/master/_notebooks/2022-01-09-lrgccf-gowalla.ipynb","timestamp":1644607994127},{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P174968%20%7C%20LR-GCCF%20on%20Gowalla.ipynb","timestamp":1644598450989}],"collapsed_sections":[],"authorship_tag":"ABX9TyNgfemhKTG44apSVA7UTAxV"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","source":["# LR-GCCF on Gowalla"],"metadata":{"id":"mVyRGyhdtRFw"}},{"cell_type":"markdown","source":["## Executive summary"],"metadata":{"id":"Y3oNohENVHAH"}},{"cell_type":"markdown","source":["| | |\n","| --- | --- |\n","| Problem | GCNs suffer from training difficulty due to non-linear activations, and over-smoothing problem. |\n","| Hypothesis | removing non-linearities would enhance recommendation performance. |\n","| Solution | Linear model with residual network structure |\n","| Dataset | Gowalla |\n","| Preprocessing | we remove users (items) that have less than 10 interaction records. After that, we randomly select 80% of the records for training, 10% for validation and the remaining 10% for test. |\n","| Metrics | HR, NDCG |\n","| Hyperparams | There are two important parameters: the dimension D of the user and item embedding matrix E, and the regularization parameter λ in the objective function. The embedding size is fixed to 64. We try the regularization parameter λ in the range [0.0001, 0.001, 0.01, 0.1], and find λ = 0.01 reaches the best performance. |\n","| Models | LR-GCCF |\n","| Cluster | PyTorch with GPU |"],"metadata":{"id":"DK7RWly8VKvU"}},{"cell_type":"markdown","source":["## Process flow\n","\n","![](https://github.com/RecoHut-Stanzas/S794944/raw/main/images/process_flow.svg)"],"metadata":{"id":"s4weKl5kcU3v"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"paRynetXLa6r"}},{"cell_type":"code","source":["import random\n","import torch \n","import time\n","import pdb\n","import math\n","import os\n","import sys\n","from shutil import copyfile\n","from collections import defaultdict\n","import numpy as np\n","import pandas as pd \n","import scipy.sparse as sp \n","\n","import torch\n","import torch.nn as nn \n","from torch.utils.data import DataLoader\n","import torch.nn.functional as F\n","import torch.autograd as autograd\n","from torch.autograd import Variable\n","import torch.utils.data as data"],"metadata":{"id":"zfsN6pxILcQG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'"],"metadata":{"id":"CkWg0kI9LjgC"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Data"],"metadata":{"id":"VhlaApRdJJDh"}},{"cell_type":"code","source":["# download\n","dataset = 'gowalla'\n","!git clone --branch v1 https://github.com/RecoHut-Datasets/gowalla.git\n","!wget -q --show-progress -O gowalla/val.txt https://github.com/RecoHut-Datasets/gowalla/raw/main/silver/v1/val.txt\n","\n","# set paths\n","training_path='./gowalla/train.txt'\n","testing_path='./gowalla/test.txt'\n","val_path='./gowalla/val.txt'\n","\n","# meta\n","user_num=29858\n","item_num=40981 \n","factor_num=64\n","batch_size=2048*512\n","top_k=20 \n","num_negative_test_val=-1##all\n","\n","#testing\n","start_i_test=3\n","end_i_test=4\n","setp=1\n","\n","path_save_base = './datanpy'\n","if not os.path.exists(path_save_base):\n"," os.makedirs(path_save_base) \n","\n","run_id='0'\n","path_save_log_base='./log/'+dataset+'/newloss'+run_id\n","if not os.path.exists(path_save_log_base):\n"," os.makedirs(path_save_log_base) \n","\n","result_file=open(path_save_log_base+'/results.txt','w+')\n","\n","path_save_model_base='./newlossModel/'+dataset+'/s'+run_id\n","if not os.path.exists(path_save_model_base):\n"," os.makedirs(path_save_model_base)"],"metadata":{"id":"V5NS5Z5ULdnO"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"iYvFa6MOwiF0"},"source":["data2npy"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1v2H-ZUPwjUi","executionInfo":{"status":"ok","timestamp":1639038539256,"user_tz":-330,"elapsed":4316,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"6f97fdcd-fb87-481f-d314-dee370d6ae73"},"source":["train_data_user = defaultdict(set)\n","train_data_item = defaultdict(set) \n","links_file = open(training_path)\n","num_u=0\n","num_u_i=0\n","for _, line in enumerate(links_file):\n"," line=line.strip('\\n')\n"," tmp = line.split(' ')\n"," num_u_i+=len(tmp)-1\n"," num_u+=1\n"," u_id=int(tmp[0])\n"," for i_id in tmp[1:]: \n"," train_data_user[u_id].add(int(i_id))\n"," train_data_item[int(i_id)].add(u_id)\n","np.save(os.path.join(path_save_base,'training_set.npy'),[train_data_user,train_data_item,num_u_i]) \n","print(num_u,num_u_i)\n"," \n","test_data_user = defaultdict(set)\n","test_data_item = defaultdict(set) \n","links_file = open(testing_path)\n","num_u=0\n","num_u_i=0\n","for _, line in enumerate(links_file):\n"," line=line.strip('\\n')\n"," tmp = line.split(' ')\n"," num_u_i+=len(tmp)-1\n"," num_u+=1\n"," u_id=int(tmp[0])\n"," for i_id in tmp[1:]: \n"," test_data_user[u_id].add(int(i_id))\n"," test_data_item[int(i_id)].add(u_id)\n","np.save(os.path.join(path_save_base,'testing_set.npy'),[test_data_user,test_data_item,num_u_i]) \n","print(num_u,num_u_i)\n","\n","\n","val_data_user = defaultdict(set)\n","val_data_item = defaultdict(set) \n","links_file = open(val_path)\n","num_u=0\n","num_u_i=0\n","for _, line in enumerate(links_file):\n"," line=line.strip('\\n')\n"," tmp = line.split(' ')\n"," num_u_i+=len(tmp)-1\n"," num_u+=1\n"," u_id=int(tmp[0])\n"," for i_id in tmp[1:]: \n"," val_data_user[u_id].add(int(i_id))\n"," val_data_item[int(i_id)].add(u_id)\n","np.save(os.path.join(path_save_base,'val_set.npy'),[val_data_user,val_data_item,num_u_i]) \n","print(num_u,num_u_i)\n","\n","\n","user_rating_set_all = defaultdict(set)\n","for u in range(num_u):\n"," train_tmp = set()\n"," test_tmp = set() \n"," val_tmp = set() \n"," if u in train_data_user:\n"," train_tmp = train_data_user[u]\n"," if u in test_data_user:\n"," test_tmp = test_data_user[u] \n"," if u in val_data_user:\n"," val_tmp = val_data_user[u] \n"," user_rating_set_all[u]=train_tmp|test_tmp|val_tmp\n","np.save(os.path.join(path_save_base,'user_rating_set_all.npy'),user_rating_set_all) "],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["29858 810128\n","29858 217242\n","29857 108621\n"]}]},{"cell_type":"markdown","metadata":{"id":"0iTyipe9wzfU"},"source":["## Dataset"]},{"cell_type":"code","metadata":{"id":"LBJTWOBOxph7"},"source":["class BPRData(data.Dataset):\n"," def __init__(self,train_dict=None,num_item=0, num_ng=1, is_training=None, data_set_count=0,all_rating=None):\n"," super(BPRData, self).__init__()\n","\n"," self.num_item = num_item\n"," self.train_dict = train_dict\n"," self.num_ng = num_ng\n"," self.is_training = is_training\n"," self.data_set_count = data_set_count\n"," self.all_rating=all_rating\n"," self.set_all_item=set(range(num_item)) \n","\n"," def ng_sample(self):\n"," # assert self.is_training, 'no need to sampling when testing'\n"," # print('ng_sample----is----call-----') \n"," self.features_fill = []\n"," for user_id in self.train_dict:\n"," positive_list=self.train_dict[user_id]#self.train_dict[user_id]\n"," all_positive_list=self.all_rating[user_id]\n"," #item_i: positive item ,,item_j:negative item \n"," # temp_neg=list(self.set_all_item-all_positive_list)\n"," # random.shuffle(temp_neg)\n"," # count=0\n"," # for item_i in positive_list:\n"," # for t in range(self.num_ng): \n"," # self.features_fill.append([user_id,item_i,temp_neg[count]])\n"," # count+=1 \n"," for item_i in positive_list: \n"," for t in range(self.num_ng):\n"," item_j=np.random.randint(self.num_item)\n"," while item_j in all_positive_list:\n"," item_j=np.random.randint(self.num_item)\n"," self.features_fill.append([user_id,item_i,item_j]) \n"," \n"," def __len__(self): \n"," return self.num_ng*self.data_set_count#return self.num_ng*len(self.train_dict)\n"," \n","\n"," def __getitem__(self, idx):\n"," features = self.features_fill \n"," \n"," user = features[idx][0]\n"," item_i = features[idx][1]\n"," item_j = features[idx][2] \n"," return user, item_i, item_j "],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class resData(data.Dataset):\n"," def __init__(self,train_dict=None,batch_size=0,num_item=0,all_pos=None):\n"," super(resData, self).__init__() \n"," \n"," self.train_dict = train_dict \n"," self.batch_size = batch_size\n"," self.all_pos_train=all_pos \n","\n"," self.features_fill = []\n"," for user_id in self.train_dict:\n"," self.features_fill.append(user_id)\n"," self.set_all=set(range(num_item))\n"," \n"," def __len__(self): \n"," return math.ceil(len(self.train_dict)*1.0/self.batch_size)#self.data_set_count==batch_size\n"," \n","\n"," def __getitem__(self, idx): \n"," \n"," user_test=[]\n"," item_test=[]\n"," split_test=[]\n"," for i in range(self.batch_size):#self.data_set_count==batch_size \n"," index_my=self.batch_size*idx+i \n"," if index_my == len(self.train_dict):\n"," break \n"," user = self.features_fill[index_my]\n"," item_i_list = list(self.train_dict[user])\n"," item_j_list = list(self.set_all-self.all_pos_train[user])\n"," # pdb.set_trace() \n"," u_i=[user]*(len(item_i_list)+len(item_j_list))\n"," user_test.extend(u_i)\n"," item_test.extend(item_i_list)\n"," item_test.extend(item_j_list) \n"," split_test.append([(len(item_i_list)+len(item_j_list)),len(item_j_list)]) \n"," \n"," return torch.from_numpy(np.array(user_test)), torch.from_numpy(np.array(item_test)), split_test"],"metadata":{"id":"c3O3h1d8J6Yx"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"acpOF47Ix24z"},"source":["## Evaluate"]},{"cell_type":"code","metadata":{"id":"7p77Vhgxx9D9"},"source":["def metrics_loss(model, test_val_loader_loss, batch_size): \n"," start_time = time.time() \n"," loss_sum=[]\n"," loss_sum2=[]\n"," for user, item_i, item_j in test_val_loader_loss:\n"," user = user.cuda()\n"," item_i = item_i.cuda()\n"," item_j = item_j.cuda() \n"," \n"," prediction_i, prediction_j,loss,loss2 = model(user, item_i, item_j) \n"," loss_sum.append(loss.item()) \n"," loss_sum2.append(loss2.item())\n","\n"," # if np.isnan(loss2.item()).any():\n"," # pdb.set_trace()\n"," # pdb.set_trace()\n"," elapsed_time = time.time() - start_time\n"," test_val_loss1=round(np.mean(loss_sum),4)\n"," test_val_loss=round(np.mean(loss_sum2),4)\n"," str_print_val_loss=' val loss:'+str(test_val_loss)\n"," # print(round(elapsed_time,3))\n"," # print(test_val_loss1,test_val_loss)\n"," return test_val_loss"],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def hr_ndcg(indices_sort_top,index_end_i,top_k): \n"," hr_topK=0\n"," ndcg_topK=0\n","\n"," ndcg_max=[0]*top_k\n"," temp_max_ndcg=0\n"," for i_topK in range(top_k):\n"," temp_max_ndcg+=1.0/math.log(i_topK+2)\n"," ndcg_max[i_topK]=temp_max_ndcg\n","\n"," max_hr=top_k\n"," max_ndcg=ndcg_max[top_k-1]\n"," if index_end_i 5.0, torch.full_like(y_pre, 5.0), y_pre)\n"," y_pre = torch.where(y_pre < 1.0, torch.full_like(y_pre, 1.0), y_pre)\n"," labels.extend(y.tolist())\n"," predicts.extend(y_pre.tolist())\n"," mse = mean_squared_error(np.array(labels), np.array(predicts))\n","\n"," return mse\n","\n","\n","def main():\n"," df = pd.read_csv('u.data', header=None, delimiter='\\t')\n"," len_df, u_max_id, i_max_id = len(df), max(df[0]) + 1, max(df[1]) + 1\n"," print(df.shape, max(df[0]), max(df[1]))\n"," x, y = df.iloc[:, :2], df.iloc[:, 2]\n"," x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=2020)\n"," x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=2020)\n"," train_loader = DataLoader(\n"," FmDataset(np.array(x_train[0]), np.array(x_train[1]), np.array(y_train).astype(np.float32)), batch_size=batch_szie)\n"," val_loader = DataLoader(FmDataset(np.array(x_val[0]), np.array(x_val[1]), np.array(y_val).astype(np.float32)), batch_size=batch_szie)\n"," test_loader = DataLoader(FmDataset(np.array(x_test[0]), np.array(x_test[1]), np.array(y_test).astype(np.float32)), batch_size=batch_szie)\n","\n"," # 模型初始化\n"," model = FM(u_max_id, i_max_id, id_embedding_dim).to(device)\n"," optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n"," loss_func = torch.nn.MSELoss().to(device)\n","\n"," # 训练模型\n"," best_val_mse, best_val_epoch = 10, 0\n"," for epoch in range(epochs):\n"," loss = train_iter(model, optimizer, train_loader, loss_func)\n"," mse = val_iter(model, val_loader)\n"," print(\"epoch:{}, loss:{:.5}, mse:{:.5}\".format(epoch, loss, mse))\n"," if best_val_mse > mse:\n"," best_val_mse, best_val_epoch = mse, epoch\n"," torch.save(model, 'best_model')\n"," print(\"best val epoch is {}, mse is {}\".format(best_val_epoch, best_val_mse))\n"," model = torch.load('best_model').to(device)\n"," test_mse = val_iter(model, test_loader)\n"," print(\"test mse is {}\".format(test_mse))\n","\n","\n","if __name__ == '__main__':\n"," main()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"q8rEahLvTbqF","executionInfo":{"status":"ok","timestamp":1641538334270,"user_tz":-330,"elapsed":12873,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"20358627-dede-48d7-8a5f-88236d55aaae"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(100000, 4) 943 1682\n","epoch:0, loss:0.2471, mse:1.7855\n","epoch:1, loss:0.095476, mse:1.3192\n","epoch:2, loss:0.074207, mse:1.1511\n","epoch:3, loss:0.066409, mse:1.0806\n","epoch:4, loss:0.062901, mse:1.046\n","epoch:5, loss:0.060974, mse:1.0257\n","epoch:6, loss:0.059704, mse:1.0118\n","epoch:7, loss:0.058753, mse:1.0015\n","epoch:8, loss:0.057982, mse:0.99323\n","epoch:9, loss:0.057325, mse:0.98636\n","best val epoch is 9, mse is 0.9863620211966587\n","test mse is 0.993012835364686\n"]}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"rsNyAAaUT5lA"}},{"cell_type":"code","source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Kk0V69zDT5lE","executionInfo":{"status":"ok","timestamp":1641538388085,"user_tz":-330,"elapsed":4265,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"5dfb8942-7c65-4d4f-c546-382b6a4f04af"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2022-01-07 06:53:08\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.144+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","pandas : 1.1.5\n","numpy : 1.19.5\n","IPython: 5.5.0\n","torch : 1.10.0+cu111\n","\n"]}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"Nyd-d0uGT5lG"}},{"cell_type":"markdown","source":["**END**"],"metadata":{"id":"rVlR650LT5lG"}}],"metadata":{"kernelspec":{"display_name":"Python 3","name":"python3"},"colab":{"name":"2022-01-10-fm-ml.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P176240%20%7C%20FM%20on%20ML-100k%20in%20PyTorch.ipynb","timestamp":1644598926871},{"file_id":"1Qbw_3372DrLKATKz_66dgUd5EdA_8lwh","timestamp":1641538393107},{"file_id":"1npp4hgFBQRflbqyRW4TayaUFIJJA_xHo","timestamp":1639737398037},{"file_id":"1vh6Mr1C7uh08K4zR4B2VIfqsK22pkraT","timestamp":1639730564985},{"file_id":"1F1wdk7jG5W0jbVM1nZBOPV0TYuSO7frV","timestamp":1639730030880},{"file_id":"https://github.com/RecoHut-Projects/recohut/blob/S394070/nbs/models/tensorflow/deepmf.ipynb","timestamp":1639729505410}],"collapsed_sections":[]}},"nbformat":4,"nbformat_minor":0} \ No newline at end of file diff --git a/_notebooks/2022-01-11-deepfm-criteo.ipynb b/_notebooks/2022-01-11-deepfm-criteo.ipynb new file mode 100644 index 0000000..f76e43a --- /dev/null +++ b/_notebooks/2022-01-11-deepfm-criteo.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-11-deepfm-criteo.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P237732%20%7C%20DeepFM%20on%20Criteo%20DAC%20sample%20dataset%20in%20PyTorch.ipynb","timestamp":1644607197734},{"file_id":"1PcrzoopQcJ6T5CwS38RIyYqoayb0ytc7","timestamp":1641537359438},{"file_id":"1FEZmnoLGIsTsGiK2gi1TsIHLAaWCXF_a","timestamp":1640329037065}],"collapsed_sections":[],"mount_file_id":"1FEZmnoLGIsTsGiK2gi1TsIHLAaWCXF_a","authorship_tag":"ABX9TyNHfJtptRpR1n+zn0BOMPle"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# DeepFM on Criteo DAC sample dataset in PyTorch"],"metadata":{"id":"2VZ8fr0qMt5t"}},{"cell_type":"code","source":["import pandas as pd\n","import torch\n","from sklearn.metrics import log_loss, roc_auc_score\n","from sklearn.model_selection import train_test_split\n","from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n","import sys\n","import os\n","\n","import torch.nn as nn\n","import numpy as np\n","import torch.utils.data as Data\n","from torch.utils.data import DataLoader\n","import torch.optim as optim\n","import torch.nn.functional as F\n","from sklearn.metrics import log_loss, roc_auc_score\n","from collections import OrderedDict, namedtuple, defaultdict\n","import random"],"metadata":{"id":"EToD4LnRLgyY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!wget -q --show-progress https://github.com/RecoHut-Datasets/criteo/raw/v1/dac_sample.txt"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"L60wRz3KLutF","executionInfo":{"status":"ok","timestamp":1641536229646,"user_tz":-330,"elapsed":1342,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"1a380f02-5cf1-43d2-8a40-974119bbecdc"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\rdac_sample.txt 0%[ ] 0 --.-KB/s \rdac_sample.txt 100%[===================>] 23.20M --.-KB/s in 0.09s \n"]}]},{"cell_type":"code","source":["class FM(nn.Module):\n"," def __init__(self, p, k):\n"," super(FM, self).__init__()\n"," self.p = p\n"," self.k = k\n"," self.linear = nn.Linear(self.p, 1, bias=True)\n"," self.v = nn.Parameter(torch.Tensor(self.p, self.k), requires_grad=True)\n"," self.v.data.uniform_(-0.01, 0.01)\n"," self.drop = nn.Dropout(0.3)\n","\n"," def forward(self, x):\n"," linear_part = self.linear(x)\n"," inter_part1 = torch.pow(torch.mm(x, self.v), 2)\n"," inter_part2 = torch.mm(torch.pow(x, 2), torch.pow(self.v, 2))\n"," pair_interactions = torch.sum(torch.sub(inter_part1, inter_part2), dim=1)\n"," self.drop(pair_interactions)\n"," output = linear_part.transpose(1, 0) + 0.5 * pair_interactions\n"," return output.view(-1, 1)"],"metadata":{"id":"D_slY7KGN44C"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class deepfm(nn.Module):\n"," def __init__(self, feat_sizes, sparse_feature_columns, dense_feature_columns,dnn_hidden_units=[400, 400,400], dnn_dropout=0.0, ebedding_size=4,\n"," l2_reg_linear=0.00001, l2_reg_embedding=0.00001, l2_reg_dnn=0, init_std=0.0001, seed=1024,\n"," device='cpu'):\n"," super(deepfm, self).__init__()\n"," self.feat_sizes = feat_sizes\n"," self.device = device\n"," self.dense_feature_columns = dense_feature_columns\n"," self.sparse_feature_columns = sparse_feature_columns\n"," self.embedding_size = ebedding_size\n"," self.l2_reg_linear = l2_reg_linear\n","\n"," self.bias = nn.Parameter(torch.zeros((1, )))\n"," self.init_std = init_std\n"," self.dnn_dropout = dnn_dropout\n","\n"," self.embedding_dic = nn.ModuleDict({feat:nn.Embedding(self.feat_sizes[feat], self.embedding_size, sparse=False)\n"," for feat in self.sparse_feature_columns})\n"," for tensor in self.embedding_dic.values():\n"," nn.init.normal_(tensor.weight, mean=0, std=self.init_std)\n"," self.embedding_dic.to(self.device)\n","\n"," self.feature_index = defaultdict(int)\n"," start = 0\n"," for feat in self.feat_sizes:\n"," if feat in self.feature_index:\n"," continue\n"," self.feature_index[feat] = start\n"," start += 1\n","\n"," # 输入维度 fm层与DNN层共享嵌入层， 输入维度应该是一样的\n"," self.input_size = self.embedding_size * len(self.sparse_feature_columns)+len(self.dense_feature_columns)\n"," # fm\n"," self.fm = FM(self.input_size, 10)\n","\n"," # DNN\n"," self.dropout = nn.Dropout(self.dnn_dropout)\n"," self.hidden_units = [self.input_size] + dnn_hidden_units\n"," self.Linears = nn.ModuleList([nn.Linear(self.hidden_units[i], self.hidden_units[i+1]) for i in range(len(self.hidden_units)-1)])\n"," self.relus = nn.ModuleList([nn.ReLU() for i in range(len(self.hidden_units)-1)])\n"," for name, tensor in self.Linears.named_parameters():\n"," if 'weight' in name:\n"," nn.init.normal_(tensor, mean=0, std=self.init_std)\n"," self.dnn_outlayer = nn.Linear(dnn_hidden_units[-1], 1, bias=False).to(self.device)\n","\n","\n"," def forward(self, x):\n"," # x shape 1024*39\n","\n"," sparse_embedding = [self.embedding_dic[feat](x[:, self.feature_index[feat]].long()) for feat in self.sparse_feature_columns]\n"," sparse_embedding = torch.cat(sparse_embedding, dim=-1)\n"," # print(sparse_embedding.shape) # batch * 208\n","\n"," dense_value = [x[:, self.feature_index[feat]] for feat in\n"," self.dense_feature_columns]\n","\n"," dense_value = torch.cat(dense_value, dim=0)\n"," dense_value = torch.reshape(dense_value, (len(self.dense_feature_columns), -1))\n"," dense_value = dense_value.T\n"," # print(dense_value.shape) # batch * 13\n","\n"," input_x = torch.cat((dense_value, sparse_embedding), dim=1)\n"," # print(input_x.shape) # batch * 221\n","\n"," fm_logit = self.fm(input_x)\n","\n"," for i in range(len(self.Linears)):\n"," fc = self.Linears[i](input_x)\n"," fc = self.relus[i](fc)\n"," fc = self.dropout(fc)\n"," input_x = fc\n"," dnn_logit = self.dnn_outlayer(input_x)\n","\n"," y_pre = torch.sigmoid(fm_logit+dnn_logit+self.bias)\n"," return y_pre"],"metadata":{"id":"ZiDRlS-GO9Y1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_auc(loader, model):\n"," pred, target = [], []\n"," model.eval()\n"," with torch.no_grad():\n"," for x, y in loader:\n"," x = x.to(device).float()\n"," y = y.to(device).float()\n"," y_hat = model(x)\n"," pred += list(y_hat.numpy())\n"," target += list(y.numpy())\n"," auc = roc_auc_score(target, pred)\n"," return auc"],"metadata":{"id":"g-zB2cR0PJQ_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["batch_size = 1024\n","lr = 0.00005\n","wd = 0.00001\n","epoches = 10\n","\n","seed = 1024\n","torch.manual_seed(seed)\n","torch.cuda.manual_seed(seed)\n","torch.cuda.manual_seed_all(seed)\n","np.random.seed(seed)\n","random.seed(seed)\n","\n","sparse_features = ['C' + str(i) for i in range(1, 27)]\n","dense_features = ['I' + str(i) for i in range(1, 14)]\n","col_names = ['label'] + dense_features + sparse_features\n","df = pd.read_csv('dac_sample.txt', names=col_names, sep='\\t')\n","feature_names = dense_features + sparse_features\n","\n","df[sparse_features] = df[sparse_features].fillna('-1', )\n","df[dense_features] = df[dense_features].fillna(0, )\n","target = ['label']\n","\n","for feat in sparse_features:\n"," lbe = LabelEncoder()\n"," df[feat] = lbe.fit_transform(df[feat])\n","\n","mms = MinMaxScaler(feature_range=(0, 1))\n","df[dense_features] = mms.fit_transform(df[dense_features])\n","\n","feat_size1 = {feat: 1 for feat in dense_features}\n","feat_size2 = {feat: len(df[feat].unique()) for feat in sparse_features}\n","feat_sizes = {}\n","feat_sizes.update(feat_size1)\n","feat_sizes.update(feat_size2)\n","\n","# print(df.head(5))\n","# print(feat_sizes)\n","\n","train, test =train_test_split(df, test_size=0.2, random_state=2021)\n","train_model_input = {name: train[name] for name in feature_names}\n","test_model_input = {name: test[name] for name in feature_names}\n","\n","device = 'cpu'\n","\n","model = deepfm(feat_sizes, sparse_feature_columns=sparse_features, dense_feature_columns=dense_features,\n"," dnn_hidden_units=[1000, 500, 250], dnn_dropout=0.9, ebedding_size=16,\n"," l2_reg_linear=1e-3, device=device)\n","\n","train_label = pd.DataFrame(train['label'])\n","train_data = train.drop(columns=['label'])\n","#print(train.head(5))\n","train_tensor_data = torch.utils.data.TensorDataset(torch.from_numpy(np.array(train_data)), torch.from_numpy(np.array(train_label)))\n","train_loader = DataLoader(dataset=train_tensor_data, shuffle=True, batch_size=batch_size)\n","\n","test_label = pd.DataFrame(test['label'])\n","test_data = test.drop(columns=['label'])\n","test_tensor_data = torch.utils.data.TensorDataset(torch.from_numpy(np.array(test_data)),\n"," torch.from_numpy(np.array(test_label)))\n","test_loader = DataLoader(dataset=test_tensor_data, shuffle=False, batch_size=batch_size)\n","\n","loss_func = nn.BCELoss(reduction='mean')\n","optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n","\n","for epoch in range(epoches):\n"," total_loss_epoch = 0.0\n"," total_tmp = 0\n","\n"," model.train()\n"," for index, (x, y) in enumerate(train_loader):\n"," x = x.to(device).float()\n"," y = y.to(device).float()\n","\n"," y_hat = model(x)\n","\n"," optimizer.zero_grad()\n"," loss = loss_func(y_hat, y)\n"," loss.backward()\n"," optimizer.step()\n"," total_loss_epoch += loss.item()\n"," total_tmp += 1\n","\n"," auc = get_auc(test_loader, model)\n"," print('epoch/epoches: {}/{}, train loss: {:.3f}, test auc: {:.3f}'.format(epoch, epoches, total_loss_epoch / total_tmp, auc))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZH69aNy6LmvA","executionInfo":{"status":"ok","timestamp":1641537334473,"user_tz":-330,"elapsed":181659,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9a504fcb-fe7b-49c4-bf15-2ccece85893d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["epoch/epoches: 0/10, train loss: 0.667, test auc: 0.569\n","epoch/epoches: 1/10, train loss: 0.564, test auc: 0.681\n","epoch/epoches: 2/10, train loss: 0.531, test auc: 0.710\n","epoch/epoches: 3/10, train loss: 0.507, test auc: 0.720\n","epoch/epoches: 4/10, train loss: 0.482, test auc: 0.727\n","epoch/epoches: 5/10, train loss: 0.455, test auc: 0.735\n","epoch/epoches: 6/10, train loss: 0.425, test auc: 0.740\n","epoch/epoches: 7/10, train loss: 0.393, test auc: 0.742\n","epoch/epoches: 8/10, train loss: 0.363, test auc: 0.739\n","epoch/epoches: 9/10, train loss: 0.337, test auc: 0.733\n"]}]},{"cell_type":"code","source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tmhWlpwiL0bb","executionInfo":{"status":"ok","timestamp":1641537342905,"user_tz":-330,"elapsed":3754,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"57e12b81-7f55-4ce0-8aa4-b43e46a80887"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2022-01-07 06:35:43\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.144+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","pandas : 1.1.5\n","sys : 3.7.12 (default, Sep 10 2021, 00:21:48) \n","[GCC 7.5.0]\n","torch : 1.10.0+cu111\n","numpy : 1.19.5\n","IPython: 5.5.0\n","\n"]}]}]} \ No newline at end of file diff --git a/_notebooks/2022-01-11-gmf-yelp.ipynb b/_notebooks/2022-01-11-gmf-yelp.ipynb new file mode 100644 index 0000000..f67fbc2 --- /dev/null +++ b/_notebooks/2022-01-11-gmf-yelp.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-11-gmf-yelp.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P254192%20%7C%20Training%20GMF%20with%20Truncated%20and%20Reweighted%20Denoising%20Losses%20on%20Yelp%20Dataset.ipynb","timestamp":1644607236067}],"collapsed_sections":[],"authorship_tag":"ABX9TyOW73ZiRmfpoXImxYpevmen"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","source":["# Training GMF with Truncated and Reweighted Denoising Losses on Yelp Dataset"],"metadata":{"id":"T5kY-7DqauRg"}},{"cell_type":"markdown","source":["## Executive summary\n","\n","| | |\n","| --- | --- |\n","| Problem | It is of importance to account for the inevitable noises in implicit feedback. However, little work on recommendation has taken the noisy nature of implicit feedback into consideration. |\n","| Prblm Stmt. | We formulate a denoising recommender training task as $Θ^∗ = \\text{argmin}_Θ \\mathcal{L}(\\textit{denoise}(\\bar{D}))$ aiming to learn a reliable recommender model with parameters $Θ^∗$ by denoising implicit feedback $\\bar{D}$. Formally, by assuming the existence of inconsistency between $y_{ui}^∗$ and $\\bar{y}_{ui}$, we define noisy interactions (a.k.a. false-positive interactions) as $\\{(u, i)|y_{ui}^∗ = 0 ∧ \\bar{y}_{ui} = 1\\}$. |\n","| Solution | Adaptive Denoising Training (ADT), which adaptively prunes the noisy interactions by two paradigms - Truncated Loss and Reweighted Loss. Furthermore, we consider extra feedback (e.g., rating) as auxiliary signal and employ three strategies to incorporate extra feedback into ADT: fine-tuning, warm-up training, and colliding inference. |\n","| Dataset | Adressa, Amazon-books, Yelp2018. |\n","| Preprocessing | We split the dataset into training, validation, and testing sets, and explored two experimental settings: 1) Extra feedback is unavailable during training. To evaluate the performance of denoising implicit feedback, we kept all interactions, including the false-positive ones, in training and validation, and tested the models only on true-positive interactions. 2) Sparse extra feedback is available during training. We assume that partial true-positive interactions have already been known, which will be used to verify the performance of the proposed three strategies: fine-tuning, warm-up training, and colliding inference. |\n","| Metrics | Recall, NDCG |\n","| Hyperparams | For GMF and NeuMF, the factor numbers of users and items are both 32. As to CDAE, the hidden size of MLP is set as 200. In addition, Adam is applied to optimize all the parameters with the learning rate initialized as 0.001 and he batch size set as 1,024. As to the ADT strategies, they have three hyper-parameters in total: α and max in the T-CE loss, and β in the R-CE loss. In detail, max is searched in {0.05, 0.1, 0.2} and β is tuned in {0.05, 0.1, ..., 0.25, 0.5, 1.0}. As for α, we controlled its range by adjusting the iteration number N to the maximum drop rate max, and N is adjusted in {1k, 5k, 10k, 20k, 30k}. In colliding inference, the number of neighbors Nu is tuned in {1, 3, 5, 10, 20, 50, 100}, wj is set as 1/|Nu|, and λ is searched in {0, 0.1, 0.2, ..., 1}. We used the validation set to tune the hyper-parameters and reported the performance on the testing set. |\n","| Models | GMF, NMF, CDAE, {GMF, NMF, CDAE}+T_CE, {GMF, NMF, CDAE}+R_CE |\n","| Cluster | Python 3.6+, PyTorch |\n","| Tags | `LossReweighting`, `TruncatedLoss`, `MatrixFactorization`, `Denoising` |\n","| Credits | Wenjie Wang |"],"metadata":{"id":"wVfZ0xpJb4nI"}},{"cell_type":"markdown","source":["## Process flow\n","\n","![](https://github.com/RecoHut-Stanzas/S063707/raw/main/images/process_flow.svg)"],"metadata":{"id":"qHcxWaZPqFyE"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"009XG-W-b4is"}},{"cell_type":"code","source":["import numpy as np \n","import pandas as pd \n","import scipy.sparse as sp\n","from copy import deepcopy\n","import random\n","import math\n","import os\n","import time\n","import argparse\n","\n","import torch\n","import torch.nn as nn\n","import torch.utils.data as data\n","import torch.nn.functional as F \n","from torch.autograd import Variable\n","import torch.optim as optim\n","import torch.backends.cudnn as cudnn\n","from torch.utils.tensorboard import SummaryWriter"],"metadata":{"id":"NhKMYe3Mb4gO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["parser = argparse.ArgumentParser()\n","parser.add_argument('--dataset', \n","\ttype = str,\n","\thelp = 'dataset used for training, options: amazon_book, yelp, adressa',\n","\tdefault = 'yelp')\n","parser.add_argument('--model', \n","\ttype = str,\n","\thelp = 'model used for training. options: GMF, NeuMF-end',\n","\tdefault = 'GMF')\n","parser.add_argument('--alpha', \n","\ttype = float, \n","\tdefault = 0.2, \n","\thelp='hyperparameter in loss function')\n","parser.add_argument('--drop_rate', \n","\ttype = float,\n","\thelp = 'drop rate',\n","\tdefault = 0.2)\n","parser.add_argument('--num_gradual', \n","\ttype = int, \n","\tdefault = 30000,\n","\thelp='how many epochs to linearly increase drop_rate')\n","parser.add_argument('--exponent', \n","\ttype = float, \n","\tdefault = 1, \n","\thelp='exponent of the drop rate {0.5, 1, 2}')\n","parser.add_argument(\"--lr\", \n","\ttype=float, \n","\tdefault=0.001, \n","\thelp=\"learning rate\")\n","parser.add_argument(\"--dropout\", \n","\ttype=float,\n","\tdefault=0.0, \n","\thelp=\"dropout rate\")\n","parser.add_argument(\"--batch_size\", \n","\ttype=int, \n","\tdefault=1024, \n","\thelp=\"batch size for training\")\n","parser.add_argument(\"--epochs\", \n","\ttype=int,\n","\tdefault=10,\n","\thelp=\"training epoches\")\n","parser.add_argument(\"--eval_freq\", \n","\ttype=int,\n","\tdefault=2000,\n","\thelp=\"the freq of eval\")\n","parser.add_argument(\"--top_k\", \n","\ttype=list, \n","\tdefault=[50, 100],\n","\thelp=\"compute metrics@top_k\")\n","parser.add_argument(\"--factor_num\", \n","\ttype=int,\n","\tdefault=32, \n","\thelp=\"predictive factors numbers in the model\")\n","parser.add_argument(\"--num_layers\", \n","\ttype=int,\n","\tdefault=3, \n","\thelp=\"number of layers in MLP model\")\n","parser.add_argument(\"--num_ng\", \n","\ttype=int,\n","\tdefault=1, \n","\thelp=\"sample negative items for training\")\n","parser.add_argument(\"--out\", \n","\tdefault=True,\n","\thelp=\"save model or not\")\n","parser.add_argument(\"--gpu\", \n","\ttype=str,\n","\tdefault=\"0\",\n","\thelp=\"gpu card ID\")\n","args = parser.parse_args([])\n","\n","os.environ[\"CUDA_VISIBLE_DEVICES\"] = args.gpu\n","cudnn.benchmark = True\n","\n","torch.manual_seed(2019) # cpu\n","torch.cuda.manual_seed(2019) #gpu\n","np.random.seed(2019) #numpy\n","random.seed(2019) #random and transforms\n","torch.backends.cudnn.deterministic=True # cudnn\n","\n","def worker_init_fn(worker_id):\n"," np.random.seed(2019 + worker_id)\n","\n","data_path = '{}/'.format(args.dataset)\n","model_path = './models/{}/'.format(args.dataset)\n","os.makedirs('./models', exist_ok=True)\n","print(\"arguments: %s \" %(args))\n","print(\"config model\", args.model)\n","print(\"config data path\", data_path)\n","print(\"config model path\", model_path)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"K4L82dHCeb1H","executionInfo":{"status":"ok","timestamp":1639311517343,"user_tz":-330,"elapsed":722,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"32c39b27-4240-478a-da19-11af8f6d2dc4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["arguments: Namespace(alpha=0.2, batch_size=1024, dataset='yelp', drop_rate=0.2, dropout=0.0, epochs=10, eval_freq=2000, exponent=1, factor_num=32, gpu='0', lr=0.001, model='GMF', num_gradual=30000, num_layers=3, num_ng=1, out=True, top_k=[50, 100]) \n","config model GMF\n","config data path yelp/\n","config model path ./models/yelp/\n"]}]},{"cell_type":"markdown","source":["## Data"],"metadata":{"id":"GXK73SDob4dR"}},{"cell_type":"code","source":["!git clone --branch v4 https://github.com/RecoHut-Datasets/yelp.git"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"D6y9RjVob4ZS","executionInfo":{"status":"ok","timestamp":1639310015219,"user_tz":-330,"elapsed":24510,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"a74f0442-81ed-4ad8-80fd-2ec961f58e1c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Cloning into 'yelp'...\n","remote: Enumerating objects: 57, done.\u001b[K\n","remote: Counting objects: 100% (57/57), done.\u001b[K\n","remote: Compressing objects: 100% (50/50), done.\u001b[K\n","remote: Total 57 (delta 5), reused 52 (delta 3), pack-reused 0\u001b[K\n","Unpacking objects: 100% (57/57), done.\n"]}]},{"cell_type":"code","source":["def load_all(dataset, data_path):\n","\n","\ttrain_rating = data_path + '{}.train.rating'.format(dataset)\n","\tvalid_rating = data_path + '{}.valid.rating'.format(dataset)\n","\ttest_negative = data_path + '{}.test.negative'.format(dataset)\n","\n","\t################# load training data #################\t\n","\ttrain_data = pd.read_csv(\n","\t\ttrain_rating, \n","\t\tsep='\\t', header=None, names=['user', 'item', 'noisy'], \n","\t\tusecols=[0, 1, 2], dtype={0: np.int32, 1: np.int32, 2: np.int32})\n","\n","\tif dataset == \"adressa\":\n","\t\tuser_num = 212231\n","\t\titem_num = 6596\n","\telse:\n","\t\tuser_num = train_data['user'].max() + 1\n","\t\titem_num = train_data['item'].max() + 1\n","\tprint(\"user, item num\")\n","\tprint(user_num, item_num)\n","\ttrain_data = train_data.values.tolist()\n","\n","\t# load ratings as a dok matrix\n","\ttrain_mat = sp.dok_matrix((user_num, item_num), dtype=np.float32)\n","\ttrain_data_list = []\n","\ttrain_data_noisy = []\n","\tfor x in train_data:\n","\t\ttrain_mat[x[0], x[1]] = 1.0\n","\t\ttrain_data_list.append([x[0], x[1]])\n","\t\ttrain_data_noisy.append(x[2])\n","\n","\t################# load validation data #################\n","\tvalid_data = pd.read_csv(\n","\t\tvalid_rating, \n","\t\tsep='\\t', header=None, names=['user', 'item', 'noisy'], \n","\t\tusecols=[0, 1, 2], dtype={0: np.int32, 1: np.int32, 2: np.int32})\n","\tvalid_data = valid_data.values.tolist()\n","\tvalid_data_list = []\n","\tfor x in valid_data:\n","\t\tvalid_data_list.append([x[0], x[1]])\n","\t\n","\tuser_pos = {}\n","\tfor x in train_data_list:\n","\t\tif x[0] in user_pos:\n","\t\t\tuser_pos[x[0]].append(x[1])\n","\t\telse:\n","\t\t\tuser_pos[x[0]] = [x[1]]\n","\tfor x in valid_data_list:\n","\t\tif x[0] in user_pos:\n","\t\t\tuser_pos[x[0]].append(x[1])\n","\t\telse:\n","\t\t\tuser_pos[x[0]] = [x[1]]\n","\n","\n","\t################# load testing data #################\n","\ttest_mat = sp.dok_matrix((user_num, item_num), dtype=np.float32)\n","\n","\ttest_data_pos = {}\n","\twith open(test_negative, 'r') as fd:\n","\t\tline = fd.readline()\n","\t\twhile line != None and line != '':\n","\t\t\tarr = line.split('\\t')\n","\t\t\tif dataset == \"adressa\":\n","\t\t\t\tu = eval(arr[0])[0]\n","\t\t\t\ti = eval(arr[0])[1]\n","\t\t\telse:\n","\t\t\t\tu = int(arr[0])\n","\t\t\t\ti = int(arr[1])\n","\t\t\tif u in test_data_pos:\n","\t\t\t\ttest_data_pos[u].append(i)\n","\t\t\telse:\n","\t\t\t\ttest_data_pos[u] = [i]\n","\t\t\ttest_mat[u, i] = 1.0\n","\t\t\tline = fd.readline()\n","\n","\n","\treturn train_data_list, valid_data_list, test_data_pos, user_pos, user_num, item_num, train_mat, train_data_noisy"],"metadata":{"id":"mpIf7UVqb7ln"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class NCFData(data.Dataset):\n","\tdef __init__(self, features,\n","\t\t\t\tnum_item, train_mat=None, num_ng=0, is_training=0, noisy_or_not=None):\n","\t\tsuper(NCFData, self).__init__()\n","\t\t\"\"\" Note that the labels are only useful when training, we thus \n","\t\t\tadd them in the ng_sample() function.\n","\t\t\"\"\"\n","\t\tself.features_ps = features\n","\t\tif is_training == 0:\n","\t\t\tself.noisy_or_not = noisy_or_not\n","\t\telse:\n","\t\t\tself.noisy_or_not = [0 for _ in range(len(features))]\n","\t\tself.num_item = num_item\n","\t\tself.train_mat = train_mat\n","\t\tself.num_ng = num_ng\n","\t\tself.is_training = is_training\n","\t\tself.labels = [0 for _ in range(len(features))]\n","\n","\tdef ng_sample(self):\n","\t\tassert self.is_training != 2, 'no need to sampling when testing'\n","\n","\t\tself.features_ng = []\n","\t\tfor x in self.features_ps:\n","\t\t\tu = x[0]\n","\t\t\tfor t in range(self.num_ng):\n","\t\t\t\tj = np.random.randint(self.num_item)\n","\t\t\t\twhile (u, j) in self.train_mat:\n","\t\t\t\t\tj = np.random.randint(self.num_item)\n","\t\t\t\tself.features_ng.append([u, j])\n","\n","\t\tlabels_ps = [1 for _ in range(len(self.features_ps))]\n","\t\tlabels_ng = [0 for _ in range(len(self.features_ng))]\n","\t\tself.noisy_or_not_fill = self.noisy_or_not + [1 for _ in range(len(self.features_ng))]\n","\t\tself.features_fill = self.features_ps + self.features_ng\n","\t\tassert len(self.noisy_or_not_fill) == len(self.features_fill)\n","\t\tself.labels_fill = labels_ps + labels_ng\n","\n","\tdef __len__(self):\n","\t\treturn (self.num_ng + 1) * len(self.labels)\n","\n","\tdef __getitem__(self, idx):\n","\t\tfeatures = self.features_fill if self.is_training != 2 \\\n","\t\t\t\t\telse self.features_ps\n","\t\tlabels = self.labels_fill if self.is_training != 2 \\\n","\t\t\t\t\telse self.labels\n","\t\tnoisy_or_not = self.noisy_or_not_fill if self.is_training != 2 \\\n","\t\t\t\t\telse self.noisy_or_not\n","\n","\t\tuser = features[idx][0]\n","\t\titem = features[idx][1]\n","\t\tlabel = labels[idx]\n","\t\tnoisy_label = noisy_or_not[idx]\n","\n","\t\treturn user, item, label, noisy_label"],"metadata":{"id":"Jv8T8msTcSU1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Model"],"metadata":{"id":"YutDztVXb7i2"}},{"cell_type":"code","source":["class NCF(nn.Module):\n","\tdef __init__(self, user_num, item_num, factor_num, num_layers,\n","\t\t\t\t\tdropout, model, GMF_model=None, MLP_model=None):\n","\t\tsuper(NCF, self).__init__()\n","\t\t\"\"\"\n","\t\tuser_num: number of users;\n","\t\titem_num: number of items;\n","\t\tfactor_num: number of predictive factors;\n","\t\tnum_layers: the number of layers in MLP model;\n","\t\tdropout: dropout rate between fully connected layers;\n","\t\tmodel: 'MLP', 'GMF', 'NeuMF-end', and 'NeuMF-pre';\n","\t\tGMF_model: pre-trained GMF weights;\n","\t\tMLP_model: pre-trained MLP weights.\n","\t\t\"\"\"\t\t\n","\t\tself.dropout = dropout\n","\t\tself.model = model\n","\t\tself.GMF_model = GMF_model\n","\t\tself.MLP_model = MLP_model\n","\n","\t\tself.embed_user_GMF = nn.Embedding(user_num, factor_num)\n","\t\tself.embed_item_GMF = nn.Embedding(item_num, factor_num)\n","\t\tself.embed_user_MLP = nn.Embedding(\n","\t\t\t\tuser_num, factor_num * (2 ** (num_layers - 1)))\n","\t\tself.embed_item_MLP = nn.Embedding(\n","\t\t\t\titem_num, factor_num * (2 ** (num_layers - 1)))\n","\n","\t\tMLP_modules = []\n","\t\tfor i in range(num_layers):\n","\t\t\tinput_size = factor_num * (2 ** (num_layers - i))\n","\t\t\tMLP_modules.append(nn.Dropout(p=self.dropout))\n","\t\t\tMLP_modules.append(nn.Linear(input_size, input_size//2))\n","\t\t\tMLP_modules.append(nn.ReLU())\n","\t\tself.MLP_layers = nn.Sequential(*MLP_modules)\n","\n","\t\tif self.model in ['MLP', 'GMF']:\n","\t\t\tpredict_size = factor_num \n","\t\telse:\n","\t\t\tpredict_size = factor_num * 2\n","\t\tself.predict_layer = nn.Linear(predict_size, 1)\n","\n","\t\tself._init_weight_()\n","\n","\tdef _init_weight_(self):\n","\t\t\"\"\" We leave the weights initialization here. \"\"\"\n","\t\tif not self.model == 'NeuMF-pre':\n","\t\t\tnn.init.normal_(self.embed_user_GMF.weight, std=0.01)\n","\t\t\tnn.init.normal_(self.embed_user_MLP.weight, std=0.01)\n","\t\t\tnn.init.normal_(self.embed_item_GMF.weight, std=0.01)\n","\t\t\tnn.init.normal_(self.embed_item_MLP.weight, std=0.01)\n","\n","\t\t\tfor m in self.MLP_layers:\n","\t\t\t\tif isinstance(m, nn.Linear):\n","\t\t\t\t\tnn.init.xavier_uniform_(m.weight)\n","\t\t\tnn.init.kaiming_uniform_(self.predict_layer.weight, \n","\t\t\t\t\t\t\t\t\ta=1, nonlinearity='sigmoid')\n","\n","\t\t\tfor m in self.modules():\n","\t\t\t\tif isinstance(m, nn.Linear) and m.bias is not None:\n","\t\t\t\t\tm.bias.data.zero_()\n","\t\telse:\n","\t\t\t# embedding layers\n","\t\t\tself.embed_user_GMF.weight.data.copy_(\n","\t\t\t\t\t\t\tself.GMF_model.embed_user_GMF.weight)\n","\t\t\tself.embed_item_GMF.weight.data.copy_(\n","\t\t\t\t\t\t\tself.GMF_model.embed_item_GMF.weight)\n","\t\t\tself.embed_user_MLP.weight.data.copy_(\n","\t\t\t\t\t\t\tself.MLP_model.embed_user_MLP.weight)\n","\t\t\tself.embed_item_MLP.weight.data.copy_(\n","\t\t\t\t\t\t\tself.MLP_model.embed_item_MLP.weight)\n","\n","\t\t\t# mlp layers\n","\t\t\tfor (m1, m2) in zip(\n","\t\t\t\tself.MLP_layers, self.MLP_model.MLP_layers):\n","\t\t\t\tif isinstance(m1, nn.Linear) and isinstance(m2, nn.Linear):\n","\t\t\t\t\tm1.weight.data.copy_(m2.weight)\n","\t\t\t\t\tm1.bias.data.copy_(m2.bias)\n","\n","\t\t\t# predict layers\n","\t\t\tpredict_weight = torch.cat([\n","\t\t\t\tself.GMF_model.predict_layer.weight, \n","\t\t\t\tself.MLP_model.predict_layer.weight], dim=1)\n","\t\t\tprecit_bias = self.GMF_model.predict_layer.bias + \\\n","\t\t\t\t\t\tself.MLP_model.predict_layer.bias\n","\n","\t\t\tself.predict_layer.weight.data.copy_(0.5 * predict_weight)\n","\t\t\tself.predict_layer.bias.data.copy_(0.5 * precit_bias)\n","\n","\tdef forward(self, user, item):\n","\t\tif not self.model == 'MLP':\n","\t\t\tembed_user_GMF = self.embed_user_GMF(user)\n","\t\t\tembed_item_GMF = self.embed_item_GMF(item)\n","\t\t\toutput_GMF = embed_user_GMF * embed_item_GMF\n","\t\tif not self.model == 'GMF':\n","\t\t\tembed_user_MLP = self.embed_user_MLP(user)\n","\t\t\tembed_item_MLP = self.embed_item_MLP(item)\n","\t\t\tinteraction = torch.cat((embed_user_MLP, embed_item_MLP), -1)\n","\t\t\toutput_MLP = self.MLP_layers(interaction)\n","\n","\t\tif self.model == 'GMF':\n","\t\t\tconcat = output_GMF\n","\t\telif self.model == 'MLP':\n","\t\t\tconcat = output_MLP\n","\t\telse:\n","\t\t\tconcat = torch.cat((output_GMF, output_MLP), -1)\n","\n","\t\tprediction = self.predict_layer(concat)\n","\t\treturn prediction.view(-1)"],"metadata":{"id":"eD0tBH5ub7gH"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Evaluate"],"metadata":{"id":"ADEh-IsUb7Xm"}},{"cell_type":"code","source":["def test_all_users(model, batch_size, item_num, test_data_pos, user_pos, top_k):\n"," \n"," predictedIndices = []\n"," GroundTruth = []\n"," for u in test_data_pos:\n"," batch_num = item_num // batch_size\n"," batch_user = torch.Tensor([u]*batch_size).long().cuda()\n"," st, ed = 0, batch_size\n"," for i in range(batch_num):\n"," batch_item = torch.Tensor([i for i in range(st, ed)]).long().cuda()\n"," pred = model(batch_user, batch_item)\n"," if i == 0:\n"," predictions = pred\n"," else:\n"," predictions = torch.cat([predictions, pred], 0)\n"," st, ed = st+batch_size, ed+batch_size\n"," ed = ed - batch_size\n"," batch_item = torch.Tensor([i for i in range(ed, item_num)]).long().cuda()\n"," batch_user = torch.Tensor([u]*(item_num-ed)).long().cuda()\n"," pred = model(batch_user, batch_item)\n"," predictions = torch.cat([predictions, pred], 0)\n"," test_data_mask = [0] * item_num\n"," if u in user_pos:\n"," for i in user_pos[u]:\n"," test_data_mask[i] = -9999\n"," predictions = predictions + torch.Tensor(test_data_mask).float().cuda()\n"," _, indices = torch.topk(predictions, top_k[-1])\n"," indices = indices.cpu().numpy().tolist()\n"," predictedIndices.append(indices)\n"," GroundTruth.append(test_data_pos[u])\n"," precision, recall, NDCG, MRR = compute_acc(GroundTruth, predictedIndices, top_k)\n"," return precision, recall, NDCG, MRR"],"metadata":{"id":"I5oF1IG7dSFb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def compute_acc(GroundTruth, predictedIndices, topN):\n"," precision = [] \n"," recall = [] \n"," NDCG = [] \n"," MRR = []\n"," \n"," for index in range(len(topN)):\n"," sumForPrecision = 0\n"," sumForRecall = 0\n"," sumForNdcg = 0\n"," sumForMRR = 0\n"," for i in range(len(predictedIndices)): # for a user,\n"," if len(GroundTruth[i]) != 0:\n"," mrrFlag = True\n"," userHit = 0\n"," userMRR = 0\n"," dcg = 0\n"," idcg = 0\n"," idcgCount = len(GroundTruth[i])\n"," ndcg = 0\n"," hit = []\n"," for j in range(topN[index]):\n"," if predictedIndices[i][j] in GroundTruth[i]:\n"," # if Hit!\n"," dcg += 1.0/math.log2(j + 2)\n"," if mrrFlag:\n"," userMRR = (1.0/(j+1.0))\n"," mrrFlag = False\n"," userHit += 1\n"," \n"," if idcgCount > 0:\n"," idcg += 1.0/math.log2(j + 2)\n"," idcgCount = idcgCount-1\n"," \n"," if(idcg != 0):\n"," ndcg += (dcg/idcg)\n"," \n"," sumForPrecision += userHit / topN[index]\n"," sumForRecall += userHit / len(GroundTruth[i]) \n"," sumForNdcg += ndcg\n"," sumForMRR += userMRR\n"," \n"," precision.append(sumForPrecision / len(predictedIndices))\n"," recall.append(sumForRecall / len(predictedIndices))\n"," NDCG.append(sumForNdcg / len(predictedIndices))\n"," MRR.append(sumForMRR / len(predictedIndices))\n"," \n"," return precision, recall, NDCG, MRR"],"metadata":{"id":"gBInZ4_jdS8X"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## T_CE"],"metadata":{"id":"K1tu6vgIfkyj"}},{"cell_type":"markdown","source":["### Loss function"],"metadata":{"id":"lfi9YaIBb7dY"}},{"cell_type":"code","source":["def loss_function(y, t, drop_rate):\n"," loss = F.binary_cross_entropy_with_logits(y, t, reduce = False)\n","\n"," loss_mul = loss * t\n"," ind_sorted = np.argsort(loss_mul.cpu().data).cuda()\n"," loss_sorted = loss[ind_sorted]\n","\n"," remember_rate = 1 - drop_rate\n"," num_remember = int(remember_rate * len(loss_sorted))\n","\n"," ind_update = ind_sorted[:num_remember]\n","\n"," loss_update = F.binary_cross_entropy_with_logits(y[ind_update], t[ind_update])\n","\n"," return loss_update"],"metadata":{"id":"SaBvZ5eVb7aw"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Training & Evaluation"],"metadata":{"id":"et6OGWMBeJiE"}},{"cell_type":"code","source":["############################## PREPARE DATASET ##########################\n","\n","train_data, valid_data, test_data_pos, user_pos, user_num ,item_num, train_mat, train_data_noisy = load_all(args.dataset, data_path)\n","\n","# construct the train and test datasets\n","train_dataset = NCFData(\n","\t\ttrain_data, item_num, train_mat, args.num_ng, 0, train_data_noisy)\n","valid_dataset = NCFData(\n","\t\tvalid_data, item_num, train_mat, args.num_ng, 1)\n","\n","train_loader = data.DataLoader(train_dataset,\n","\t\tbatch_size=args.batch_size, shuffle=True, num_workers=2, pin_memory=True, worker_init_fn=worker_init_fn)\n","valid_loader = data.DataLoader(valid_dataset,\n","\t\tbatch_size=args.batch_size, shuffle=True, num_workers=2, pin_memory=True, worker_init_fn=worker_init_fn)\n","\n","print(\"data loaded! user_num:{}, item_num:{} train_data_len:{} test_user_num:{}\".format(user_num, item_num, len(train_data), len(test_data_pos)))\n","\n","########################### CREATE MODEL #################################\n","if args.model == 'NeuMF-pre': # pre-training. Not used in our work.\n","\tGMF_model_path = model_path + 'GMF.pth'\n","\tMLP_model_path = model_path + 'MLP.pth'\n","\tNeuMF_model_path = model_path + 'NeuMF.pth'\n","\tassert os.path.exists(GMF_model_path), 'lack of GMF model'\n","\tassert os.path.exists(MLP_model_path), 'lack of MLP model'\n","\tGMF_model = torch.load(GMF_model_path)\n","\tMLP_model = torch.load(MLP_model_path)\n","else:\n","\tGMF_model = None\n","\tMLP_model = None\n","\n","model = NCF(user_num, item_num, args.factor_num, args.num_layers, \n","\t\t\t\t\t\targs.dropout, args.model, GMF_model, MLP_model)\n","\n","model.cuda()\n","BCE_loss = nn.BCEWithLogitsLoss()\n","\n","if args.model == 'NeuMF-pre':\n","\toptimizer = optim.SGD(model.parameters(), lr=args.lr)\n","else:\n","\toptimizer = optim.Adam(model.parameters(), lr=args.lr)\n","\n","# writer = SummaryWriter() # for visualization\n","\n","# define drop rate schedule\n","def drop_rate_schedule(iteration):\n","\n","\tdrop_rate = np.linspace(0, args.drop_rate**args.exponent, args.num_gradual)\n","\tif iteration < args.num_gradual:\n","\t\treturn drop_rate[iteration]\n","\telse:\n","\t\treturn args.drop_rate\n","\n","\n","########################### Eval #####################################\n","def eval(model, valid_loader, best_loss, count):\n","\t\n","\tmodel.eval()\n","\tepoch_loss = 0\n","\tvalid_loader.dataset.ng_sample() # negative sampling\n","\tfor user, item, label, noisy_or_not in valid_loader:\n","\t\tuser = user.cuda()\n","\t\titem = item.cuda()\n","\t\tlabel = label.float().cuda()\n","\n","\t\tprediction = model(user, item)\n","\t\tloss = loss_function(prediction, label, drop_rate_schedule(count))\n","\t\tepoch_loss += loss.detach()\n","\tprint(\"################### EVAL ######################\")\n","\tprint(\"Eval loss:{}\".format(epoch_loss))\n","\tif epoch_loss < best_loss:\n","\t\tbest_loss = epoch_loss\n","\t\tif args.out:\n","\t\t\tif not os.path.exists(model_path):\n","\t\t\t\tos.mkdir(model_path)\n","\t\t\ttorch.save(model, '{}{}_{}-{}.pth'.format(model_path, args.model, args.drop_rate, args.num_gradual))\n","\treturn best_loss\n","\n","########################### Test #####################################\n","def test(model, test_data_pos, user_pos):\n","\ttop_k = args.top_k\n","\tmodel.eval()\n","\t_, recall, NDCG, _ = test_all_users(model, 4096, item_num, test_data_pos, user_pos, top_k)\n","\n","\tprint(\"################### TEST ######################\")\n","\tprint(\"Recall {:.4f}-{:.4f}\".format(recall[0], recall[1]))\n","\tprint(\"NDCG {:.4f}-{:.4f}\".format(NDCG[0], NDCG[1]))\n","\n","########################### TRAINING #####################################\n","count, best_hr = 0, 0\n","best_loss = 1e9\n","\n","for epoch in range(args.epochs):\n","\tmodel.train() # Enable dropout (if have).\n","\n","\tstart_time = time.time()\n","\ttrain_loader.dataset.ng_sample()\n","\n","\tfor user, item, label, noisy_or_not in train_loader:\n","\t\tuser = user.cuda()\n","\t\titem = item.cuda()\n","\t\tlabel = label.float().cuda()\n","\n","\t\tmodel.zero_grad()\n","\t\tprediction = model(user, item)\n","\t\tloss = loss_function(prediction, label, drop_rate_schedule(count))\n","\t\tloss.backward()\n","\t\toptimizer.step()\n","\n","\t\tif count % args.eval_freq == 0 and count != 0:\n","\t\t\tprint(\"epoch: {}, iter: {}, loss:{}\".format(epoch, count, loss))\n","\t\t\tbest_loss = eval(model, valid_loader, best_loss, count)\n","\t\t\tmodel.train()\n","\n","\t\tcount += 1\n","\n","print(\"############################## Training End. ##############################\")\n","test_model = torch.load('{}{}_{}-{}.pth'.format(model_path, args.model, args.drop_rate, args.num_gradual))\n","test_model.cuda()\n","test(test_model, test_data_pos, user_pos)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vK6RkgUteJdS","executionInfo":{"status":"ok","timestamp":1639311150350,"user_tz":-330,"elapsed":1134452,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"27de75fc-3cec-4162-e83a-1fcaf217206d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["user, item num\n","45548 57396\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," cpuset_checked))\n"]},{"output_type":"stream","name":"stdout","text":["data loaded! user_num:45548, item_num:57396 train_data_len:1672520 test_user_num:45525\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n"," warnings.warn(warning.format(ret))\n"]},{"output_type":"stream","name":"stdout","text":["epoch: 0, iter: 2000, loss:0.4857825040817261\n","################### EVAL ######################\n","Eval loss:184.8551025390625\n","epoch: 1, iter: 4000, loss:0.24887847900390625\n","################### EVAL ######################\n","Eval loss:115.4634780883789\n","epoch: 1, iter: 6000, loss:0.21611060202121735\n","################### EVAL ######################\n","Eval loss:90.331787109375\n","epoch: 2, iter: 8000, loss:0.18297776579856873\n","################### EVAL ######################\n","Eval loss:77.42632293701172\n","epoch: 3, iter: 10000, loss:0.1536407172679901\n","################### EVAL ######################\n","Eval loss:68.5382080078125\n","epoch: 3, iter: 12000, loss:0.1829090118408203\n","################### EVAL ######################\n","Eval loss:62.065673828125\n","epoch: 4, iter: 14000, loss:0.11369739472866058\n","################### EVAL ######################\n","Eval loss:56.34415817260742\n","epoch: 4, iter: 16000, loss:0.08793307840824127\n","################### EVAL ######################\n","Eval loss:51.920955657958984\n","epoch: 5, iter: 18000, loss:0.1071598008275032\n","################### EVAL ######################\n","Eval loss:48.343868255615234\n","epoch: 6, iter: 20000, loss:0.07359810173511505\n","################### EVAL ######################\n","Eval loss:45.544002532958984\n","epoch: 6, iter: 22000, loss:0.1074337512254715\n","################### EVAL ######################\n","Eval loss:41.1801643371582\n","epoch: 7, iter: 24000, loss:0.07862947136163712\n","################### EVAL ######################\n","Eval loss:39.67675018310547\n","epoch: 7, iter: 26000, loss:0.08733634650707245\n","################### EVAL ######################\n","Eval loss:35.83267593383789\n","epoch: 8, iter: 28000, loss:0.0823584571480751\n","################### EVAL ######################\n","Eval loss:32.80326461791992\n","epoch: 9, iter: 30000, loss:0.029638517647981644\n","################### EVAL ######################\n","Eval loss:31.047956466674805\n","epoch: 9, iter: 32000, loss:0.08437806367874146\n","################### EVAL ######################\n","Eval loss:30.97898292541504\n","############################## Training End. ##############################\n","################### TEST ######################\n","Recall 0.0899-0.1473\n","NDCG 0.0364-0.0494\n"]}]},{"cell_type":"markdown","source":["## R_CE"],"metadata":{"id":"QVTERrOWhwQL"}},{"cell_type":"markdown","source":["### Loss function"],"metadata":{"id":"UgioesbdhwNS"}},{"cell_type":"code","source":["def loss_function(y, t, alpha):\n","\n"," loss = F.binary_cross_entropy_with_logits(y, t, reduce = False)\n"," y_ = torch.sigmoid(y).detach()\n"," weight = torch.pow(y_, alpha) * t + torch.pow((1-y_), alpha) * (1-t)\n"," loss_ = loss * weight\n"," loss_ = torch.mean(loss_)\n"," return loss_"],"metadata":{"id":"xhBZZF99hwJM"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Training & Evaluation"],"metadata":{"id":"VcNPdUPYh6D3"}},{"cell_type":"code","source":["############################## PREPARE DATASET ##########################\n","\n","train_data, valid_data, test_data_pos, user_pos, user_num ,item_num, train_mat, train_data_noisy = load_all(args.dataset, data_path)\n","\n","# construct the train and test datasets\n","train_dataset = NCFData(\n","\t\ttrain_data, item_num, train_mat, args.num_ng, 0, train_data_noisy)\n","valid_dataset = NCFData(\n","\t\tvalid_data, item_num, train_mat, args.num_ng, 1)\n","\n","train_loader = data.DataLoader(train_dataset,\n","\t\tbatch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True, worker_init_fn=worker_init_fn)\n","valid_loader = data.DataLoader(valid_dataset,\n","\t\tbatch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True, worker_init_fn=worker_init_fn)\n","\n","print(\"data loaded! user_num:{}, item_num:{} train_data_len:{} test_user_num:{}\".format(user_num, item_num, len(train_data), len(test_data_pos)))\n","\n","########################### CREATE MODEL #################################\n","if args.model == 'NeuMF-pre': # pre-training. Not used in our work.\n","\tGMF_model_path = model_path + 'GMF.pth'\n","\tMLP_model_path = model_path + 'MLP.pth'\n","\tNeuMF_model_path = model_path + 'NeuMF.pth'\n","\tassert os.path.exists(GMF_model_path), 'lack of GMF model'\n","\tassert os.path.exists(MLP_model_path), 'lack of MLP model'\n","\tGMF_model = torch.load(GMF_model_path)\n","\tMLP_model = torch.load(MLP_model_path)\n","else:\n","\tGMF_model = None\n","\tMLP_model = None\n","\n","model = NCF(user_num, item_num, args.factor_num, args.num_layers, \n","\t\t\t\t\t\targs.dropout, args.model, GMF_model, MLP_model)\n","\n","model.cuda()\n","BCE_loss = nn.BCEWithLogitsLoss()\n","\n","if args.model == 'NeuMF-pre':\n","\toptimizer = optim.SGD(model.parameters(), lr=args.lr)\n","else:\n","\toptimizer = optim.Adam(model.parameters(), lr=args.lr)\n","\n","# writer = SummaryWriter() # for visualization\n","\n","########################### Eval #####################################\n","def eval(model, valid_loader, best_loss, count):\n","\t\n","\tmodel.eval()\n","\tepoch_loss = 0\n","\tvalid_loader.dataset.ng_sample() # negative sampling\n","\tfor user, item, label, noisy_or_not in valid_loader:\n","\t\tuser = user.cuda()\n","\t\titem = item.cuda()\n","\t\tlabel = label.float().cuda()\n","\n","\t\tprediction = model(user, item)\n","\t\tloss = loss_function(prediction, label, args.alpha)\n","\t\tepoch_loss += loss.detach()\n","\tprint(\"################### EVAL ######################\")\n","\tprint(\"Eval loss:{}\".format(epoch_loss))\n","\tif epoch_loss < best_loss:\n","\t\tbest_loss = epoch_loss\n","\t\tif args.out:\n","\t\t\tif not os.path.exists(model_path):\n","\t\t\t\tos.mkdir(model_path)\n","\t\t\ttorch.save(model, '{}{}_{}.pth'.format(model_path, args.model, args.alpha))\n","\treturn best_loss\n","\n","########################### Test #####################################\n","def test(model, test_data_pos, user_pos):\n","\ttop_k = args.top_k\n","\tmodel.eval()\n","\t_, recall, NDCG, _ = test_all_users(model, 4096, item_num, test_data_pos, user_pos, top_k)\n","\n","\tprint(\"################### TEST ######################\")\n","\tprint(\"Recall {:.4f}-{:.4f}\".format(recall[0], recall[1]))\n","\tprint(\"NDCG {:.4f}-{:.4f}\".format(NDCG[0], NDCG[1]))\n","\n","########################### TRAINING #####################################\n","count, best_hr = 0, 0\n","best_loss = 1e9\n","\n","for epoch in range(args.epochs):\n","\tmodel.train() # Enable dropout (if have).\n","\n","\tstart_time = time.time()\n","\ttrain_loader.dataset.ng_sample()\n","\n","\tfor user, item, label, noisy_or_not in train_loader:\n","\t\tuser = user.cuda()\n","\t\titem = item.cuda()\n","\t\tlabel = label.float().cuda()\n","\n","\t\tmodel.zero_grad()\n","\t\tprediction = model(user, item)\n","\t\tloss = loss_function(prediction, label, args.alpha)\n","\t\tloss.backward()\n","\t\toptimizer.step()\n","\n","\t\tif count % args.eval_freq == 0 and count != 0:\n","\t\t\tprint(\"epoch: {}, iter: {}, loss:{}\".format(epoch, count, loss))\n","\t\t\tbest_loss = eval(model, valid_loader, best_loss, count)\n","\t\t\tmodel.train()\n","\n","\t\tcount += 1\n","\n","print(\"############################## Training End. ##############################\")\n","test_model = torch.load('{}{}_{}.pth'.format(model_path, args.model, args.alpha))\n","test_model.cuda()\n","test(test_model, test_data_pos, user_pos)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7Fr3rITCh6A7","executionInfo":{"status":"ok","timestamp":1639312641090,"user_tz":-330,"elapsed":1118311,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9321198d-3a69-4e28-e875-55fc46790edb"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["user, item num\n","45548 57396\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," cpuset_checked))\n"]},{"output_type":"stream","name":"stdout","text":["data loaded! user_num:45548, item_num:57396 train_data_len:1672520 test_user_num:45525\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n"," warnings.warn(warning.format(ret))\n"]},{"output_type":"stream","name":"stdout","text":["epoch: 0, iter: 2000, loss:0.41513800621032715\n","################### EVAL ######################\n","Eval loss:161.4785614013672\n","epoch: 1, iter: 4000, loss:0.22459657490253448\n","################### EVAL ######################\n","Eval loss:99.95635223388672\n","epoch: 1, iter: 6000, loss:0.19919627904891968\n","################### EVAL ######################\n","Eval loss:84.1415786743164\n","epoch: 2, iter: 8000, loss:0.17733421921730042\n","################### EVAL ######################\n","Eval loss:77.36993408203125\n","epoch: 3, iter: 10000, loss:0.15221725404262543\n","################### EVAL ######################\n","Eval loss:73.90560150146484\n","epoch: 3, iter: 12000, loss:0.16913804411888123\n","################### EVAL ######################\n","Eval loss:70.4941177368164\n","epoch: 4, iter: 14000, loss:0.14335578680038452\n","################### EVAL ######################\n","Eval loss:69.64494323730469\n","epoch: 4, iter: 16000, loss:0.12221892923116684\n","################### EVAL ######################\n","Eval loss:67.81050109863281\n","epoch: 5, iter: 18000, loss:0.11312472820281982\n","################### EVAL ######################\n","Eval loss:68.0766830444336\n","epoch: 6, iter: 20000, loss:0.09356789290904999\n","################### EVAL ######################\n","Eval loss:69.34413146972656\n","epoch: 6, iter: 22000, loss:0.125381737947464\n","################### EVAL ######################\n","Eval loss:68.00005340576172\n","epoch: 7, iter: 24000, loss:0.10218605399131775\n","################### EVAL ######################\n","Eval loss:69.70828247070312\n","epoch: 7, iter: 26000, loss:0.10325159877538681\n","################### EVAL ######################\n","Eval loss:68.36780548095703\n","epoch: 8, iter: 28000, loss:0.09633355587720871\n","################### EVAL ######################\n","Eval loss:70.18128204345703\n","epoch: 9, iter: 30000, loss:0.0662553533911705\n","################### EVAL ######################\n","Eval loss:72.70501708984375\n","epoch: 9, iter: 32000, loss:0.09527254104614258\n","################### EVAL ######################\n","Eval loss:71.64075469970703\n","############################## Training End. ##############################\n","################### TEST ######################\n","Recall 0.0864-0.1365\n","NDCG 0.0367-0.0481\n"]}]},{"cell_type":"markdown","source":["## References\n","\n","1. [https://github.com/RecoHut-Stanzas/S063707](https://github.com/RecoHut-Stanzas/S063707)\n","2. [https://arxiv.org/pdf/2112.01160v1.pdf](https://arxiv.org/pdf/2112.01160v1.pdf)\n","3. [https://github.com/WenjieWWJ/DenoisingRec](https://github.com/WenjieWWJ/DenoisingRec)"],"metadata":{"id":"wH6KkvQ7xVob"}},{"cell_type":"markdown","source":["---"],"metadata":{"id":"OHHk65wEo8uk"}},{"cell_type":"code","source":["!apt-get -qq install tree\n","!rm -r sample_data"],"metadata":{"id":"v0OE_IPAo8um"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!tree -h --du ."],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_-c-bayco8um","executionInfo":{"status":"ok","timestamp":1639312774688,"user_tz":-330,"elapsed":27,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"e9f3917f-b46d-41c5-ec0a-14c8a0fc8bf7"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[".\n","├── [126M] models\n","│   └── [126M] yelp\n","│   ├── [ 63M] GMF_0.2-30000.pth\n","│   └── [ 63M] GMF_0.2.pth\n","└── [ 26M] yelp\n"," ├── [ 241] README.md\n"," ├── [1.8M] yelp.test.negative\n"," ├── [ 21M] yelp.train.rating\n"," └── [2.7M] yelp.valid.rating\n","\n"," 152M used in 3 directories, 6 files\n"]}]},{"cell_type":"code","source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uHYQT0AMo8un","executionInfo":{"status":"ok","timestamp":1639312785478,"user_tz":-330,"elapsed":4282,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"666f7d5e-5e1f-48e8-f2dc-656995d46897"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2021-12-12 12:39:53\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.104+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","argparse: 1.1\n","IPython : 5.5.0\n","scipy : 1.4.1\n","torch : 1.10.0+cu111\n","numpy : 1.19.5\n","pandas : 1.1.5\n","\n"]}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"CrZCu7YIo8uo"}},{"cell_type":"markdown","source":["**END**"],"metadata":{"id":"YL6VsvtSo8uq"}}]} \ No newline at end of file diff --git a/_notebooks/2022-01-11-mbgmn-beibei.ipynb b/_notebooks/2022-01-11-mbgmn-beibei.ipynb new file mode 100644 index 0000000..50fa26e --- /dev/null +++ b/_notebooks/2022-01-11-mbgmn-beibei.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-11-mbgmn-beibei.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P197505%20%7C%20MB-GMN%20on%20BeiBei.ipynb","timestamp":1644603071396}],"collapsed_sections":[],"authorship_tag":"ABX9TyOnRz+yrAZ52jPIuWSjRWoM"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"24c87c2912974b6d8becb5b9190aa829":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_4314716903e64982b95392e7e37156bd","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_6f654bcada5d41f78b08c66d2e692fde","IPY_MODEL_9dd6ba2d32ff4c0cb76b467e51e8060c","IPY_MODEL_7fd736fa68424784a086f63ba9a43657"]}},"4314716903e64982b95392e7e37156bd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6f654bcada5d41f78b08c66d2e692fde":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0bbb945cc6d54d289cbf67827ae1b723","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_551b14c6f28644eea7c40845274e672c"}},"9dd6ba2d32ff4c0cb76b467e51e8060c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_02be78ca98ab4b39adaa14cfd532ce00","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0343c7f1998946c68782b6fa7993b824"}},"7fd736fa68424784a086f63ba9a43657":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_439b1cc5861d446b84eba9b023d4d410","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [10:19<00:00, 58.37s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1ceba6676a0f4736961546da9a54dac6"}},"0bbb945cc6d54d289cbf67827ae1b723":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"551b14c6f28644eea7c40845274e672c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"02be78ca98ab4b39adaa14cfd532ce00":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"0343c7f1998946c68782b6fa7993b824":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"439b1cc5861d446b84eba9b023d4d410":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"1ceba6676a0f4736961546da9a54dac6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","source":["# MB-GMN on BeiBei"],"metadata":{"id":"LOAjWAu1zj56"}},{"cell_type":"markdown","source":["## Executive summary\n","| | |\n","| --- | --- |\n","| Problem | Modern recommender systems often embed users and items into low-dimensional latent representations, based on their observed interactions. In practical recommendation scenarios, users often exhibit various intents which drive them to interact with items with multiple behavior types (e.g., click, tag-as-favorite, purchase). However, the diversity of user behaviors is ignored in most of the existing approaches, which makes them difficult to capture heterogeneous relational structures across different types of interactive behaviors. |\n","| Prblm Stmt. | We define a three-way tensor $X \\in \\mathbb{R}^{𝐼×𝐽×𝐾}$ to reflect the multi-typed interaction (e.g., click, tag-as-favorite, purchase) between user ($𝑢_𝑖 \\in U$) and item ($𝑣_𝑗 \\in V$). Here, 𝐾 denotes the number of behavior types. Specifically, the individual element $𝑥_{i,j}^k \\in X$ is set as 1 if user $𝑢_𝑖$ interacts with item $𝑣_𝑗$ under the 𝑘-th behavior type, and $𝑘_{𝑖,𝑗}$ = 0 otherwise. In the multi-behavior recommendation scenario, one type of user-item interaction will be treated as target behavior (e.g., purchase). Other behaviors are referred to as context behaviors (e.g., click, tag-as-favorite, add-to-cart) for providing insightful knowledge in assisting the target behavior prediction. Based on the aforementioned definitions, we formally state our studied problem as: • Input: the observed multi-behavior interaction tensor $X \\in \\mathbb{R}^{𝐼×𝐽×𝐾}$ between users U and items V across 𝐾 behavior types. • Output: the predictive function which estimates the likelihood of user $𝑢_𝑖$ adopts the item $𝑣_𝑗$ with the target behavior type. |\n","| Solution | Multi-Behavior recommendation framework with Graph Meta Network (MB-GMN) incorporate the multi-behavior pattern modeling into a meta-learning paradigm. Our developed MB-GMN empowers the user-item interaction learning with the capability of uncovering type-dependent behavior representations, which automatically distills the behavior heterogeneity and interaction diversity for recommendations. MB-GMN is composed of two key components: i) multi-behavior pattern encoding, a meta-knowledge learner that captures the personalized multi-behavior characteristics; ii) cross-type behavior dependency modeling, a transfer learning paradigm which learns a well-customized prediction network by transferring knowledge across different behavior types. |\n","| Dataset | BeiBei. |\n","| Preprocessing | Leave-one-out evaluation is leveraged for training and test set partition. We generate the test data set by including the last interactive item and consider the rest of data for training. For efficient and fair model evaluation, we pair each positive item instance with 99 randomly sampled non-interactive items for each user. We regard users’ purchases as the target behaviors. |\n","| Metrics | NDCG, HR |\n","| Models | MB-GMN |\n","| Cluster | Python 3.6+, Tensorflow 1.x |\n","| Tags | `MetaLearning`, `GNN` |\n","| Credits | akaxlh |"],"metadata":{"id":"bpPKhl0fzj2G"}},{"cell_type":"markdown","source":["## Process flow\n","\n","![](https://github.com/RecoHut-Stanzas/S346877/raw/main/images/process_flow.svg)"],"metadata":{"id":"B3NwTUYRzjyz"}},{"cell_type":"markdown","source":["## Model architecture\n","\n","![](https://github.com/RecoHut-Stanzas/S346877/raw/main/images/mbgmn_model.png)"],"metadata":{"id":"UiUls4wo5IBp"}},{"cell_type":"markdown","source":["*The model architecture of MB-GMN. (a) Multi-behavior pattern modeling with meta-knowledge learner for behavior heterogeneity; (b) Meta graph neural network which preserves the behavior semantics with high-order connectivity; (c) Metaknowledge transfer networks that customize the parameter of prediction layer to capture cross-type behavior dependency.*"],"metadata":{"id":"i35fbAIe5P2d"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"uZ5MRgBc0v49"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"A3mT2WvpnDfP","executionInfo":{"status":"ok","timestamp":1639298230167,"user_tz":-330,"elapsed":541,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"2819ceb9-ae56-40fe-e732-f7879df70c5e"},"outputs":[{"output_type":"stream","name":"stdout","text":["TensorFlow 1.x selected.\n"]}],"source":["%tensorflow_version 1.x"]},{"cell_type":"code","source":["import os\n","import numpy as np\n","import tensorflow as tf\n","from tensorflow.core.protobuf import config_pb2\n","import pickle\n","import argparse\n","import random\n","import gc\n","import datetime\n","from tensorflow.contrib.layers import xavier_initializer\n","from scipy.sparse import csr_matrix\n","import scipy.sparse as sp\n","from tqdm.notebook import tqdm"],"metadata":{"id":"_sPDAiS8nNN_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'"],"metadata":{"id":"kIIytIU8s3Jq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["os.makedirs('./History', exist_ok=True)"],"metadata":{"id":"vSKHNr0Atb8S"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def parse_args():\n","\tparser = argparse.ArgumentParser(description='Model Params')\n","\tparser.add_argument('--lr', default=1e-3, type=float, help='learning rate')\n","\tparser.add_argument('--batch', default=256, type=int, help='batch size')\n","\tparser.add_argument('--reg', default=1e-2, type=float, help='weight decay regularizer')\n","\tparser.add_argument('--epoch', default=10, type=int, help='number of epochs')\n","\tparser.add_argument('--decay', default=0.96, type=float, help='weight decay rate')\n","\tparser.add_argument('--save_path', default='tem', help='file name to save model and training record')\n","\tparser.add_argument('--latdim', default=32, type=int, help='embedding size')\n","\tparser.add_argument('--rank', default=4, type=int, help='embedding size')\n","\tparser.add_argument('--memosize', default=2, type=int, help='memory size')\n","\tparser.add_argument('--sampNum', default=40, type=int, help='batch size for sampling')\n","\tparser.add_argument('--att_head', default=2, type=int, help='number of attention heads')\n","\tparser.add_argument('--gnn_layer', default=2, type=int, help='number of gnn layers')\n","\tparser.add_argument('--trnNum', default=10000, type=int, help='number of training instances per epoch')\n","\tparser.add_argument('--load_model', default=None, help='model name to load')\n","\tparser.add_argument('--shoot', default=10, type=int, help='K of top k')\n","\tparser.add_argument('--data', default='beibei', type=str, help='name of dataset')\n","\tparser.add_argument('--target', default='buy', type=str, help='target behavior to predict on')\n","\tparser.add_argument('--deep_layer', default=0, type=int, help='number of deep layers to make the final prediction')\n","\tparser.add_argument('--mult', default=100, type=float, help='multiplier for the result')\n","\tparser.add_argument('--keepRate', default=0.7, type=float, help='rate for dropout')\n","\tparser.add_argument('--slot', default=5, type=float, help='length of time slots')\n","\tparser.add_argument('--graphSampleN', default=15000, type=int, help='use 25000 for training and 200000 for testing, empirically')\n","\tparser.add_argument('--divSize', default=10000, type=int, help='div size for smallTestEpoch')\n","\tparser.add_argument('--tstEpoch', default=3, type=int, help='number of epoch to test while training')\n","\tparser.add_argument('--subUsrSize', default=10, type=int, help='number of item for each sub-user')\n","\tparser.add_argument('--subUsrDcy', default=0.9, type=float, help='decay factor for sub-users over time')\n","\treturn parser.parse_args([])\n"," \n","args = parse_args()\n","\n","args.user = 21716\n","args.item = 7977\n","args.decay_step = args.trnNum//args.batch"],"metadata":{"id":"92B7jEwLtA0j"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Utils"],"metadata":{"id":"wVCw2Kc31DdY"}},{"cell_type":"code","source":["logmsg = ''\n","timemark = dict()\n","saveDefault = False\n","\n","def log(msg, save=None, oneline=False):\n","\tglobal logmsg\n","\tglobal saveDefault\n","\ttime = datetime.datetime.now()\n","\ttem = '%s: %s' % (time, msg)\n","\tif save != None:\n","\t\tif save:\n","\t\t\tlogmsg += tem + '\\n'\n","\telif saveDefault:\n","\t\tlogmsg += tem + '\\n'\n","\tif oneline:\n","\t\tprint(tem, end='\\r')\n","\telse:\n","\t\tprint(tem)\n","\n","def marktime(marker):\n","\tglobal timemark\n","\ttimemark[marker] = datetime.datetime.now()\n","\n","def SpentTime(marker):\n","\tglobal timemark\n","\tif marker not in timemark:\n","\t\tmsg = 'LOGGER ERROR, marker', marker, ' not found'\n","\t\ttem = '%s: %s' % (time, msg)\n","\t\tprint(tem)\n","\t\treturn False\n","\treturn datetime.datetime.now() - timemark[marker]\n","\n","def SpentTooLong(marker, day=0, hour=0, minute=0, second=0):\n","\tglobal timemark\n","\tif marker not in timemark:\n","\t\tmsg = 'LOGGER ERROR, marker', marker, ' not found'\n","\t\ttem = '%s: %s' % (time, msg)\n","\t\tprint(tem)\n","\t\treturn False\n","\treturn datetime.datetime.now() - timemark[marker] >= datetime.timedelta(days=day, hours=hour, minutes=minute, seconds=second)"],"metadata":{"id":"Z80jSTrUtr8g"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def RandomShuffle(infile, outfile, deleteSchema=False):\n","\twith open(infile, 'r', encoding='utf-8') as fs:\n","\t\tarr = fs.readlines()\n","\tif not arr[-1].endswith('\\n'):\n","\t\tarr[-1] += '\\n'\n","\tif deleteSchema:\n","\t\tarr = arr[1:]\n","\trandom.shuffle(arr)\n","\twith open(outfile, 'w', encoding='utf-8') as fs:\n","\t\tfor line in arr:\n","\t\t\tfs.write(line)\n","\tdel arr\n","\n","def WriteToBuff(buff, line, out):\n","\tBUFF_SIZE = 1000000\n","\tbuff.append(line)\n","\tif len(buff) == BUFF_SIZE:\n","\t\tWriteToDisk(buff, out)\n","\n","def WriteToDisk(buff, out):\n","\twith open(out, 'a', encoding='utf-8') as fs:\n","\t\tfor line in buff:\n","\t\t\tfs.write(line)\n","\tbuff.clear()\n","\n","\n","def SubDataSet(infile, outfile1, outfile2, rate):\n","\tout1 = list()\n","\tout2 = list()\n","\twith open(infile, 'r', encoding='utf-8') as fs:\n","\t\tfor line in fs:\n","\t\t\tif random.random() < rate:\n","\t\t\t\tWriteToBuff(out1, line, outfile1)\n","\t\t\telse:\n","\t\t\t\tWriteToBuff(out2, line, outfile2)\n","\tWriteToDisk(out1, outfile1)\n","\tWriteToDisk(out2, outfile2)\n","\n","def CombineFiles(files, out):\n","\tbuff = list()\n","\tfor file in files:\n","\t\twith open(file, 'r') as fs:\n","\t\t\tfor line in fs:\n","\t\t\t\tWriteToBuff(buff, line, out)\n","\tWriteToDisk(buff, out)"],"metadata":{"id":"0BvkMCQytcGV"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## NN Layers"],"metadata":{"id":"X5d4EeIU1ARg"}},{"cell_type":"code","source":["paramId = 0\n","biasDefault = False\n","params = {}\n","regParams = {}\n","ita = 0.2\n","leaky = 0.1\n","\n","def getParamId():\n","\tglobal paramId\n","\tparamId += 1\n","\treturn paramId\n","\n","def setIta(ITA):\n","\tita = ITA\n","\n","def setBiasDefault(val):\n","\tglobal biasDefault\n","\tbiasDefault = val\n","\n","def getParam(name):\n","\treturn params[name]\n","\n","def addReg(name, param):\n","\tglobal regParams\n","\tif name not in regParams:\n","\t\tregParams[name] = param\n","\telse:\n","\t\tprint('ERROR: Parameter already exists')\n","\n","def addParam(name, param):\n","\tglobal params\n","\tif name not in params:\n","\t\tparams[name] = param\n","\n","def defineRandomNameParam(shape, dtype=tf.float32, reg=False, initializer='xavier', trainable=True):\n","\tname = 'defaultParamName%d'%getParamId()\n","\treturn defineParam(name, shape, dtype, reg, initializer, trainable)\n","\n","def defineParam(name, shape, dtype=tf.float32, reg=False, initializer='xavier', trainable=True):\n","\tglobal params\n","\tglobal regParams\n","\tassert name not in params, 'name %s already exists' % name\n","\tif initializer == 'xavier':\n","\t\tret = tf.get_variable(name=name, dtype=dtype, shape=shape,\n","\t\t\tinitializer=xavier_initializer(dtype=tf.float32),\n","\t\t\ttrainable=trainable)\n","\telif initializer == 'trunc_normal':\n","\t\tret = tf.get_variable(name=name, initializer=tf.random.truncated_normal(shape=[int(shape[0]), shape[1]], mean=0.0, stddev=0.03, dtype=dtype))\n","\telif initializer == 'zeros':\n","\t\tret = tf.get_variable(name=name, dtype=dtype,\n","\t\t\tinitializer=tf.zeros(shape=shape, dtype=tf.float32),\n","\t\t\ttrainable=trainable)\n","\telif initializer == 'ones':\n","\t\tret = tf.get_variable(name=name, dtype=dtype, initializer=tf.ones(shape=shape, dtype=tf.float32), trainable=trainable)\n","\telif not isinstance(initializer, str):\n","\t\tret = tf.get_variable(name=name, dtype=dtype,\n","\t\t\tinitializer=initializer, trainable=trainable)\n","\telse:\n","\t\tprint('ERROR: Unrecognized initializer')\n","\t\texit()\n","\tparams[name] = ret\n","\tif reg:\n","\t\tregParams[name] = ret\n","\treturn ret\n","\n","def getOrDefineParam(name, shape, dtype=tf.float32, reg=False, initializer='xavier', trainable=True, reuse=False):\n","\tglobal params\n","\tglobal regParams\n","\tif name in params:\n","\t\tassert reuse, 'Reusing Param %s Not Specified' % name\n","\t\tif reg and name not in regParams:\n","\t\t\tregParams[name] = params[name]\n","\t\treturn params[name]\n","\treturn defineParam(name, shape, dtype, reg, initializer, trainable)\n","\n","def BN(inp, name=None):\n","\tglobal ita\n","\tdim = inp.get_shape()[1]\n","\tname = 'defaultParamName%d'%getParamId()\n","\tscale = tf.Variable(tf.ones([dim]))\n","\tshift = tf.Variable(tf.zeros([dim]))\n","\tfcMean, fcVar = tf.nn.moments(inp, axes=[0])\n","\tema = tf.train.ExponentialMovingAverage(decay=0.5)\n","\temaApplyOp = ema.apply([fcMean, fcVar])\n","\twith tf.control_dependencies([emaApplyOp]):\n","\t\tmean = tf.identity(fcMean)\n","\t\tvar = tf.identity(fcVar)\n","\tret = tf.nn.batch_normalization(inp, mean, var, shift,\n","\t\tscale, 1e-8)\n","\treturn ret\n","\n","def FC(inp, outDim, name=None, useBias=False, activation=None, reg=False, useBN=False, dropout=None, initializer='xavier', reuse=False, biasReg=False, biasInitializer='zeros'):\n","\tglobal params\n","\tglobal regParams\n","\tglobal leaky\n","\tinDim = inp.get_shape()[1]\n","\ttemName = name if name!=None else 'defaultParamName%d'%getParamId()\n","\tW = getOrDefineParam(temName, [inDim, outDim], reg=reg, initializer=initializer, reuse=reuse)\n","\tif dropout != None:\n","\t\tret = tf.nn.dropout(inp, rate=dropout) @ W\n","\telse:\n","\t\tret = inp @ W\n","\tif useBias:\n","\t\tret = Bias(ret, name=name, reuse=reuse, reg=biasReg, initializer=biasInitializer)\n","\tif useBN:\n","\t\tret = BN(ret)\n","\tif activation != None:\n","\t\tret = Activate(ret, activation)\n","\treturn ret\n","\n","def Bias(data, name=None, reg=False, reuse=False, initializer='zeros'):\n","\tinDim = data.get_shape()[-1]\n","\ttemName = name if name!=None else 'defaultParamName%d'%getParamId()\n","\ttemBiasName = temName + 'Bias'\n","\tbias = getOrDefineParam(temBiasName, inDim, reg=False, initializer=initializer, reuse=reuse)\n","\tif reg:\n","\t\tregParams[temBiasName] = bias\n","\treturn data + bias\n","\n","def ActivateHelp(data, method):\n","\tif method == 'relu':\n","\t\tret = tf.nn.relu(data)\n","\telif method == 'sigmoid':\n","\t\tret = tf.nn.sigmoid(data)\n","\telif method == 'tanh':\n","\t\tret = tf.nn.tanh(data)\n","\telif method == 'softmax':\n","\t\tret = tf.nn.softmax(data, axis=-1)\n","\telif method == 'leakyRelu':\n","\t\tret = tf.maximum(leaky*data, data)\n","\telif method == 'twoWayLeakyRelu6':\n","\t\ttemMask = tf.to_float(tf.greater(data, 6.0))\n","\t\tret = temMask * (6 + leaky * (data - 6)) + (1 - temMask) * tf.maximum(leaky * data, data)\n","\telif method == '-1relu':\n","\t\tret = tf.maximum(-1.0, data)\n","\telif method == 'relu6':\n","\t\tret = tf.maximum(0.0, tf.minimum(6.0, data))\n","\telif method == 'relu3':\n","\t\tret = tf.maximum(0.0, tf.minimum(3.0, data))\n","\telse:\n","\t\traise Exception('Error Activation Function')\n","\treturn ret\n","\n","def Activate(data, method, useBN=False):\n","\tglobal leaky\n","\tif useBN:\n","\t\tret = BN(data)\n","\telse:\n","\t\tret = data\n","\tret = ActivateHelp(ret, method)\n","\treturn ret\n","\n","def Regularize(names=None, method='L2'):\n","\tret = 0\n","\tif method == 'L1':\n","\t\tif names != None:\n","\t\t\tfor name in names:\n","\t\t\t\tret += tf.reduce_sum(tf.abs(getParam(name)))\n","\t\telse:\n","\t\t\tfor name in regParams:\n","\t\t\t\tret += tf.reduce_sum(tf.abs(regParams[name]))\n","\telif method == 'L2':\n","\t\tif names != None:\n","\t\t\tfor name in names:\n","\t\t\t\tret += tf.reduce_sum(tf.square(getParam(name)))\n","\t\telse:\n","\t\t\tfor name in regParams:\n","\t\t\t\tret += tf.reduce_sum(tf.square(regParams[name]))\n","\treturn ret\n","\n","def Dropout(data, rate):\n","\tif rate == None:\n","\t\treturn data\n","\telse:\n","\t\treturn tf.nn.dropout(data, rate=rate)\n","\n","def selfAttention(localReps, number, inpDim, numHeads):\n","\tQ = defineRandomNameParam([inpDim, inpDim], reg=True)\n","\tK = defineRandomNameParam([inpDim, inpDim], reg=True)\n","\tV = defineRandomNameParam([inpDim, inpDim], reg=True)\n","\trspReps = tf.reshape(tf.stack(localReps, axis=1), [-1, inpDim])\n","\tq = tf.reshape(rspReps @ Q, [-1, number, 1, numHeads, inpDim//numHeads])\n","\tk = tf.reshape(rspReps @ K, [-1, 1, number, numHeads, inpDim//numHeads])\n","\tv = tf.reshape(rspReps @ V, [-1, 1, number, numHeads, inpDim//numHeads])\n","\tatt = tf.nn.softmax(tf.reduce_sum(q * k, axis=-1, keepdims=True) / tf.sqrt(inpDim/numHeads), axis=2)\n","\tattval = tf.reshape(tf.reduce_sum(att * v, axis=2), [-1, number, inpDim])\n","\trets = [None] * number\n","\tparamId = 'dfltP%d' % getParamId()\n","\tfor i in range(number):\n","\t\ttem1 = tf.reshape(tf.slice(attval, [0, i, 0], [-1, 1, -1]), [-1, inpDim])\n","\t\t# tem2 = FC(tem1, inpDim, useBias=True, name=paramId+'_1', reg=True, activation='relu', reuse=True) + localReps[i]\n","\t\trets[i] = tem1 + localReps[i]\n","\treturn rets\n","\n","def lightSelfAttention(localReps, number, inpDim, numHeads):\n","\tQ = defineRandomNameParam([inpDim, inpDim], reg=True)\n","\trspReps = tf.reshape(tf.stack(localReps, axis=1), [-1, inpDim])\n","\ttem = rspReps @ Q\n","\tq = tf.reshape(tem, [-1, number, 1, numHeads, inpDim//numHeads])\n","\tk = tf.reshape(tem, [-1, 1, number, numHeads, inpDim//numHeads])\n","\tv = tf.reshape(rspReps, [-1, 1, number, numHeads, inpDim//numHeads])\n","\t# att = tf.nn.softmax(tf.reduce_sum(q * k, axis=-1, keepdims=True) * tf.sqrt(inpDim/numHeads), axis=2)\n","\tatt = tf.nn.softmax(tf.reduce_sum(q * k, axis=-1, keepdims=True) / tf.sqrt(inpDim/numHeads), axis=2)\n","\tattval = tf.reshape(tf.reduce_sum(att * v, axis=2), [-1, number, inpDim])\n","\trets = [None] * number\n","\tparamId = 'dfltP%d' % getParamId()\n","\tfor i in range(number):\n","\t\ttem1 = tf.reshape(tf.slice(attval, [0, i, 0], [-1, 1, -1]), [-1, inpDim])\n","\t\t# tem2 = FC(tem1, inpDim, useBias=True, name=paramId+'_1', reg=True, activation='relu', reuse=True) + localReps[i]\n","\t\trets[i] = tem1 + localReps[i]\n","\treturn rets#, tf.squeeze(att)"],"metadata":{"id":"C5z3c78ctcD0"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Dataset"],"metadata":{"id":"ZBlNyUpR08Qr"}},{"cell_type":"code","source":["!git clone --branch v1 https://github.com/RecoHut-Datasets/beibei.git"],"metadata":{"id":"LiC15-XSnFCC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def transpose(mat):\n","\tcoomat = sp.coo_matrix(mat)\n","\treturn csr_matrix(coomat.transpose())\n","\n","def negSamp(temLabel, sampSize, nodeNum):\n","\tnegset = [None] * sampSize\n","\tcur = 0\n","\twhile cur < sampSize:\n","\t\trdmItm = np.random.choice(nodeNum)\n","\t\tif temLabel[rdmItm] == 0:\n","\t\t\tnegset[cur] = rdmItm\n","\t\t\tcur += 1\n","\treturn negset\n","\n","def transToLsts(mat, mask=False, norm=False):\n","\tshape = [mat.shape[0], mat.shape[1]]\n","\tcoomat = sp.coo_matrix(mat)\n","\tindices = np.array(list(map(list, zip(coomat.row, coomat.col))), dtype=np.int32)\n","\tdata = coomat.data.astype(np.float32)\n","\n","\tif norm:\n","\t\trowD = np.squeeze(np.array(1 / (np.sqrt(np.sum(mat, axis=1) + 1e-8) + 1e-8)))\n","\t\tcolD = np.squeeze(np.array(1 / (np.sqrt(np.sum(mat, axis=0) + 1e-8) + 1e-8)))\n","\t\tfor i in range(len(data)):\n","\t\t\trow = indices[i, 0]\n","\t\t\tcol = indices[i, 1]\n","\t\t\tdata[i] = data[i] * rowD[row] * colD[col]\n","\n","\t# half mask\n","\tif mask:\n","\t\tspMask = (np.random.uniform(size=data.shape) > 0.5) * 1.0\n","\t\tdata = data * spMask\n","\n","\tif indices.shape[0] == 0:\n","\t\tindices = np.array([[0, 0]], dtype=np.int32)\n","\t\tdata = np.array([0.0], np.float32)\n","\treturn indices, data, shape\n","\n","class DataHandler:\n"," def __init__(self):\n"," predir = './beibei/'\n"," behs = ['pv', 'cart', 'buy']\n"," self.predir = predir\n"," self.behs = behs\n"," self.trnfile = predir + 'trn_'\n"," self.tstfile = predir + 'tst_'\n","\n"," def LoadData(self):\n"," trnMats = list()\n"," for i in range(len(self.behs)):\n"," beh = self.behs[i]\n"," path = self.trnfile + beh\n"," with open(path, 'rb') as fs:\n"," mat = (pickle.load(fs) != 0).astype(np.float32)\n"," trnMats.append(mat)\n"," # test set\n"," path = self.tstfile + 'int'\n"," with open(path, 'rb') as fs:\n"," tstInt = np.array(pickle.load(fs))\n"," tstStat = (tstInt != None)\n"," tstUsrs = np.reshape(np.argwhere(tstStat != False), [-1])\n"," self.trnMats = trnMats\n"," self.tstInt = tstInt\n"," self.tstUsrs = tstUsrs\n"," args.user, args.item = self.trnMats[0].shape\n"," args.behNum = len(self.behs)\n"," self.prepareGlobalData()\n","\n"," def prepareGlobalData(self):\n"," adj = 0\n"," for i in range(args.behNum):\n"," adj = adj + self.trnMats[i]\n"," adj = (adj != 0).astype(np.float32)\n"," self.labelP = np.squeeze(np.array(np.sum(adj, axis=0)))\n"," tpadj = transpose(adj)\n"," adjNorm = np.reshape(np.array(np.sum(adj, axis=1)), [-1])\n"," tpadjNorm = np.reshape(np.array(np.sum(tpadj, axis=1)), [-1])\n"," for i in range(adj.shape[0]):\n"," for j in range(adj.indptr[i], adj.indptr[i+1]):\n"," adj.data[j] /= adjNorm[i]\n"," for i in range(tpadj.shape[0]):\n"," for j in range(tpadj.indptr[i], tpadj.indptr[i+1]):\n"," tpadj.data[j] /= tpadjNorm[i]\n"," self.adj = adj\n"," self.tpadj = tpadj\n","\n"," def sampleLargeGraph(self, pckUsrs, pckItms=None, sampDepth=2, sampNum=args.graphSampleN, preSamp=False):\n"," adj = self.adj\n"," tpadj = self.tpadj\n"," def makeMask(nodes, size):\n"," mask = np.ones(size)\n"," if not nodes is None:\n"," mask[nodes] = 0.0\n"," return mask\n","\n"," def updateBdgt(adj, nodes):\n"," if nodes is None:\n"," return 0\n"," tembat = 1000\n"," ret = 0\n"," for i in range(int(np.ceil(len(nodes) / tembat))):\n"," st = tembat * i\n"," ed = min((i+1) * tembat, len(nodes))\n"," temNodes = nodes[st: ed]\n"," ret += np.sum(adj[temNodes], axis=0)\n"," return ret\n","\n"," def sample(budget, mask, sampNum):\n"," score = (mask * np.reshape(np.array(budget), [-1])) ** 2\n"," norm = np.sum(score)\n"," if norm == 0:\n"," return np.random.choice(len(score), 1), sampNum - 1\n"," score = list(score / norm)\n"," arrScore = np.array(score)\n"," posNum = np.sum(np.array(score)!=0)\n"," if posNum < sampNum:\n"," pckNodes1 = np.squeeze(np.argwhere(arrScore!=0))\n"," # pckNodes2 = np.random.choice(np.squeeze(np.argwhere(arrScore==0.0)), min(len(score) - posNum, sampNum - posNum), replace=False)\n"," # pckNodes = np.concatenate([pckNodes1, pckNodes2], axis=0)\n"," pckNodes = pckNodes1\n"," else:\n"," pckNodes = np.random.choice(len(score), sampNum, p=score, replace=False)\n"," return pckNodes, max(sampNum - posNum, 0)\n","\n"," def constructData(usrs, itms):\n"," adjs = self.trnMats\n"," pckAdjs = []\n"," pckTpAdjs = []\n"," for i in range(len(adjs)):\n"," pckU = adjs[i][usrs]\n"," tpPckI = transpose(pckU)[itms]\n"," pckTpAdjs.append(tpPckI)\n"," pckAdjs.append(transpose(tpPckI))\n"," return pckAdjs, pckTpAdjs, usrs, itms\n","\n"," usrMask = makeMask(pckUsrs, adj.shape[0])\n"," itmMask = makeMask(pckItms, adj.shape[1])\n"," itmBdgt = updateBdgt(adj, pckUsrs)\n"," if pckItms is None:\n"," pckItms, _ = sample(itmBdgt, itmMask, len(pckUsrs))\n"," itmMask = itmMask * makeMask(pckItms, adj.shape[1])\n"," usrBdgt = updateBdgt(tpadj, pckItms)\n"," uSampRes = 0\n"," iSampRes = 0\n"," for i in range(sampDepth + 1):\n"," uSamp = uSampRes + (sampNum if i < sampDepth else 0)\n"," iSamp = iSampRes + (sampNum if i < sampDepth else 0)\n"," newUsrs, uSampRes = sample(usrBdgt, usrMask, uSamp)\n"," usrMask = usrMask * makeMask(newUsrs, adj.shape[0])\n"," newItms, iSampRes = sample(itmBdgt, itmMask, iSamp)\n"," itmMask = itmMask * makeMask(newItms, adj.shape[1])\n"," if i == sampDepth or i == sampDepth and uSampRes == 0 and iSampRes == 0:\n"," break\n"," usrBdgt += updateBdgt(tpadj, newItms)\n"," itmBdgt += updateBdgt(adj, newUsrs)\n"," usrs = np.reshape(np.argwhere(usrMask==0), [-1])\n"," itms = np.reshape(np.argwhere(itmMask==0), [-1])\n"," return constructData(usrs, itms)"],"metadata":{"id":"WJf_ck4htcBW"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Model"],"metadata":{"id":"cxWiPevg2o4x"}},{"cell_type":"code","source":["class Recommender:\n","\tdef __init__(self, sess, handler):\n","\t\tself.sess = sess\n","\t\tself.handler = handler\n","\n","\t\tprint('USER', args.user, 'ITEM', args.item)\n","\t\tself.metrics = dict()\n","\t\tmets = ['Loss', 'preLoss', 'HR', 'NDCG']\n","\t\tfor met in mets:\n","\t\t\tself.metrics['Train' + met] = list()\n","\t\t\tself.metrics['Test' + met] = list()\n","\n","\tdef makePrint(self, name, ep, reses, save):\n","\t\tret = 'Epoch %d/%d, %s: ' % (ep, args.epoch, name)\n","\t\tfor metric in reses:\n","\t\t\tval = reses[metric]\n","\t\t\tret += '%s = %.4f, ' % (metric, val)\n","\t\t\ttem = name + metric\n","\t\t\tif save and tem in self.metrics:\n","\t\t\t\tself.metrics[tem].append(val)\n","\t\tret = ret[:-2] + ' '\n","\t\treturn ret\n","\n","\tdef run(self):\n","\t\tself.prepareModel()\n","\t\tlog('Model Prepared')\n","\t\tif args.load_model != None:\n","\t\t\tself.loadModel()\n","\t\t\tstloc = len(self.metrics['TrainLoss']) * args.tstEpoch - (args.tstEpoch - 1)\n","\t\telse:\n","\t\t\tstloc = 0\n","\t\t\tinit = tf.global_variables_initializer()\n","\t\t\tself.sess.run(init)\n","\t\t\tlog('Variables Initiated')\n","\t\tfor ep in tqdm(range(stloc, args.epoch)):\n","\t\t\ttest = (ep % args.tstEpoch == 0)\n","\t\t\treses = self.trainEpoch()\n","\t\t\tlog(self.makePrint('Train', ep, reses, test))\n","\t\t\tif test:\n","\t\t\t\treses = self.testEpoch()\n","\t\t\t\tlog(self.makePrint('Test', ep, reses, test))\n","\t\t\tif ep % args.tstEpoch == 0:\n","\t\t\t\tself.saveHistory()\n","\t\t\tprint()\n","\t\treses = self.testEpoch()\n","\t\tlog(self.makePrint('Test', args.epoch, reses, True))\n","\t\tself.saveHistory()\n","\n","\tdef messagePropagate(self, lats, adj, lats2):\n","\t\treturn Activate(tf.sparse.sparse_dense_matmul(adj, lats), self.actFunc)\n","\n","\tdef metaForSpecialize(self, uEmbed, iEmbed, behEmbed, adjs, tpAdjs):\n","\t\tlatdim = args.latdim // 2\n","\t\trank = args.rank\n","\t\tassert len(adjs) == len(tpAdjs)\n","\t\tuNeighbor = iNeighbor = 0\n","\t\tfor i in range(len(adjs)):\n","\t\t\tuNeighbor += tf.sparse.sparse_dense_matmul(adjs[i], iEmbed)\n","\t\t\tiNeighbor += tf.sparse.sparse_dense_matmul(tpAdjs[i], uEmbed)\n","\t\tubehEmbed = tf.expand_dims(behEmbed, axis=0) * tf.ones_like(uEmbed)\n","\t\tibehEmbed = tf.expand_dims(behEmbed, axis=0) * tf.ones_like(iEmbed)\n","\t\tuMetaLat = FC(tf.concat([ubehEmbed, uEmbed, uNeighbor], axis=-1), latdim, useBias=True, activation=self.actFunc, reg=True, name='specMeta_FC1', reuse=True)\n","\t\tiMetaLat = FC(tf.concat([ibehEmbed, iEmbed, iNeighbor], axis=-1), latdim, useBias=True, activation=self.actFunc, reg=True, name='specMeta_FC1', reuse=True)\n","\t\tuW1 = tf.reshape(FC(uMetaLat, rank * latdim, useBias=True, reg=True, biasInitializer='xavier', biasReg=True, name='specMeta_FC2', reuse=True), [-1, latdim, rank])\n","\t\tuW2 = tf.reshape(FC(uMetaLat, rank * latdim, useBias=True, reg=True, biasInitializer='xavier', biasReg=True, name='specMeta_FC3', reuse=True), [-1, rank, latdim])\n","\t\tiW1 = tf.reshape(FC(iMetaLat, rank * latdim, useBias=True, reg=True, biasInitializer='xavier', biasReg=True, name='specMeta_FC4', reuse=True), [-1, latdim, rank])\n","\t\tiW2 = tf.reshape(FC(iMetaLat, rank * latdim, useBias=True, reg=True, biasInitializer='xavier', biasReg=True, name='specMeta_FC5', reuse=True), [-1, rank, latdim])\n","\n","\t\tparams = {'uW1': uW1, 'uW2': uW2, 'iW1': iW1, 'iW2': iW2}\n","\t\treturn params\n","\n","\tdef specialize(self, uEmbed, iEmbed, params):\n","\t\tretUEmbed = tf.reduce_sum(tf.expand_dims(uEmbed, axis=-1) * params['uW1'], axis=1)\n","\t\tretUEmbed = tf.reduce_sum(tf.expand_dims(retUEmbed, axis=-1) * params['uW2'], axis=1)\n","\t\tretUEmbed = tf.concat([retUEmbed, uEmbed], axis=-1)\n","\t\tretIEmbed = tf.reduce_sum(tf.expand_dims(iEmbed, axis=-1) * params['iW1'], axis=1)\n","\t\tretIEmbed = tf.reduce_sum(tf.expand_dims(retIEmbed, axis=-1) * params['iW2'], axis=1)\n","\t\tretIEmbed = tf.concat([retIEmbed, iEmbed], axis=-1)\n","\t\treturn retUEmbed, retIEmbed\n","\n","\tdef defineModel(self):\n","\t\tuEmbed0 = defineParam('uEmbed0', [args.user, args.latdim//2], reg=True)\n","\t\tiEmbed0 = defineParam('iEmbed0', [args.item, args.latdim//2], reg=True)\n","\t\tbehEmbeds = defineParam('behEmbeds', [args.behNum + 1, args.latdim//2])\n","\t\tself.ulat = [0] * (args.behNum + 1)\n","\t\tself.ilat = [0] * (args.behNum + 1)\n","\t\tfor beh in range(args.behNum):\n","\t\t\tparams = self.metaForSpecialize(uEmbed0, iEmbed0, behEmbeds[beh], [self.adjs[beh]], [self.tpAdjs[beh]])\n","\t\t\tbehUEmbed0, behIEmbed0 = self.specialize(uEmbed0, iEmbed0, params)\n","\t\t\t# behUEmbed0 = uEmbed0\n","\t\t\t# behIEmbed0 = iEmbed0\n","\t\t\tulats = [behUEmbed0]\n","\t\t\tilats = [behIEmbed0]\n","\t\t\tfor i in range(args.gnn_layer):\n","\t\t\t\tulat = self.messagePropagate(ilats[-1], self.adjs[beh], ulats[-1])\n","\t\t\t\tilat = self.messagePropagate(ulats[-1], self.tpAdjs[beh], ilats[-1])\n","\t\t\t\tulats.append(ulat + ulats[-1])\n","\t\t\t\tilats.append(ilat + ilats[-1])\n","\t\t\tself.ulat[beh] = tf.add_n(ulats)\n","\t\t\tself.ilat[beh] = tf.add_n(ilats)\n","\n","\t\tparams = self.metaForSpecialize(uEmbed0, iEmbed0, behEmbeds[-1], self.adjs, self.tpAdjs)\n","\t\tbehUEmbed0, behIEmbed0 = self.specialize(uEmbed0, iEmbed0, params)\n","\t\tulats = [behUEmbed0]\n","\t\tilats = [behIEmbed0]\n","\t\tfor i in range(args.gnn_layer):\n","\t\t\tubehLats = []\n","\t\t\tibehLats = []\n","\t\t\tfor beh in range(args.behNum):\n","\t\t\t\tulat = self.messagePropagate(ilats[-1], self.adjs[beh], ulats[-1])\n","\t\t\t\tilat = self.messagePropagate(ulats[-1], self.tpAdjs[beh], ilats[-1])\n","\t\t\t\tubehLats.append(ulat)\n","\t\t\t\tibehLats.append(ilat)\n","\t\t\tulat = tf.add_n(lightSelfAttention(ubehLats, args.behNum, args.latdim, args.att_head))\n","\t\t\tilat = tf.add_n(lightSelfAttention(ibehLats, args.behNum, args.latdim, args.att_head))\n","\t\t\tulats.append(ulat)\n","\t\t\tilats.append(ilat)\n","\t\tself.ulat[-1] = tf.add_n(ulats)\n","\t\tself.ilat[-1] = tf.add_n(ilats)\n","\n","\tdef metaForPredict(self, src_ulat, src_ilat, tgt_ulat, tgt_ilat):\n","\t\tlatdim = args.latdim\n","\t\tsrc_ui = FC(tf.concat([src_ulat * src_ilat, src_ulat, src_ilat], axis=-1), latdim, reg=True, useBias=True, activation=self.actFunc, name='predMeta_FC1', reuse=True)\n","\t\ttgt_ui = FC(tf.concat([tgt_ulat * tgt_ilat, tgt_ulat, tgt_ilat], axis=-1), latdim, reg=True, useBias=True, activation=self.actFunc, name='predMeta_FC1', reuse=True)\n","\t\tmetalat = FC(tf.concat([src_ui * tgt_ui, src_ui, tgt_ui], axis=-1), latdim * 3, reg=True, useBias=True, activation=self.actFunc, name='predMeta_FC2', reuse=True)\n","\t\tw1 = tf.reshape(FC(metalat, latdim * 3 * latdim, reg=True, useBias=True, name='predMeta_FC3', reuse=True, biasReg=True, biasInitializer='xavier'), [-1, latdim * 3, latdim])\n","\t\tb1 = tf.reshape(FC(metalat, latdim, reg=True, useBias=True, name='predMeta_FC4', reuse=True), [-1, 1, latdim])\n","\t\tw2 = tf.reshape(FC(metalat, latdim, reg=True, useBias=True, name='predMeta_FC5', reuse=True, biasReg=True,biasInitializer='xavier'), [-1, latdim, 1])\n","\n","\t\tparams = {\n","\t\t\t'w1': w1,\n","\t\t\t'b1': b1,\n","\t\t\t'w2': w2\n","\t\t}\n","\t\treturn params\n","\n","\tdef _predict(self, ulat, ilat, params):\n","\t\tpredEmbed = tf.expand_dims(tf.concat([ulat * ilat, ulat, ilat], axis=-1), axis=1)\n","\t\tpredEmbed = Activate(predEmbed @ params['w1'] + params['b1'], self.actFunc)\n","\t\tpreds = tf.squeeze(predEmbed @ params['w2'])\n","\t\treturn preds\n","\n","\tdef predict(self, src, tgt):\n","\t\tuids = self.uids[tgt]\n","\t\tiids = self.iids[tgt]\n","\n","\t\tsrc_ulat = tf.nn.embedding_lookup(self.ulat[src], uids)\n","\t\tsrc_ilat = tf.nn.embedding_lookup(self.ilat[src], iids)\n","\t\ttgt_ulat = tf.nn.embedding_lookup(self.ulat[tgt], uids)\n","\t\ttgt_ilat = tf.nn.embedding_lookup(self.ilat[tgt], iids)\n","\n","\t\tpredParams = self.metaForPredict(src_ulat, src_ilat, tgt_ulat, tgt_ilat)\n","\t\treturn self._predict(src_ulat, src_ilat, predParams) * args.mult\n","\n","\tdef prepareModel(self):\n","\t\tself.actFunc = 'leakyRelu'\n","\t\tself.adjs = []\n","\t\tself.tpAdjs = []\n","\t\tself.uids, self.iids = [], []\n","\t\tfor i in range(args.behNum):\n","\t\t\tadj = self.handler.trnMats[i]\n","\t\t\tidx, data, shape = transToLsts(adj, norm=True)\n","\t\t\tself.adjs.append(tf.sparse.SparseTensor(idx, data, shape))\n","\t\t\tidx, data, shape = transToLsts(transpose(adj), norm=True)\n","\t\t\tself.tpAdjs.append(tf.sparse.SparseTensor(idx, data, shape))\n","\t\t\tself.uids.append(tf.placeholder(name='uids'+str(i), dtype=tf.int32, shape=[None]))\n","\t\t\tself.iids.append(tf.placeholder(name='iids'+str(i), dtype=tf.int32, shape=[None]))\n","\t\t\n","\t\tself.defineModel()\n","\t\tself.preLoss = 0\n","\t\tfor src in range(args.behNum + 1):\n","\t\t\tfor tgt in range(args.behNum):\n","\t\t\t\tpreds = self.predict(src, tgt)\n","\t\t\t\tsampNum = tf.shape(self.uids[tgt])[0] // 2\n","\t\t\t\tposPred = tf.slice(preds, [0], [sampNum])\n","\t\t\t\tnegPred = tf.slice(preds, [sampNum], [-1])\n","\t\t\t\tself.preLoss += tf.reduce_mean(tf.maximum(0.0, 1.0 - (posPred - negPred)))\n","\t\t\t\tif src == args.behNum and tgt == args.behNum - 1:\n","\t\t\t\t\tself.targetPreds = preds\n","\t\tself.regLoss = args.reg * Regularize()\n","\t\tself.loss = self.preLoss + self.regLoss\n","\n","\t\tglobalStep = tf.Variable(0, trainable=False)\n","\t\tlearningRate = tf.train.exponential_decay(args.lr, globalStep, args.decay_step, args.decay, staircase=True)\n","\t\tself.optimizer = tf.train.AdamOptimizer(learningRate).minimize(self.loss, global_step=globalStep)\n","\n","\tdef sampleTrainBatch(self, batIds, labelMat):\n","\t\ttemLabel = labelMat[batIds].toarray()\n","\t\tbatch = len(batIds)\n","\t\ttemlen = batch * 2 * args.sampNum\n","\t\tuLocs = [None] * temlen\n","\t\tiLocs = [None] * temlen\n","\t\tcur = 0\n","\t\tfor i in range(batch):\n","\t\t\tposset = np.reshape(np.argwhere(temLabel[i]!=0), [-1])\n","\t\t\tsampNum = min(args.sampNum, len(posset))\n","\t\t\tif sampNum == 0:\n","\t\t\t\tposlocs = [np.random.choice(args.item)]\n","\t\t\t\tneglocs = [poslocs[0]]\n","\t\t\telse:\n","\t\t\t\tposlocs = np.random.choice(posset, sampNum)\n","\t\t\t\tneglocs = negSamp(temLabel[i], sampNum, args.item)\n","\t\t\tfor j in range(sampNum):\n","\t\t\t\tposloc = poslocs[j]\n","\t\t\t\tnegloc = neglocs[j]\n","\t\t\t\tuLocs[cur] = uLocs[cur+temlen//2] = batIds[i]\n","\t\t\t\tiLocs[cur] = posloc\n","\t\t\t\tiLocs[cur+temlen//2] = negloc\n","\t\t\t\tcur += 1\n","\t\tuLocs = uLocs[:cur] + uLocs[temlen//2: temlen//2 + cur]\n","\t\tiLocs = iLocs[:cur] + iLocs[temlen//2: temlen//2 + cur]\n","\t\treturn uLocs, iLocs\n","\n","\tdef trainEpoch(self):\n","\t\tnum = args.user\n","\t\tsfIds = np.random.permutation(num)[:args.trnNum]\n","\t\tepochLoss, epochPreLoss = [0] * 2\n","\t\tnum = len(sfIds)\n","\t\tsteps = int(np.ceil(num / args.batch))\n","\n","\t\tfor i in range(steps):\n","\t\t\tst = i * args.batch\n","\t\t\ted = min((i+1) * args.batch, num)\n","\t\t\tbatIds = sfIds[st: ed]\n","\n","\t\t\ttarget = [self.optimizer, self.preLoss, self.regLoss, self.loss]\n","\t\t\tfeed_dict = {}\n","\t\t\tfor beh in range(args.behNum):\n","\t\t\t\tuLocs, iLocs = self.sampleTrainBatch(batIds, self.handler.trnMats[beh])\n","\t\t\t\tfeed_dict[self.uids[beh]] = uLocs\n","\t\t\t\tfeed_dict[self.iids[beh]] = iLocs\n","\n","\t\t\tres = self.sess.run(target, feed_dict=feed_dict, options=config_pb2.RunOptions(report_tensor_allocations_upon_oom=True))\n","\n","\t\t\tpreLoss, regLoss, loss = res[1:]\n","\n","\t\t\tepochLoss += loss\n","\t\t\tepochPreLoss += preLoss\n","\t\t\tlog('Step %d/%d: loss = %.2f, regLoss = %.2f ' % (i, steps, loss, regLoss), save=False, oneline=True)\n","\t\tret = dict()\n","\t\tret['Loss'] = epochLoss / steps\n","\t\tret['preLoss'] = epochPreLoss / steps\n","\t\treturn ret\n","\n","\tdef sampleTestBatch(self, batIds, labelMat):\n","\t\tbatch = len(batIds)\n","\t\ttemTst = self.handler.tstInt[batIds]\n","\t\ttemLabel = labelMat[batIds].toarray()\n","\t\ttemlen = batch * 100\n","\t\tuLocs = [None] * temlen\n","\t\tiLocs = [None] * temlen\n","\t\ttstLocs = [None] * batch\n","\t\tcur = 0\n","\t\tfor i in range(batch):\n","\t\t\tposloc = temTst[i]\n","\t\t\tnegset = np.reshape(np.argwhere(temLabel[i]==0), [-1])\n","\t\t\trdnNegSet = np.random.permutation(negset)[:99]\n","\t\t\tlocset = np.concatenate((rdnNegSet, np.array([posloc])))\n","\t\t\ttstLocs[i] = locset\n","\t\t\tfor j in range(100):\n","\t\t\t\tuLocs[cur] = batIds[i]\n","\t\t\t\tiLocs[cur] = locset[j]\n","\t\t\t\tcur += 1\n","\t\treturn uLocs, iLocs, temTst, tstLocs\n","\n","\tdef testEpoch(self):\n","\t\tepochHit, epochNdcg = [0] * 2\n","\t\tids = self.handler.tstUsrs\n","\t\tnum = len(ids)\n","\t\ttstBat = args.batch\n","\t\tsteps = int(np.ceil(num / tstBat))\n","\t\tfor i in range(steps):\n","\t\t\tst = i * tstBat\n","\t\t\ted = min((i+1) * tstBat, num)\n","\t\t\tbatIds = ids[st: ed]\n","\t\t\tfeed_dict = {}\n","\t\t\tuLocs, iLocs, temTst, tstLocs = self.sampleTestBatch(batIds, self.handler.trnMats[-1])\n","\t\t\tfeed_dict[self.uids[-1]] = uLocs\n","\t\t\tfeed_dict[self.iids[-1]] = iLocs\n","\t\t\tpreds = self.sess.run(self.targetPreds, feed_dict=feed_dict, options=config_pb2.RunOptions(report_tensor_allocations_upon_oom=True))\n","\t\t\thit, ndcg = self.calcRes(np.reshape(preds, [ed-st, 100]), temTst, tstLocs)\n","\t\t\tepochHit += hit\n","\t\t\tepochNdcg += ndcg\n","\t\t\tlog('Steps %d/%d: hit = %d, ndcg = %d ' % (i, steps, hit, ndcg), save=False, oneline=True)\n","\t\tret = dict()\n","\t\tret['HR'] = epochHit / num\n","\t\tret['NDCG'] = epochNdcg / num\n","\t\treturn ret\n","\n","\tdef calcRes(self, preds, temTst, tstLocs):\n","\t\thit = 0\n","\t\tndcg = 0\n","\t\tfor j in range(preds.shape[0]):\n","\t\t\tpredvals = list(zip(preds[j], tstLocs[j]))\n","\t\t\tpredvals.sort(key=lambda x: x[0], reverse=True)\n","\t\t\tshoot = list(map(lambda x: x[1], predvals[:args.shoot]))\n","\t\t\tif temTst[j] in shoot:\n","\t\t\t\thit += 1\n","\t\t\t\tndcg += np.reciprocal(np.log2(shoot.index(temTst[j])+2))\n","\t\treturn hit, ndcg\n","\t\n","\tdef saveHistory(self):\n","\t\tif args.epoch == 0:\n","\t\t\treturn\n","\t\twith open('History/' + args.save_path + '.his', 'wb') as fs:\n","\t\t\tpickle.dump(self.metrics, fs)\n","\n","\t\tsaver = tf.train.Saver()\n","\t\tsaver.save(self.sess, 'Models/' + args.save_path)\n","\t\tlog('Model Saved: %s' % args.save_path)\n","\n","\tdef loadModel(self):\n","\t\tsaver = tf.train.Saver()\n","\t\tsaver.restore(sess, 'Models/' + args.load_model)\n","\t\twith open('History/' + args.load_model + '.his', 'rb') as fs:\n","\t\t\tself.metrics = pickle.load(fs)\n","\t\tlog('Model Loaded')\t"],"metadata":{"id":"TNDAiXW0tb-l"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Training & Evaluation"],"metadata":{"id":"VV1cEgWJ2q9L"}},{"cell_type":"code","source":["if __name__ == '__main__':\n","\tsaveDefault = True\n","\tconfig = tf.ConfigProto()\n","\tconfig.gpu_options.allow_growth = True\n","\n","\tlog('Start')\n","\thandler = DataHandler()\n","\thandler.LoadData()\n","\tlog('Load Data')\n","\n","\twith tf.Session(config=config) as sess:\n","\t\trecom = Recommender(sess, handler)\n","\t\trecom.run()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":729,"referenced_widgets":["24c87c2912974b6d8becb5b9190aa829","4314716903e64982b95392e7e37156bd","6f654bcada5d41f78b08c66d2e692fde","9dd6ba2d32ff4c0cb76b467e51e8060c","7fd736fa68424784a086f63ba9a43657","0bbb945cc6d54d289cbf67827ae1b723","551b14c6f28644eea7c40845274e672c","02be78ca98ab4b39adaa14cfd532ce00","0343c7f1998946c68782b6fa7993b824","439b1cc5861d446b84eba9b023d4d410","1ceba6676a0f4736961546da9a54dac6"]},"id":"j9AC1tZouQWn","executionInfo":{"status":"ok","timestamp":1639298905239,"user_tz":-330,"elapsed":669639,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"051d89cd-13b8-45fe-ac02-ba61e5b5ccf8"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["2021-12-12 08:37:23.971121: Start\n","2021-12-12 08:37:27.161611: Load Data\n","USER 21716 ITEM 7977\n","WARNING:tensorflow:From /tensorflow-1.15.2/python3.7/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.where in 2.0, which has the same broadcast rule as np.where\n","2021-12-12 08:38:00.972851: Model Prepared\n","2021-12-12 08:38:01.834669: Variables Initiated\n"]},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"24c87c2912974b6d8becb5b9190aa829","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/10 [00:00] 2.57K --.-KB/s in 0s \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dco-CyHmKTAb","executionInfo":{"status":"ok","timestamp":1630047507321,"user_tz":-330,"elapsed":737,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"5e012369-a54c-4841-9897-d1b7f1892473"},"source":["!cat code/cloudformation/immersion_day.yaml"],"execution_count":null,"outputs":[{"output_type":"stream","text":["---\n","AWSTemplateFormatVersion: '2010-09-09'\n","\n","Description: Creates an S3 Bucket, IAM Policies, and SageMaker Notebook to work with Personalize.\n","\n","Parameters:\n"," NotebookName:\n"," Type: String\n"," Default: AmazonPersonalizeImmersionDay\n"," Description: Enter the name of the SageMaker notebook instance. Default is PersonalizeImmersionDay.\n","\n"," VolumeSize:\n"," Type: Number\n"," Default: 64\n"," MinValue: 5\n"," MaxValue: 16384\n"," ConstraintDescription: Must be an integer between 5 (GB) and 16384 (16 TB).\n"," Description: Enter the size of the EBS volume in GB.\n"," \n"," domain:\n"," Type: String\n"," Default: Media\n"," Description: Enter the name of the domain (Retail, Media, or CPG) you would like to use in your Amazon Personalize Immersion Day.\n","\n","\n","Resources:\n"," SAMArtifactsBucket:\n"," Type: AWS::S3::Bucket\n"," # SageMaker Execution Role\n"," SageMakerIamRole:\n"," Type: \"AWS::IAM::Role\"\n"," Properties:\n"," AssumeRolePolicyDocument:\n"," Version: \"2012-10-17\"\n"," Statement:\n"," -\n"," Effect: Allow\n"," Principal:\n"," Service: sagemaker.amazonaws.com\n"," Action: sts:AssumeRole\n"," Path: \"/\"\n"," ManagedPolicyArns:\n"," - \"arn:aws:iam::aws:policy/IAMFullAccess\"\n"," - \"arn:aws:iam::aws:policy/AWSCloudFormationFullAccess\"\n"," - \"arn:aws:iam::aws:policy/AmazonS3FullAccess\"\n"," - \"arn:aws:iam::aws:policy/AmazonSageMakerFullAccess\"\n"," - \"arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess\"\n"," - \"arn:aws:iam::aws:policy/AWSLambda_FullAccess\"\n"," - \"arn:aws:iam::aws:policy/AmazonSNSFullAccess\"\n"," - \"arn:aws:iam::aws:policy/service-role/AmazonPersonalizeFullAccess\"\n"," \n"," \n","\n"," # SageMaker notebook\n"," NotebookInstance:\n"," Type: \"AWS::SageMaker::NotebookInstance\"\n"," Properties:\n"," InstanceType: \"ml.t2.medium\"\n"," NotebookInstanceName: !Ref NotebookName\n"," RoleArn: !GetAtt SageMakerIamRole.Arn\n"," VolumeSizeInGB: !Ref VolumeSize\n"," LifecycleConfigName: !GetAtt AmazonPersonalizeMLOpsLifecycleConfig.NotebookInstanceLifecycleConfigName\n","\n","\n"," AmazonPersonalizeMLOpsLifecycleConfig:\n"," Type: \"AWS::SageMaker::NotebookInstanceLifecycleConfig\"\n"," Properties:\n"," OnStart:\n"," - Content:\n"," Fn::Base64: \n"," !Sub |\n"," #!/bin/bash\n"," sudo -u ec2-user -i <<'EOF'\n"," cd /home/ec2-user/SageMaker/\n"," git clone https://github.com/aws-samples/amazon-personalize-immersion-day.git\n"," cd /home/ec2-user/SageMaker/amazon-personalize-immersion-day/automation/ml_ops/\n"," nohup sh deploy.sh \"${SAMArtifactsBucket}\" \"${domain}\" &\n"," EOF"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"am9RumpUcLWx"},"source":["## Data Preparation"]},{"cell_type":"code","metadata":{"id":"MVMVDeLWKYzk"},"source":["import time\n","from time import sleep\n","import json\n","from datetime import datetime\n","import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2UVWuFVVaF81","executionInfo":{"status":"ok","timestamp":1630047692472,"user_tz":-330,"elapsed":437,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"111803e5-aa5d-4010-965e-a041645eead2"},"source":["original_data = pd.read_csv('./data/bronze/ml-latest-small/ratings.csv')\n","original_data.info()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["\n","RangeIndex: 100836 entries, 0 to 100835\n","Data columns (total 4 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 userId 100836 non-null int64 \n"," 1 movieId 100836 non-null int64 \n"," 2 rating 100836 non-null float64\n"," 3 timestamp 100836 non-null int64 \n","dtypes: float64(1), int64(3)\n","memory usage: 3.1 MB\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"GLJtDOC7aZmV"},"source":["The int64 format is clearly suitable for userId and movieId. However, we need to dive deeper to understand the timestamps in the data. To use Amazon Personalize, you need to save timestamps in Unix Epoch format. Currently, the timestamp values are not human-readable. So let's grab an arbitrary timestamp value and figure out how to interpret it. Do a quick sanity check on the transformed dataset by picking an arbitrary timestamp and transforming it to a human-readable format."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5KhFzjnHaoIV","executionInfo":{"status":"ok","timestamp":1630047743488,"user_tz":-330,"elapsed":461,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"f2966174-a31b-4484-e4bd-edce90c03721"},"source":["arb_time_stamp = original_data.iloc[50]['timestamp']\n","print(arb_time_stamp)\n","print(datetime.utcfromtimestamp(arb_time_stamp).strftime('%Y-%m-%d %H:%M:%S'))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["964982681.0\n","2000-07-30 18:44:41\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"LfF4C69ea9CK"},"source":["Since this is a dataset of an explicit feedback movie ratings, it includes movies rated from 1 to 5. We want to include only moves that were \"liked\" by the users, and simulate a dataset of data that would be gathered by a VOD platform. In order to do that, we will filter out all interactions under 2 out of 5, and create two EVENT_Types \"click\" and and \"watch\". We will then assign all movies rated 2 and above as \"click\" and movies rated 4 and above as both \"click\" and \"watch\".\n","\n","Note that this is to correspond with the events we are modeling, for a real data set you would actually model based on implicit feedback such as clicks, watches and/or explicit feedback such as ratings, likes etc."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":578},"id":"3TmlOUgNa-Sr","executionInfo":{"status":"ok","timestamp":1630047929545,"user_tz":-330,"elapsed":613,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"69a4ab49-f41c-4db4-b4d3-25ad8e3dc7ae"},"source":["watched_df = original_data.copy()\n","watched_df = watched_df[watched_df['rating'] > 3]\n","watched_df = watched_df[['userId', 'movieId', 'timestamp']]\n","watched_df['EVENT_TYPE']='watch'\n","display(watched_df.head())\n","\n","clicked_df = original_data.copy()\n","clicked_df = clicked_df[clicked_df['rating'] > 1]\n","clicked_df = clicked_df[['userId', 'movieId', 'timestamp']]\n","clicked_df['EVENT_TYPE']='click'\n","display(clicked_df.head())\n","\n","interactions_df = clicked_df.copy()\n","interactions_df = interactions_df.append(watched_df)\n","interactions_df.sort_values(\"timestamp\", axis = 0, ascending = True, \n"," inplace = True, na_position ='last') \n","interactions_df.info()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdtimestampEVENT_TYPE
011964982703watch
113964981247watch
216964982224watch
3147964983815watch
4150964982931watch
\n","
"],"text/plain":[" userId movieId timestamp EVENT_TYPE\n","0 1 1 964982703 watch\n","1 1 3 964981247 watch\n","2 1 6 964982224 watch\n","3 1 47 964983815 watch\n","4 1 50 964982931 watch"]},"metadata":{}},{"output_type":"display_data","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
userIdmovieIdtimestampEVENT_TYPE
011964982703click
113964981247click
216964982224click
3147964983815click
4150964982931click
\n","
"],"text/plain":[" userId movieId timestamp EVENT_TYPE\n","0 1 1 964982703 click\n","1 1 3 964981247 click\n","2 1 6 964982224 click\n","3 1 47 964983815 click\n","4 1 50 964982931 click"]},"metadata":{}},{"output_type":"stream","text":["\n","Int64Index: 158371 entries, 66679 to 81092\n","Data columns (total 4 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 userId 158371 non-null int64 \n"," 1 movieId 158371 non-null int64 \n"," 2 timestamp 158371 non-null int64 \n"," 3 EVENT_TYPE 158371 non-null object\n","dtypes: int64(3), object(1)\n","memory usage: 6.0+ MB\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"NXV-FREFbX4_"},"source":["Amazon Personalize has default column names for users, items, and timestamp. These default column names are USER_ID, ITEM_ID, AND TIMESTAMP. So the final modification to the dataset is to replace the existing column headers with the default headers."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"JuhWwOV7bbnJ","executionInfo":{"status":"ok","timestamp":1630048048895,"user_tz":-330,"elapsed":545,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"f1f62fd8-566c-4333-a124-df2150d74259"},"source":["interactions_df.rename(columns = {'userId':'USER_ID', 'movieId':'ITEM_ID', \n"," 'timestamp':'TIMESTAMP'}, inplace = True)\n","interactions_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
USER_IDITEM_IDTIMESTAMPEVENT_TYPE
66679429222828124615watch
66681429227828124615click
66719429595828124615watch
66718429592828124615watch
66717429590828124615watch
\n","
"],"text/plain":[" USER_ID ITEM_ID TIMESTAMP EVENT_TYPE\n","66679 429 222 828124615 watch\n","66681 429 227 828124615 click\n","66719 429 595 828124615 watch\n","66718 429 592 828124615 watch\n","66717 429 590 828124615 watch"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","metadata":{"id":"X_bHOc_0b1HK"},"source":["interactions_df.to_csv('./data/silver/ml-latest-small/interactions.csv', index=False, float_format='%.0f')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"H03qEFm25Otd","executionInfo":{"status":"ok","timestamp":1630055796089,"user_tz":-330,"elapsed":537,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"a4c83035-39c1-4159-b564-9c65bab224c2"},"source":["original_data = pd.read_csv('./data/bronze/ml-latest-small/movies.csv')\n","original_data.info()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["\n","RangeIndex: 9742 entries, 0 to 9741\n","Data columns (total 3 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 movieId 9742 non-null int64 \n"," 1 title 9742 non-null object\n"," 2 genres 9742 non-null object\n","dtypes: int64(1), object(2)\n","memory usage: 228.5+ KB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"GR6mdTge5Yzz","executionInfo":{"status":"ok","timestamp":1630055932177,"user_tz":-330,"elapsed":622,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"0891fc8c-67cc-45c5-92a1-21fafab26734"},"source":["original_data['year'] =original_data['title'].str.extract('.*\\((.*)\\).*',expand = False)\n","original_data = original_data.dropna(axis=0)\n","\n","itemmetadata_df = original_data.copy()\n","itemmetadata_df = itemmetadata_df[['movieId', 'genres', 'year']]\n","itemmetadata_df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
movieIdgenresyear
01Adventure|Animation|Children|Comedy|Fantasy1995
12Adventure|Children|Fantasy1995
23Comedy|Romance1995
34Comedy|Drama|Romance1995
45Comedy1995
\n","
"],"text/plain":[" movieId genres year\n","0 1 Adventure|Animation|Children|Comedy|Fantasy 1995\n","1 2 Adventure|Children|Fantasy 1995\n","2 3 Comedy|Romance 1995\n","3 4 Comedy|Drama|Romance 1995\n","4 5 Comedy 1995"]},"metadata":{},"execution_count":24}]},{"cell_type":"markdown","metadata":{"id":"XwgGUdk654pB"},"source":["We will add a new dataframe to help us generate a creation timestamp. If you don’t provide the CREATION_TIMESTAMP for an item, the model infers this information from the interaction dataset and uses the timestamp of the item’s earliest interaction as its corresponding release date. If an item doesn’t have an interaction, its release date is set as the timestamp of the latest interaction in the training set and it is considered a new item. For the current dataset we will set the CREATION_TIMESTAMP to 0."]},{"cell_type":"code","metadata":{"id":"pHUtEDT2522e"},"source":["itemmetadata_df['CREATION_TIMESTAMP'] = 0\n","itemmetadata_df.rename(columns = {'genres':'GENRE', 'movieId':'ITEM_ID', 'year':'YEAR'}, inplace = True) \n","itemmetadata_df.to_csv('./data/silver/ml-latest-small/item-meta.csv', index=False, float_format='%.0f')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"eBy5V4Z4b2rC"},"source":["## AWS Personalize"]},{"cell_type":"code","metadata":{"id":"C4xjNqJZcw2a"},"source":["!pip install -q boto3\n","import boto3\n","import json\n","import time"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1oZPIuJFcXN1"},"source":["!mkdir -p ~/.aws && cp /content/drive/MyDrive/AWS/d01_admin/* ~/.aws"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"XeBuVJpGFsvb"},"source":["### ETL Job for Interactions data without using generic data loading module"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iwv-AGiWcJeD","executionInfo":{"status":"ok","timestamp":1630048308723,"user_tz":-330,"elapsed":402,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"8c08cc2d-607c-4ed2-8077-ae0cb909bce9"},"source":["# Configure the SDK to Personalize:\n","personalize = boto3.client('personalize')\n","personalize_runtime = boto3.client('personalize-runtime')\n","print(\"We can communicate with Personalize!\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["We can communicate with Personalize!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XkvOHA9ack0x"},"source":["# create the dataset group (the highest level of abstraction)\n","create_dataset_group_response = personalize.create_dataset_group(\n"," name = \"immersion-day-dataset-group-movielens-latest\"\n",")\n","\n","dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n","print(json.dumps(create_dataset_group_response, indent=2))\n","\n","# wait for it to become active\n","max_time = time.time() + 3*60*60 # 3 hours\n","while time.time() < max_time:\n"," describe_dataset_group_response = personalize.describe_dataset_group(\n"," datasetGroupArn = dataset_group_arn\n"," )\n"," status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n"," print(\"DatasetGroup: {}\".format(status))\n"," \n"," if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n"," break\n"," \n"," time.sleep(60)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"k-6qvw4oflsH"},"source":["interactions_schema = schema = {\n"," \"type\": \"record\",\n"," \"name\": \"Interactions\",\n"," \"namespace\": \"com.amazonaws.personalize.schema\",\n"," \"fields\": [\n"," {\n"," \"name\": \"USER_ID\",\n"," \"type\": \"string\"\n"," },\n"," {\n"," \"name\": \"ITEM_ID\",\n"," \"type\": \"string\"\n"," },\n"," {\n"," \"name\": \"EVENT_TYPE\",\n"," \"type\": \"string\"\n"," },\n"," {\n"," \"name\": \"TIMESTAMP\",\n"," \"type\": \"long\"\n"," }\n"," ],\n"," \"version\": \"1.0\"\n","}\n","\n","create_schema_response = personalize.create_schema(\n"," name = \"personalize-poc-movielens-interactions\",\n"," schema = json.dumps(interactions_schema)\n",")\n","\n","interaction_schema_arn = create_schema_response['schemaArn']\n","print(json.dumps(create_schema_response, indent=2))\n","\n","dataset_type = \"INTERACTIONS\"\n","create_dataset_response = personalize.create_dataset(\n"," name = \"personalize-poc-movielens-ints\",\n"," datasetType = dataset_type,\n"," datasetGroupArn = dataset_group_arn,\n"," schemaArn = interaction_schema_arn\n",")\n","\n","interactions_dataset_arn = create_dataset_response['datasetArn']\n","print(json.dumps(create_dataset_response, indent=2))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8fN7IWbagYkT","executionInfo":{"status":"ok","timestamp":1630049380307,"user_tz":-330,"elapsed":472,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"998ba781-2064-492f-9ff0-9bbdc8e1aef3"},"source":["region = 'us-east-1'\n","s3 = boto3.client('s3')\n","account_id = boto3.client('sts').get_caller_identity().get('Account')\n","bucket_name = account_id + \"-\" + region + \"-\" + \"personalizepocvod\"\n","print(bucket_name)\n","if region == \"us-east-1\":\n"," s3.create_bucket(Bucket=bucket_name)\n","else:\n"," s3.create_bucket(\n"," Bucket=bucket_name,\n"," CreateBucketConfiguration={'LocationConstraint': region}\n"," )"],"execution_count":null,"outputs":[{"output_type":"stream","text":["746888961694-us-east-1-personalizepocvod\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SQsQs56hg6I2"},"source":["interactions_file_path = './data/silver/ml-latest-small/interactions.csv'\n","interactions_filename = 'interactions.csv'\n","boto3.Session().resource('s3').Bucket(bucket_name).Object(interactions_filename).upload_file(interactions_file_path)\n","interactions_s3DataPath = \"s3://\"+bucket_name+\"/\"+interactions_filename"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uj3Ua9zOhfFW"},"source":["policy = {\n"," \"Version\": \"2012-10-17\",\n"," \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n"," \"Statement\": [\n"," {\n"," \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n"," \"Effect\": \"Allow\",\n"," \"Principal\": {\n"," \"Service\": \"personalize.amazonaws.com\"\n"," },\n"," \"Action\": [\n"," \"s3:*Object\",\n"," \"s3:ListBucket\"\n"," ],\n"," \"Resource\": [\n"," \"arn:aws:s3:::{}\".format(bucket_name),\n"," \"arn:aws:s3:::{}/*\".format(bucket_name)\n"," ]\n"," }\n"," ]\n","}\n","\n","s3.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(policy))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"jI5D1AlViHAZ","executionInfo":{"status":"ok","timestamp":1630049797032,"user_tz":-330,"elapsed":60956,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"69e83310-bb91-4ed2-d439-a5783e33a56a"},"source":["iam = boto3.client(\"iam\")\n","\n","role_name = \"PersonalizeRolePOC\"\n","assume_role_policy_document = {\n"," \"Version\": \"2012-10-17\",\n"," \"Statement\": [\n"," {\n"," \"Effect\": \"Allow\",\n"," \"Principal\": {\n"," \"Service\": \"personalize.amazonaws.com\"\n"," },\n"," \"Action\": \"sts:AssumeRole\"\n"," }\n"," ]\n","}\n","\n","create_role_response = iam.create_role(\n"," RoleName = role_name,\n"," AssumeRolePolicyDocument = json.dumps(assume_role_policy_document)\n",")\n","\n","# AmazonPersonalizeFullAccess provides access to any S3 bucket with a name that includes \"personalize\" or \"Personalize\" \n","# if you would like to use a bucket with a different name, please consider creating and attaching a new policy\n","# that provides read access to your bucket or attaching the AmazonS3ReadOnlyAccess policy to the role\n","policy_arn = \"arn:aws:iam::aws:policy/service-role/AmazonPersonalizeFullAccess\"\n","iam.attach_role_policy(\n"," RoleName = role_name,\n"," PolicyArn = policy_arn\n",")\n","\n","# Now add S3 support\n","iam.attach_role_policy(\n"," PolicyArn='arn:aws:iam::aws:policy/AmazonS3FullAccess',\n"," RoleName=role_name\n",")\n","time.sleep(60) # wait for a minute to allow IAM role policy attachment to propagate\n","\n","role_arn = create_role_response[\"Role\"][\"Arn\"]\n","print(role_arn)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["arn:aws:iam::746888961694:role/PersonalizeRolePOC\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"asaTiB0oiRGr"},"source":["create_dataset_import_job_response = personalize.create_dataset_import_job(\n"," jobName = \"personalize-poc-import1\",\n"," datasetArn = interactions_dataset_arn,\n"," dataSource = {\n"," \"dataLocation\": \"s3://{}/{}\".format(bucket_name, interactions_filename)\n"," },\n"," roleArn = role_arn\n",")\n","\n","dataset_import_job_arn = create_dataset_import_job_response['datasetImportJobArn']\n","print(json.dumps(create_dataset_import_job_response, indent=2))\n","\n","# wait fir this import job to gets activated\n","\n","max_time = time.time() + 6*60*60 # 6 hours\n","while time.time() < max_time:\n"," describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n"," datasetImportJobArn = dataset_import_job_arn\n"," )\n"," status = describe_dataset_import_job_response[\"datasetImportJob\"]['status']\n"," print(\"DatasetImportJob: {}\".format(status))\n"," \n"," if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n"," break\n"," \n"," time.sleep(60)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"7jPFKY3qFHwh"},"source":["### ETL Job for Item meta using generic data loading module"]},{"cell_type":"code","metadata":{"id":"Y5zv_4760056"},"source":["import sys\n","sys.path.insert(0,'./code')\n","\n","from generic_modules.import_dataset import personalize_dataset"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"91OcwOQm9Ai6"},"source":["dataset_group_arn = 'arn:aws:personalize:us-east-1:746888961694:dataset-group/immersion-day-dataset-group-movielens-latest'\n","bucket_name = '746888961694-us-east-1-personalizepocvod'\n","role_arn = 'arn:aws:iam::746888961694:role/PersonalizeRolePOC'\n","\n","dataset_type = 'ITEMS'\n","source_data_path = './data/silver/ml-latest-small/item-meta.csv'\n","target_file_name = 'item-meta.csv'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rIv3OWn18qVV"},"source":["personalize_item_meta = personalize_dataset(\n"," dataset_group_arn = dataset_group_arn,\n"," bucket_name = bucket_name,\n"," role_arn = role_arn,\n"," dataset_type = dataset_type,\n"," source_data_path = source_data_path,\n"," target_file_name = target_file_name,\n"," dataset_arn = dataset_arn,\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pVwE0fkL-b51","executionInfo":{"status":"ok","timestamp":1630057962385,"user_tz":-330,"elapsed":7,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"369468f7-275c-45f1-8794-d503260db547"},"source":["personalize_item_meta.setup_connection()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["SUCCESS | We can communicate with Personalize!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2OaRUWs8-hAE"},"source":["itemmetadata_schema = {\n"," \"type\": \"record\",\n"," \"name\": \"Items\",\n"," \"namespace\": \"com.amazonaws.personalize.schema\",\n"," \"fields\": [\n"," {\n"," \"name\": \"ITEM_ID\",\n"," \"type\": \"string\"\n"," },\n"," {\n"," \"name\": \"GENRE\",\n"," \"type\": \"string\",\n"," \"categorical\": True\n"," },{\n"," \"name\": \"YEAR\",\n"," \"type\": \"int\",\n"," },\n"," {\n"," \"name\": \"CREATION_TIMESTAMP\",\n"," \"type\": \"long\",\n"," }\n"," ],\n"," \"version\": \"1.0\"\n","}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ELGoa3ng-uD1"},"source":["personalize_item_meta.create_dataset(schema=itemmetadata_schema,\n"," schema_name='personalize-poc-movielens-item',\n"," dataset_name='personalize-poc-movielens-items')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"BrwOj2VpCemy","executionInfo":{"status":"ok","timestamp":1630058213530,"user_tz":-330,"elapsed":635,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"bd706745-e758-43d2-ac2e-4b44e47baff7"},"source":["personalize_item_meta.dataset_arn"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'arn:aws:personalize:us-east-1:746888961694:dataset/immersion-day-dataset-group-movielens-latest/ITEMS'"]},"metadata":{},"execution_count":55}]},{"cell_type":"code","metadata":{"id":"9ZksZbihBmBz"},"source":["personalize_item_meta.upload_data_to_s3()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0fYXJO3lB6EP"},"source":["personalize_item_meta.import_data_from_s3(import_job_name='personalize-poc-item-import1')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2M5MUMAeGpFN"},"source":["\n","import boto3\n","import json\n","import time\n","\n","\n","class personalize_dataset:\n"," def __init__(self,\n"," dataset_group_arn=None,\n"," schema_arn=None,\n"," dataset_arn=None,\n"," dataset_type='INTERACTIONS',\n"," region='us-east-1',\n"," bucket_name=None,\n"," role_arn=None,\n"," source_data_path=None,\n"," target_file_name=None,\n"," dataset_import_job_arn=None\n"," ):\n"," self.personalize = None\n"," self.personalize_runtime = None\n"," self.s3 = None\n"," self.iam = None\n"," self.dataset_group_arn = dataset_group_arn\n"," self.schema_arn = schema_arn\n"," self.dataset_arn = dataset_arn\n"," self.dataset_type = dataset_type\n"," self.region = region\n"," self.bucket_name = bucket_name\n"," self.role_arn = role_arn\n"," self.source_data_path = source_data_path\n"," self.target_file_name = target_file_name\n"," self.dataset_import_job_arn = dataset_import_job_arn\n","\n"," def setup_connection(self):\n"," try:\n"," self.personalize = boto3.client('personalize')\n"," self.personalize_runtime = boto3.client('personalize-runtime')\n"," self.s3 = boto3.client('s3')\n"," self.iam = boto3.client(\"iam\")\n"," print(\"SUCCESS | We can communicate with Personalize!\")\n"," except:\n"," print(\"ERROR | Connection can't be established!\")\n"," \n"," def create_dataset_group(self, dataset_group_name=None):\n"," \"\"\"\n"," The highest level of isolation and abstraction with Amazon Personalize\n"," is a dataset group. Information stored within one of these dataset groups\n"," has no impact on any other dataset group or models created from one. they\n"," are completely isolated. This allows you to run many experiments and is\n"," part of how we keep your models private and fully trained only on your data.\n"," \"\"\"\n"," create_dataset_group_response = self.personalize.create_dataset_group(name=dataset_group_name)\n"," self.dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n"," # print(json.dumps(create_dataset_group_response, indent=2))\n","\n"," # Before we can use the dataset group, it must be active. \n"," # This can take a minute or two. Execute the cell below and wait for it\n"," # to show the ACTIVE status. It checks the status of the dataset group\n"," # every minute, up to a maximum of 3 hours.\n"," max_time = time.time() + 3*60*60 # 3 hours\n"," while time.time() < max_time:\n"," status = self.check_dataset_group_status()\n"," print(\"DatasetGroup: {}\".format(status))\n"," if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n"," break\n"," time.sleep(60)\n","\n"," def check_dataset_group_status(self):\n"," \"\"\"\n"," Check the status of dataset group\n"," \"\"\"\n"," describe_dataset_group_response = self.personalize.describe_dataset_group(\n"," datasetGroupArn = self.dataset_group_arn\n"," )\n"," status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n"," return status\n","\n"," def create_dataset(self, schema=None, schema_name=None, dataset_name=None):\n"," \"\"\"\n"," First, define a schema to tell Amazon Personalize what type of dataset\n"," you are uploading. There are several reserved and mandatory keywords\n"," required in the schema, based on the type of dataset. More detailed\n"," information can be found in the documentation.\n"," \"\"\"\n"," create_schema_response = self.personalize.create_schema(\n"," name = schema_name,\n"," schema = json.dumps(schema)\n"," )\n"," self.schema_arn = create_schema_response['schemaArn']\n","\n"," \"\"\"\n"," With a schema created, you can create a dataset within the dataset group.\n"," Note that this does not load the data yet, it just defines the schema for\n"," the data. The data will be loaded a few steps later.\n"," \"\"\"\n"," create_dataset_response = self.personalize.create_dataset(\n"," name = dataset_name,\n"," datasetType = self.dataset_type,\n"," datasetGroupArn = self.dataset_group_arn,\n"," schemaArn = self.schema_arn\n"," )\n"," self.dataset_arn = create_dataset_response['datasetArn']\n"," \n"," def create_s3_bucket(self):\n"," if region == \"us-east-1\":\n"," self.s3.create_bucket(Bucket=self.bucket_name)\n"," else:\n"," self.s3.create_bucket(\n"," Bucket=self.bucket_name,\n"," CreateBucketConfiguration={'LocationConstraint': self.region}\n"," )\n"," \n"," def upload_data_to_s3(self):\n"," \"\"\"\n"," Now that your Amazon S3 bucket has been created, upload the CSV file of\n"," our user-item-interaction data.\n"," \"\"\"\n"," boto3.Session().resource('s3').Bucket(self.bucket_name).Object(self.target_file_name).upload_file(self.source_data_path)\n"," s3DataPath = \"s3://\"+self.bucket_name+\"/\"+self.target_file_name\n"," \n"," def set_s3_bucket_policy(self, policy=None):\n"," \"\"\"\n"," Amazon Personalize needs to be able to read the contents of your S3\n"," bucket. So add a bucket policy which allows that.\n"," \"\"\"\n"," if not policy:\n"," policy = {\n"," \"Version\": \"2012-10-17\",\n"," \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n"," \"Statement\": [\n"," {\n"," \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n"," \"Effect\": \"Allow\",\n"," \"Principal\": {\n"," \"Service\": \"personalize.amazonaws.com\"\n"," },\n"," \"Action\": [\n"," \"s3:*Object\",\n"," \"s3:ListBucket\"\n"," ],\n"," \"Resource\": [\n"," \"arn:aws:s3:::{}\".format(self.bucket_name),\n"," \"arn:aws:s3:::{}/*\".format(self.bucket_name)\n"," ]\n"," }\n"," ]\n"," }\n","\n"," self.s3.put_bucket_policy(Bucket=self.bucket_name, Policy=json.dumps(policy))\n","\n"," def create_iam_role(self, role_name=None):\n"," \"\"\"\n"," Amazon Personalize needs the ability to assume roles in AWS in order to\n"," have the permissions to execute certain tasks. Let's create an IAM role\n"," and attach the required policies to it. The code below attaches very permissive\n"," policies; please use more restrictive policies for any production application.\n"," \"\"\"\n"," assume_role_policy_document = {\n"," \"Version\": \"2012-10-17\",\n"," \"Statement\": [\n"," {\n"," \"Effect\": \"Allow\",\n"," \"Principal\": {\n"," \"Service\": \"personalize.amazonaws.com\"\n"," },\n"," \"Action\": \"sts:AssumeRole\"\n"," }\n"," ]\n"," }\n"," create_role_response = self.iam.create_role(\n"," RoleName = role_name,\n"," AssumeRolePolicyDocument = json.dumps(assume_role_policy_document)\n"," )\n","\n"," # AmazonPersonalizeFullAccess provides access to any S3 bucket with a name that includes \"personalize\" or \"Personalize\" \n"," # if you would like to use a bucket with a different name, please consider creating and attaching a new policy\n"," # that provides read access to your bucket or attaching the AmazonS3ReadOnlyAccess policy to the role\n"," policy_arn = \"arn:aws:iam::aws:policy/service-role/AmazonPersonalizeFullAccess\"\n"," self.iam.attach_role_policy(\n"," RoleName = role_name,\n"," PolicyArn = policy_arn\n"," )\n"," # Now add S3 support\n"," self.iam.attach_role_policy(\n"," PolicyArn='arn:aws:iam::aws:policy/AmazonS3FullAccess',\n"," RoleName=role_name\n"," )\n"," time.sleep(60) # wait for a minute to allow IAM role policy attachment to propagate\n"," self.role_arn = create_role_response[\"Role\"][\"Arn\"]\n","\n"," def import_data_from_s3(self, import_job_name=None):\n"," \"\"\"\n"," Earlier you created the dataset group and dataset to house your information,\n"," so now you will execute an import job that will load the data from the S3\n"," bucket into the Amazon Personalize dataset.\n"," \"\"\"\n"," create_dataset_import_job_response = self.personalize.create_dataset_import_job(\n"," jobName = import_job_name,\n"," datasetArn = self.dataset_arn,\n"," dataSource = {\n"," \"dataLocation\": \"s3://{}/{}\".format(self.bucket_name, self.target_file_name)\n"," },\n"," roleArn = self.role_arn\n"," )\n"," self.dataset_import_job_arn = create_dataset_import_job_response['datasetImportJobArn']\n","\n"," \"\"\"\n"," Before we can use the dataset, the import job must be active. Execute the\n"," cell below and wait for it to show the ACTIVE status. It checks the status\n"," of the import job every minute, up to a maximum of 6 hours.\n"," Importing the data can take some time, depending on the size of the dataset.\n"," In this workshop, the data import job should take around 15 minutes.\n"," \"\"\"\n"," max_time = time.time() + 6*60*60 # 6 hours\n"," while time.time() < max_time:\n"," describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n"," datasetImportJobArn = dataset_import_job_arn\n"," )\n"," status = self.check_import_job_status()\n"," print(\"DatasetImportJob: {}\".format(status))\n"," if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n"," break\n"," time.sleep(60)\n"," \n"," def check_import_job_status(self):\n"," describe_dataset_import_job_response = self.personalize.describe_dataset_import_job(\n"," datasetImportJobArn = self.dataset_import_job_arn\n"," )\n"," status = describe_dataset_import_job_response[\"datasetImportJob\"]['status']\n"," return status\n","\n"," def __getstate__(self):\n"," attributes = self.__dict__.copy()\n"," del attributes['personalize']\n"," del attributes['personalize_runtime']\n"," del attributes['s3']\n"," del attributes['iam']\n"," return attributes"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fq131onaDHkM","executionInfo":{"status":"ok","timestamp":1630059835081,"user_tz":-330,"elapsed":613,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"c101b288-131e-457b-d2c5-20dcf755308c"},"source":["dataset_arn = 'arn:aws:personalize:us-east-1:746888961694:dataset/immersion-day-dataset-group-movielens-latest/ITEMS'\n","dataset_import_job_arn = 'arn:aws:personalize:us-east-1:746888961694:dataset-import-job/personalize-poc-item-import1'\n","\n","personalize_item_meta = personalize_dataset(\n"," dataset_group_arn = dataset_group_arn,\n"," bucket_name = bucket_name,\n"," role_arn = role_arn,\n"," dataset_type = dataset_type,\n"," source_data_path = source_data_path,\n"," target_file_name = target_file_name,\n"," dataset_arn = dataset_arn,\n"," dataset_import_job_arn = dataset_import_job_arn\n",")\n","\n","personalize_item_meta.setup_connection()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["SUCCESS | We can communicate with Personalize!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"Au1JEi8UDSI7","executionInfo":{"status":"ok","timestamp":1630059836609,"user_tz":-330,"elapsed":11,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"7f2a0e7b-f374-4eb7-d0d5-ed9502f8d361"},"source":["personalize_item_meta.check_import_job_status()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'ACTIVE'"]},"metadata":{},"execution_count":88}]},{"cell_type":"markdown","metadata":{"id":"-AU3W_1IEub6"},"source":["### Saving the state"]},{"cell_type":"code","metadata":{"id":"z1zTY8rOGWyz"},"source":["import pickle\n","\n","with open('./artifacts/etc/personalize_item_meta.pkl', 'wb') as outp:\n"," pickle.dump(personalize_item_meta, outp, pickle.HIGHEST_PROTOCOL)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"x_cde5DTHbS-","executionInfo":{"status":"ok","timestamp":1630059841764,"user_tz":-330,"elapsed":426,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"fe78a27b-cc40-42b4-b621-6860f8277727"},"source":["personalize_item_meta.__getstate__()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'bucket_name': '746888961694-us-east-1-personalizepocvod',\n"," 'dataset_arn': 'arn:aws:personalize:us-east-1:746888961694:dataset/immersion-day-dataset-group-movielens-latest/ITEMS',\n"," 'dataset_group_arn': 'arn:aws:personalize:us-east-1:746888961694:dataset-group/immersion-day-dataset-group-movielens-latest',\n"," 'dataset_import_job_arn': 'arn:aws:personalize:us-east-1:746888961694:dataset-import-job/personalize-poc-item-import1',\n"," 'dataset_type': 'ITEMS',\n"," 'region': 'us-east-1',\n"," 'role_arn': 'arn:aws:iam::746888961694:role/PersonalizeRolePOC',\n"," 'schema_arn': None,\n"," 'source_data_path': './data/silver/ml-latest-small/item-meta.csv',\n"," 'target_file_name': 'item-meta.csv'}"]},"metadata":{},"execution_count":89}]}]} \ No newline at end of file diff --git a/_notebooks/2022-01-12-olx-baselines.ipynb b/_notebooks/2022-01-12-olx-baselines.ipynb new file mode 100644 index 0000000..02fd5a1 --- /dev/null +++ b/_notebooks/2022-01-12-olx-baselines.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-12-olx-baselines.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P245068%20%7C%20OLX%20Job%20Recommendations%20using%20LightFM%2C%20SLIM%2C%20ALS%20and%20baseline%20models.ipynb","timestamp":1644607233984},{"file_id":"1jAKIE2Lz_IL3da8Weo1SbuO7aE9pZHjH","timestamp":1639120458418}],"collapsed_sections":[],"mount_file_id":"1EyYY4uJ79QidIPXef8QICcH1PBVHEY5o","authorship_tag":"ABX9TyOfrjrZU+pF+imFWxOboKdr"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"9968de8625a44cd5845470867cd1e547":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_cdfb05654b1b4039aba929e38b5e11ab","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a7dcc1acdc8c45edaf07863a3bfb1030","IPY_MODEL_7d7e8822c9ce44879d76a786bcd482ab","IPY_MODEL_46fafbbc6545402f891b745e1995a671"]}},"cdfb05654b1b4039aba929e38b5e11ab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a7dcc1acdc8c45edaf07863a3bfb1030":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_6f51152ee8a24d8d82d8b580d049f581","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ea5f5db35ee24faabf20a62cabc099a6"}},"7d7e8822c9ce44879d76a786bcd482ab":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_b2ec7053720e49319db46cc44bd5a023","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":6,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":6,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_32c31fca0efb4eea86fd74daeae46c68"}},"46fafbbc6545402f891b745e1995a671":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_e62cf418ba9b4278a008d24c69033fb0","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 6/6 [00:54<00:00, 9.20s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ffb1a52956204822a6d5481632c587f6"}},"6f51152ee8a24d8d82d8b580d049f581":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ea5f5db35ee24faabf20a62cabc099a6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b2ec7053720e49319db46cc44bd5a023":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"32c31fca0efb4eea86fd74daeae46c68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e62cf418ba9b4278a008d24c69033fb0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ffb1a52956204822a6d5481632c587f6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","source":["# OLX Job Recommendations using LightFM, SLIM, ALS and baseline models"],"metadata":{"id":"80C6Pqsz94QC"}},{"cell_type":"markdown","source":["## Process flow\n","\n","![](https://github.com/RecoHut-Stanzas/S883757/raw/main/images/process_flow.svg)"],"metadata":{"id":"BpEmYZalDStj"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"SZL5_NPA3agC"}},{"cell_type":"code","metadata":{"id":"ucqa6oooaGjm"},"source":["!pip install -q implicit\n","!pip install -q lightfm"],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!pip install -q -U kaggle\n","!pip install --upgrade --force-reinstall --no-deps kaggle\n","!mkdir ~/.kaggle\n","!cp /content/drive/MyDrive/kaggle.json ~/.kaggle/\n","!chmod 600 ~/.kaggle/kaggle.json\n","!kaggle datasets download -d olxdatascience/olx-jobs-interactions\n","!unzip /content/olx-jobs-interactions.zip"],"metadata":{"id":"xuNKchEC2L0Y"},"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ud_lPO0hsMqN"},"source":["import sys\n","import pandas as pd\n","import matplotlib\n","import matplotlib.pyplot as plt\n","from datetime import datetime\n","import random\n","\n","import os\n","from pathlib import Path\n","\n","from collections import defaultdict\n","\n","import numpy as np\n","from tqdm import tqdm\n","from scipy import sparse\n","import scipy.sparse as sparse\n","from sklearn.preprocessing import normalize\n","from sklearn.exceptions import ConvergenceWarning\n","from sklearn.linear_model import ElasticNet\n","from sklearn.utils._testing import ignore_warnings\n","\n","import implicit\n","from lightfm import LightFM\n","\n","import tracemalloc\n","from datetime import datetime\n","from time import time\n","\n","from functools import partial\n","import multiprocessing\n","from multiprocessing.pool import ThreadPool"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Data Loading and Sampling"],"metadata":{"id":"B1juGTmI3XU2"}},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"soGVvynisU9T","executionInfo":{"status":"ok","timestamp":1639131572816,"user_tz":-330,"elapsed":25732,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"cc0380c6-afae-4f79-b412-8b836dc67f75"},"source":["df = pd.read_csv('interactions.csv')\n","df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
useritemeventtimestamp
02790156865click1581465600
1124480115662click1581465600
21595095150click1581465600
3188861109981click1581465600
420734888746click1581465600
\n","
"],"text/plain":[" user item event timestamp\n","0 27901 56865 click 1581465600\n","1 124480 115662 click 1581465600\n","2 159509 5150 click 1581465600\n","3 188861 109981 click 1581465600\n","4 207348 88746 click 1581465600"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0fKfhZiXsZHJ","executionInfo":{"status":"ok","timestamp":1639131572820,"user_tz":-330,"elapsed":26,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"d10cecfd-a990-4c74-cd3b-a47e0ed16ab9"},"source":["df.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","RangeIndex: 65502201 entries, 0 to 65502200\n","Data columns (total 4 columns):\n"," # Column Dtype \n","--- ------ ----- \n"," 0 user int64 \n"," 1 item int64 \n"," 2 event object\n"," 3 timestamp int64 \n","dtypes: int64(3), object(1)\n","memory usage: 2.0+ GB\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"clRjRRBJshTB","executionInfo":{"status":"ok","timestamp":1639131642383,"user_tz":-330,"elapsed":69583,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"e4c66ba6-0f8f-4dea-b01c-49c17ce379a7"},"source":["df.user.astype('str').nunique()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["3295942"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"w8XSJVfCuEjq"},"source":["def get_interactions_subset(\n"," interactions, fraction_users, fraction_items, random_seed=10\n","):\n"," \"\"\"\n"," Select subset from interactions based on fraction of users and items\n"," :param interactions: Original interactions\n"," :param fraction_users: Fraction of users\n"," :param fraction_items: Fraction of items\n"," :param random_seed: Random seed\n"," :return: Dataframe with subset of interactions\n"," \"\"\"\n","\n"," def _get_subset_by_column(column, fraction):\n"," column_df = interactions[column].unique()\n"," subset = set(np.random.choice(column_df, int(len(column_df) * fraction)))\n"," return interactions[interactions[column].isin(subset)]\n","\n"," np.random.seed(random_seed)\n"," if fraction_users < 1:\n"," interactions = _get_subset_by_column(\"user\", fraction_users)\n","\n"," if fraction_items < 1:\n"," interactions = _get_subset_by_column(\"item\", fraction_items)\n","\n"," return interactions"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"UovYoB1Bug8r"},"source":["df_subset_users = get_interactions_subset(\n"," df, fraction_users=0.1, fraction_items=1, random_seed=10\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kzsQLHpjvNrh","executionInfo":{"status":"ok","timestamp":1639131663591,"user_tz":-330,"elapsed":27,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9651837f-75a1-430f-c8cd-5abd19af0245"},"source":["df_subset_users.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Int64Index: 6210394 entries, 2 to 65502187\n","Data columns (total 4 columns):\n"," # Column Dtype \n","--- ------ ----- \n"," 0 user int64 \n"," 1 item int64 \n"," 2 event object\n"," 3 timestamp int64 \n","dtypes: int64(3), object(1)\n","memory usage: 236.9+ MB\n"]}]},{"cell_type":"code","metadata":{"id":"BaMGfWsQvZPF"},"source":["df_subset_users.to_parquet('df_subset_users.parquet.snappy', compression='snappy')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"EA64YNBFvBy6"},"source":["df_subset_items = get_interactions_subset(\n"," df, fraction_users=1, fraction_items=0.1, random_seed=10\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vnVylGFqvx5G","executionInfo":{"status":"ok","timestamp":1639131684412,"user_tz":-330,"elapsed":53,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9e2e17b6-1e64-437e-aa09-a563c915b5ae"},"source":["df_subset_items.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Int64Index: 6159193 entries, 10 to 65502199\n","Data columns (total 4 columns):\n"," # Column Dtype \n","--- ------ ----- \n"," 0 user int64 \n"," 1 item int64 \n"," 2 event object\n"," 3 timestamp int64 \n","dtypes: int64(3), object(1)\n","memory usage: 235.0+ MB\n"]}]},{"cell_type":"code","metadata":{"id":"O_zEfvJTvz1B"},"source":["df_subset_items.to_parquet('df_subset_items.parquet.snappy', compression='snappy')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"YX4VZMyB3C-g"},"source":["## Utils"]},{"cell_type":"markdown","metadata":{"id":"MEQquHAn-Krv"},"source":["### Data split"]},{"cell_type":"code","metadata":{"id":"Duvevtfq-Mt7"},"source":["def splitting_functions_factory(function_name):\n"," \"\"\"Returns splitting function based on name\"\"\"\n"," if function_name == \"by_time\":\n"," return split_by_time\n","\n","\n","def split_by_time(interactions, fraction_test, random_state=30):\n"," \"\"\"\n"," Splits interactions by time. Returns tuple of dataframes: train and test.\n"," \"\"\"\n","\n"," np.random.seed(random_state)\n","\n"," test_min_timestamp = np.percentile(\n"," interactions[\"timestamp\"], 100 * (1 - fraction_test)\n"," )\n","\n"," train = interactions[interactions[\"timestamp\"] < test_min_timestamp]\n"," test = interactions[interactions[\"timestamp\"] >= test_min_timestamp]\n","\n"," return train, test\n","\n","\n","def filtering_restrict_to_train_users(train, test):\n"," \"\"\"\n"," Returns test DataFrame restricted to users from train set.\n"," \"\"\"\n"," train_users = set(train[\"user\"])\n"," return test[test[\"user\"].isin(train_users)]\n","\n","\n","def filtering_already_interacted_items(train, test):\n"," \"\"\"\n"," Filters out (user, item) pairs from the test set if the given user interacted with a given item in train set.\n"," \"\"\"\n"," columns = test.columns\n"," already_interacted_items = train[[\"user\", \"item\"]].drop_duplicates()\n"," merged = pd.merge(\n"," test, already_interacted_items, on=[\"user\", \"item\"], how=\"left\", indicator=True\n"," )\n"," test = merged[merged[\"_merge\"] == \"left_only\"]\n"," return test[columns]\n","\n","\n","def filtering_restrict_to_unique_user_item_pair(dataframe):\n"," \"\"\"\n"," Returns pd.DataFrame where each (user, item) pair appears only once.\n"," A list of corresponding events is stores instead of a single event.\n"," Returned timestamp is the timestamp of the first (user, item) interaction.\n"," \"\"\"\n"," return (\n"," dataframe.groupby([\"user\", \"item\"])\n"," .agg({\"event\": list, \"timestamp\": \"min\"})\n"," .reset_index()\n"," )\n","\n","\n","def split(\n"," interactions,\n"," splitting_config=None,\n"," restrict_to_train_users=True,\n"," filter_out_already_interacted_items=True,\n"," restrict_train_to_unique_user_item_pairs=True,\n"," restrict_test_to_unique_user_item_pairs=True,\n"," replace_events_by_ones=True,\n","):\n"," \"\"\"\n"," Main function used for splitting the dataset into the train and test sets.\n"," Parameters\n"," ----------\n"," interactions: pd.DataFrame\n"," Interactions dataframe\n"," splitting_config : dict, optional\n"," Dict with name and parameters passed to splitting function.\n"," Currently only name=\"by_time\" supported.\n"," restrict_to_train_users : boolean, optional\n"," Whether to restrict users in the test set only to users from the train set.\n"," filter_out_already_interacted_items : boolean, optional\n"," Whether to filter out (user, item) pairs from the test set if the given user interacted with a given item\n"," in the train set.\n"," restrict_test_to_unique_user_item_pairs\n"," Whether to return only one row per (user, item) pair in test set.\n"," \"\"\"\n","\n"," if splitting_config is None:\n"," splitting_config = {\n"," \"name\": \"by_time\",\n"," \"fraction_test\": 0.2,\n"," }\n","\n"," splitting_name = splitting_config[\"name\"]\n"," splitting_config = {k: v for k, v in splitting_config.items() if k != \"name\"}\n","\n"," train, test = splitting_functions_factory(splitting_name)(\n"," interactions=interactions, **splitting_config\n"," )\n","\n"," if restrict_to_train_users:\n"," test = filtering_restrict_to_train_users(train, test)\n","\n"," if filter_out_already_interacted_items:\n"," test = filtering_already_interacted_items(train, test)\n","\n"," if restrict_train_to_unique_user_item_pairs:\n"," train = filtering_restrict_to_unique_user_item_pair(train)\n","\n"," if restrict_test_to_unique_user_item_pairs:\n"," test = filtering_restrict_to_unique_user_item_pair(test)\n","\n"," if replace_events_by_ones:\n"," train[\"event\"] = 1\n"," test[\"event\"] = 1\n","\n"," return train, test"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kjTjnPFT3Itb"},"source":["### Metrics"]},{"cell_type":"code","metadata":{"id":"O4-YCPgl3Hlc"},"source":["def ranking_metrics(test_matrix, recommendations, k=10):\n"," \"\"\"\n"," Calculates ranking metrics (precision, recall, F1, F0.5, NDCG, mAP, MRR, LAUC, HR)\n"," based on test interactions matrix and recommendations\n"," :param test_matrix: Test interactions matrix\n"," :param recommendations: Recommendations\n"," :param k: Number of top recommendations to calculate metrics on\n"," :return: Dataframe with metrics\n"," \"\"\"\n","\n"," items_number = test_matrix.shape[1]\n"," metrics = {\n"," \"precision\": 0,\n"," \"recall\": 0,\n"," \"F_1\": 0,\n"," \"F_05\": 0,\n"," \"ndcg\": 0,\n"," \"mAP\": 0,\n"," \"MRR\": 0,\n"," \"LAUC\": 0,\n"," \"HR\": 0,\n"," }\n","\n"," denominators = {\n"," \"relevant_users\": 0,\n"," }\n","\n"," for (user_count, user) in tqdm(enumerate(recommendations[:, 0])):\n"," u_interacted_items = get_interacted_items(test_matrix, user)\n"," interacted_items_amount = len(u_interacted_items)\n","\n"," if interacted_items_amount > 0: # skip users with no items in test set\n"," denominators[\"relevant_users\"] += 1\n","\n"," # evaluation\n"," success_statistics = calculate_successes(\n"," k, recommendations, u_interacted_items, user_count\n"," )\n","\n"," user_metrics = calculate_ranking_metrics(\n"," success_statistics,\n"," interacted_items_amount,\n"," items_number,\n"," k,\n"," )\n","\n"," for metric_name in metrics:\n"," metrics[metric_name] += user_metrics[metric_name]\n","\n"," metrics = {\n"," name: metric / denominators[\"relevant_users\"]\n"," for name, metric in metrics.items()\n"," }\n","\n"," return pd.DataFrame.from_dict(metrics, orient=\"index\").T\n","\n","\n","def calculate_ranking_metrics(\n"," success_statistics,\n"," interacted_items_amount,\n"," items_number,\n"," k,\n","):\n"," \"\"\"\n"," Calculates ranking metrics based on success statistics\n"," :param success_statistics: Success statistics dictionary\n"," :param interacted_items_amount:\n"," :param items_number:\n"," :param k: Number of top recommendations to calculate metrics on\n"," :return: Dictionary with metrics\n"," \"\"\"\n"," precision = success_statistics[\"total_amount\"] / k\n"," recall = success_statistics[\"total_amount\"] / interacted_items_amount\n"," user_metrics = dict(\n"," precision=precision,\n"," recall=recall,\n"," F_1=calculate_f(precision, recall, 1),\n"," F_05=calculate_f(precision, recall, 0.5),\n"," ndcg=calculate_ndcg(interacted_items_amount, k, success_statistics[\"total\"]),\n"," mAP=calculate_map(success_statistics, interacted_items_amount, k),\n"," MRR=calculate_mrr(success_statistics[\"total\"]),\n"," LAUC=calculate_lauc(\n"," success_statistics, interacted_items_amount, items_number, k\n"," ),\n"," HR=success_statistics[\"total_amount\"] > 0,\n"," )\n"," return user_metrics\n","\n","\n","def calculate_mrr(user_successes):\n"," return (\n"," 1 / (user_successes.nonzero()[0][0] + 1)\n"," if user_successes.nonzero()[0].size > 0\n"," else 0\n"," )\n","\n","\n","def calculate_f(precision, recall, f):\n"," return (\n"," (f ** 2 + 1) * (precision * recall) / (f ** 2 * precision + recall)\n"," if precision + recall > 0\n"," else 0\n"," )\n","\n","\n","def calculate_lauc(successes, interacted_items_amount, items_number, k):\n"," return (\n"," np.dot(successes[\"cumsum\"], 1 - successes[\"total\"])\n"," + (successes[\"total_amount\"] + interacted_items_amount)\n"," / 2\n"," * ((items_number - interacted_items_amount) - (k - successes[\"total_amount\"]))\n"," ) / ((items_number - interacted_items_amount) * interacted_items_amount)\n","\n","\n","def calculate_map(successes, interacted_items_amount, k):\n"," return np.dot(successes[\"cumsum\"] / np.arange(1, k + 1), successes[\"total\"]) / min(\n"," k, interacted_items_amount\n"," )\n","\n","\n","def calculate_ndcg(interacted_items_amount, k, user_successes):\n"," cumulative_gain = 1.0 / np.log2(np.arange(2, k + 2))\n"," cg_sum = np.cumsum(cumulative_gain)\n"," return (\n"," np.dot(user_successes, cumulative_gain)\n"," / cg_sum[min(k, interacted_items_amount) - 1]\n"," )\n","\n","\n","def calculate_successes(k, recommendations, u_interacted_items, user_count):\n","\n"," items = recommendations[user_count, 1 : k + 1]\n"," user_successes = np.isin(items, u_interacted_items)\n","\n"," return dict(\n"," total=user_successes.astype(int),\n"," total_amount=user_successes.sum(),\n"," cumsum=np.cumsum(user_successes),\n"," )\n","\n","\n","def get_reactions(test_matrix, user):\n"," return test_matrix.data[test_matrix.indptr[user] : test_matrix.indptr[user + 1]]\n","\n","\n","def get_interacted_items(test_matrix, user):\n"," return test_matrix.indices[test_matrix.indptr[user] : test_matrix.indptr[user + 1]]\n","\n","\n","def diversity_metrics(\n"," test_matrix, formatted_recommendations, original_recommendations, k=10\n","):\n"," \"\"\"\n"," Calculates diversity metrics\n"," (% if recommendations in test, test coverage, Shannon, Gini, users without recommendations)\n"," based on test interactions matrix and recommendations\n"," :param test_matrix: user/item interactions' matrix\n"," :param formatted_recommendations: recommendations where user and item ids were replaced by respective codes based on test_matrix\n"," :param original_recommendations: original format recommendations\n"," :param k: Number of top recommendations to calculate metrics on\n"," :return: Dataframe with metrics\n"," \"\"\"\n","\n"," formatted_recommendations = formatted_recommendations[:, : k + 1]\n","\n"," frequency_statistics = calculate_frequencies(formatted_recommendations, test_matrix)\n","\n"," with np.errstate(\n"," divide=\"ignore\"\n"," ): # let's put zeros we items with 0 frequency and ignore division warning\n"," log_frequencies = np.nan_to_num(\n"," np.log(frequency_statistics[\"frequencies\"]), posinf=0, neginf=0\n"," )\n","\n"," metrics = dict(\n"," reco_in_test=frequency_statistics[\"recommendations_in_test_n\"]\n"," / frequency_statistics[\"total_recommendations_n\"],\n"," test_coverage=frequency_statistics[\"recommended_items_n\"]\n"," / test_matrix.shape[1],\n"," Shannon=-np.dot(frequency_statistics[\"frequencies\"], log_frequencies),\n"," Gini=calculate_gini(\n"," frequency_statistics[\"frequencies\"], frequency_statistics[\"items_in_test_n\"]\n"," ),\n"," users_without_reco=original_recommendations.iloc[:, 1].isna().sum()\n"," / len(original_recommendations),\n"," users_without_k_reco=original_recommendations.iloc[:, k - 1].isna().sum()\n"," / len(original_recommendations),\n"," )\n","\n"," return pd.DataFrame.from_dict(metrics, orient=\"index\").T\n","\n","\n","def calculate_gini(frequencies, items_in_test_n):\n"," return (\n"," np.dot(\n"," frequencies,\n"," np.arange(\n"," 1 - items_in_test_n,\n"," items_in_test_n,\n"," 2,\n"," ),\n"," )\n"," / (items_in_test_n - 1)\n"," )\n","\n","\n","def calculate_frequencies(formatted_recommendations, test_matrix):\n"," frequencies = defaultdict(\n"," int, [(item, 0) for item in list(set(test_matrix.indices))]\n"," )\n"," for item in formatted_recommendations[:, 1:].flat:\n"," frequencies[item] += 1\n"," recommendations_out_test_n = frequencies[-1]\n"," del frequencies[-1]\n"," frequencies = np.array(list(frequencies.values()))\n"," items_in_test_n = len(frequencies)\n"," recommended_items_n = len(frequencies[frequencies > 0])\n"," recommendations_in_test_n = np.sum(frequencies)\n"," frequencies = frequencies / np.sum(frequencies)\n"," frequencies = np.sort(frequencies)\n"," return dict(\n"," frequencies=frequencies,\n"," items_in_test_n=items_in_test_n,\n"," recommended_items_n=recommended_items_n,\n"," recommendations_in_test_n=recommendations_in_test_n,\n"," total_recommendations_n=recommendations_out_test_n + recommendations_in_test_n,\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Z1ccnvbl3SdZ"},"source":["### Evaluator"]},{"cell_type":"code","metadata":{"id":"ycJ-RJPs3Tne"},"source":["def preprocess_test(test: pd.DataFrame):\n"," \"\"\"\n"," Preprocesses test set to speed up evaluation\n"," \"\"\"\n","\n"," def _map_column(test, column):\n"," test[f\"{column}_code\"] = test[column].astype(\"category\").cat.codes\n"," return dict(zip(test[column], test[f\"{column}_code\"]))\n","\n"," test = test.copy()\n","\n"," test.columns = [\"user\", \"item\", \"event\", \"timestamp\"]\n"," user_map = _map_column(test, \"user\")\n"," item_map = _map_column(test, \"item\")\n","\n"," test_matrix = sparse.csr_matrix(\n"," (np.ones(len(test)), (test[\"user_code\"], test[\"item_code\"]))\n"," )\n"," return user_map, item_map, test_matrix\n","\n","\n","class Evaluator:\n"," \"\"\"\n"," Class used for models evaluation\n"," \"\"\"\n","\n"," # pylint: disable=too-many-instance-attributes\n"," # pylint: disable=too-many-arguments\n"," def __init__(\n"," self,\n"," recommendations_path: Path,\n"," test_path: Path,\n"," k,\n"," models_to_evaluate,\n"," ):\n"," self.recommendations_path = recommendations_path\n"," self.test_path = test_path\n"," self.k = k\n"," self.models_to_evaluate = models_to_evaluate\n"," self.located_models = None\n"," self.test = None\n"," self.user_map = None\n"," self.item_map = None\n"," self.test_matrix = None\n","\n"," self.evaluation_results = []\n","\n"," def prepare(self):\n"," \"\"\"\n"," Prepares test set and models to evaluate\n"," \"\"\"\n","\n"," def _get_models(models_to_evaluate, recommendations_path):\n"," models = [\n"," (file_name.split(\".\")[0], file_name)\n"," for file_name in os.listdir(recommendations_path)\n"," ]\n"," if models_to_evaluate:\n"," return [model for model in models if model[0] in models_to_evaluate]\n"," return models\n","\n"," self.test = pd.read_csv(self.test_path, compression=\"gzip\").astype(\n"," {\"user\": str, \"item\": str}\n"," )\n"," self.user_map, self.item_map, self.test_matrix = preprocess_test(self.test)\n","\n"," self.located_models = _get_models(\n"," self.models_to_evaluate, self.recommendations_path\n"," )\n","\n"," def evaluate_models(self):\n"," \"\"\"\n"," Evaluating multiple models\n"," \"\"\"\n","\n"," def _read_recommendations(file_name):\n"," return pd.read_csv(\n"," os.path.join(self.recommendations_path, file_name),\n"," header=None,\n"," compression=\"gzip\",\n"," dtype=str,\n"," )\n","\n"," for model, file_name in self.located_models:\n"," recommendations = _read_recommendations(file_name)\n"," evaluation_result = self.evaluate(\n"," original_recommendations=recommendations,\n"," )\n","\n"," evaluation_result.insert(0, \"model_name\", model)\n"," self.evaluation_results.append(evaluation_result)\n"," self.evaluation_results = pd.concat(self.evaluation_results).set_index(\n"," \"model_name\"\n"," )\n"," if \"precision\" in self.evaluation_results.columns:\n"," self.evaluation_results = self.evaluation_results.sort_values(\n"," by=\"precision\", ascending=False\n"," )\n","\n"," def evaluate(\n"," self,\n"," original_recommendations: pd.DataFrame,\n"," ):\n"," \"\"\"\n"," Evaluate single model\n"," \"\"\"\n","\n"," def _format_recommendations(recommendations, user_id_code, item_id_code):\n"," users = recommendations.iloc[:, :1].applymap(\n"," lambda x: user_id_code.setdefault(str(x), -1)\n"," )\n"," items = recommendations.iloc[:, 1:].applymap(\n"," lambda x: -1 if pd.isna(x) else item_id_code.setdefault(x, -1)\n"," )\n"," return np.array(pd.concat([users, items], axis=1))\n","\n"," original_recommendations = original_recommendations.iloc[:, : self.k + 1].copy()\n","\n"," formatted_recommendations = _format_recommendations(\n"," original_recommendations, self.user_map, self.item_map\n"," )\n","\n"," evaluation_results = pd.concat(\n"," [\n"," ranking_metrics(\n"," self.test_matrix,\n"," formatted_recommendations,\n"," k=self.k,\n"," ),\n"," diversity_metrics(\n"," self.test_matrix,\n"," formatted_recommendations,\n"," original_recommendations,\n"," self.k,\n"," ),\n"," ],\n"," axis=1,\n"," )\n","\n"," return evaluation_results"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Pt57vODj33uG"},"source":["### Helpers"]},{"cell_type":"code","metadata":{"id":"Re1BE-v0347n"},"source":["def overlap(df1, df2):\n"," \"\"\"\n"," Returns the Overlap Coefficient with respect to (user, item) pairs.\n"," We assume uniqueness of (user, item) pairs in DataFrames\n"," (not recommending the same item to the same users multiple times)).\n"," :param df1: DataFrame which index is user_id and column [\"items\"] is a list of recommended items\n"," :param df2: DataFrame which index is user_id and column [\"items\"] is a list of recommended items\n"," \"\"\"\n"," nb_items = min(df1[\"items\"].apply(len).sum(), df2[\"items\"].apply(len).sum())\n","\n"," merged_df = pd.merge(df1, df2, left_index=True, right_index=True)\n"," nb_common_items = merged_df.apply(\n"," lambda x: len(set(x[\"items_x\"]) & set(x[\"items_y\"])), axis=1\n"," ).sum()\n","\n"," return 1.00 * nb_common_items / nb_items\n","\n","\n","def get_recommendations(models_to_evaluate, recommendations_path):\n"," \"\"\"\n"," Returns dictionary with model_names as keys and recommendations as values.\n"," :param models_to_evaluate: List of model names\n"," :param recommendations_path: Stored recommendations directory\n"," \"\"\"\n"," models = [\n"," (file_name.split(\".\")[0], file_name)\n"," for file_name in os.listdir(recommendations_path)\n"," ]\n","\n"," return {\n"," model[0]: pd.read_csv(\n"," os.path.join(recommendations_path, model[1]),\n"," header=None,\n"," compression=\"gzip\",\n"," dtype=str,\n"," )\n"," for model in models\n"," if model[0] in models_to_evaluate\n"," }"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Qxnw4e6e4Wkj"},"source":["def dict_to_df(dictionary):\n"," \"\"\"\n"," Creates pandas dataframe from dictionary\n"," :param dictionary: Original dictionary\n"," :return: Dataframe from original dictionary\n"," \"\"\"\n"," return pd.DataFrame({k: [v] for k, v in dictionary.items()})\n","\n","\n","def efficiency(path, base_params=None):\n"," \"\"\"\n"," Parametrized decorator for executing function with efficiency logging and\n"," storing the results under the given path\n"," \"\"\"\n"," base_params = base_params or {}\n","\n"," def efficiency_decorator(func):\n"," def wrapper(*args, **kwargs):\n"," tracemalloc.start()\n"," start_time = time()\n"," result = func(*args, **kwargs)\n"," execution_time = time() - start_time\n"," _, peak = tracemalloc.get_traced_memory()\n"," dict_to_df(\n"," {\n"," **base_params,\n"," **{\n"," \"function_name\": func.__name__,\n"," \"execution_time\": execution_time,\n"," \"memory_peak\": peak,\n"," },\n"," }\n"," ).to_csv(path, index=False)\n"," tracemalloc.stop()\n"," return result\n","\n"," return wrapper\n","\n"," return efficiency_decorator\n","\n","\n","def get_unix_path(path):\n"," \"\"\"\n"," Returns the input path with unique csv filename\n"," \"\"\"\n"," return path / f\"{datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')}.csv\"\n","\n","\n","def df_from_dir(dir_path):\n"," \"\"\"\n"," Returns pd.DataFrame with concatenated files from the given path\n"," \"\"\"\n"," files_read = [\n"," pd.read_csv(dir_path / filename)\n"," for filename in os.listdir(dir_path)\n"," if filename.endswith(\".csv\")\n"," ]\n"," return pd.concat(files_read, axis=0, ignore_index=True)\n","\n","\n","def get_interactions_subset(\n"," interactions, fraction_users, fraction_items, random_seed=10\n","):\n"," \"\"\"\n"," Select subset from interactions based on fraction of users and items\n"," :param interactions: Original interactions\n"," :param fraction_users: Fraction of users\n"," :param fraction_items: Fraction of items\n"," :param random_seed: Random seed\n"," :return: Dataframe with subset of interactions\n"," \"\"\"\n","\n"," def _get_subset_by_column(column, fraction):\n"," column_df = interactions[column].unique()\n"," subset = set(np.random.choice(column_df, int(len(column_df) * fraction)))\n"," return interactions[interactions[column].isin(subset)]\n","\n"," np.random.seed(random_seed)\n"," if fraction_users < 1:\n"," interactions = _get_subset_by_column(\"user\", fraction_users)\n","\n"," if fraction_items < 1:\n"," interactions = _get_subset_by_column(\"item\", fraction_items)\n","\n"," return interactions"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"d1P7p2mP3EHa"},"source":["## Data loading"]},{"cell_type":"code","metadata":{"id":"UZWq1oM_Ci9M"},"source":["def load_interactions(data_path):\n"," return pd.read_csv(data_path,\n"," compression='gzip',\n"," header=0,names=[\"user\", \"item\", \"event\", \"timestamp\"],\n"," ).astype({\"user\": str, \"item\": str, \"event\": str, \"timestamp\": int})\n","\n","def load_target_users(path):\n"," return list(pd.read_csv(path, compression=\"gzip\", header=None).astype(str).iloc[:, 0])\n","\n","def save_recommendations(recommendations, path):\n"," recommendations.to_csv(path, index=False, header=False, compression=\"gzip\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"zzy1ttnO05XP","executionInfo":{"status":"ok","timestamp":1639131781858,"user_tz":-330,"elapsed":11314,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"f90285a5-684d-4932-f75c-5113893ef11a"},"source":["data_path = 'df_subset_items.parquet.snappy'\n","\n","interactions = pd.read_parquet(data_path)\n","interactions.columns = [\"user\", \"item\", \"event\", \"timestamp\"]\n","interactions = interactions.astype({\"user\": str, \"item\": str, \"event\": str, \"timestamp\": int})\n","interactions.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
useritemeventtimestamp
10748807167669click1581465600
1913984808064click1581465600
2115419847050click1581465600
24168607450001click1581465600
509627678456click1581465601
\n","
"],"text/plain":[" user item event timestamp\n","10 748807 167669 click 1581465600\n","19 1398480 8064 click 1581465600\n","21 1541984 7050 click 1581465600\n","24 1686074 50001 click 1581465600\n","50 96276 78456 click 1581465601"]},"metadata":{},"execution_count":20}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9ObBEXjD0u4-","executionInfo":{"status":"ok","timestamp":1639131785120,"user_tz":-330,"elapsed":561,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9a8f9ed7-6e94-4f0e-8cca-c2877f6e2e68"},"source":["interactions.info()"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Int64Index: 6159193 entries, 10 to 65502199\n","Data columns (total 4 columns):\n"," # Column Dtype \n","--- ------ ----- \n"," 0 user object\n"," 1 item object\n"," 2 event object\n"," 3 timestamp int64 \n","dtypes: int64(1), object(3)\n","memory usage: 235.0+ MB\n"]}]},{"cell_type":"markdown","metadata":{"id":"9KRQj2dK3GCU"},"source":["## EDA"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"X-U-Lwa91rJ_","executionInfo":{"status":"ok","timestamp":1630826644408,"user_tz":-330,"elapsed":10555,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"510c65e4-675d-4e87-f292-bcd0a4c529af"},"source":["n_users = interactions[\"user\"].nunique()\n","n_items = interactions[\"item\"].nunique()\n","n_interactions = len(interactions)\n","\n","interactions_per_user = interactions.groupby(\"user\").size()\n","interactions_per_item = interactions.groupby(\"item\").size()\n","\n","print(f\"We have {n_users} users, {n_items} items and {n_interactions} interactions.\\n\")\n","\n","print(\n"," f\"Data sparsity (% of missing entries) is {round(100 * (1- n_interactions / (n_users * n_items)), 4)}%.\\n\"\n",")\n","\n","print(\n"," f\"Average number of interactions per user is {round(interactions_per_user.mean(), 3)}\\\n"," (standard deviation {round(interactions_per_user.std(ddof=0),3)}).\\n\"\n",")\n","\n","print(\n"," f\"Average number of interactions per item is {round(interactions_per_item.mean(), 3)}\\\n"," (standard deviation {round(interactions_per_item.std(ddof=0),3)}).\\n\"\n",")"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["We have 1535397 users, 17668 items and 6159193 interactions.\n","\n","Data sparsity (% of missing entries) is 99.9773%.\n","\n","Average number of interactions per user is 4.011 (standard deviation 6.539).\n","\n","Average number of interactions per item is 348.607 (standard deviation 608.238).\n","\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":283},"id":"jsD132QP15fO","executionInfo":{"status":"ok","timestamp":1630826694292,"user_tz":-330,"elapsed":475,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"9d0175f1-a9db-452d-db43-6ac791bd0f03"},"source":["def compute_quantiles(series, quantiles=[0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99]):\n"," return pd.DataFrame(\n"," [[quantile, series.quantile(quantile)] for quantile in quantiles],\n"," columns=[\"quantile\", \"value\"],\n"," )\n","\n","\n","print(\"Interactions distribution per user:\")\n","compute_quantiles(interactions_per_user)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Interactions distribution per user:\n"]},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
quantilevalue
00.011.0
10.101.0
20.251.0
30.502.0
40.754.0
50.909.0
60.9932.0
\n","
"],"text/plain":[" quantile value\n","0 0.01 1.0\n","1 0.10 1.0\n","2 0.25 1.0\n","3 0.50 2.0\n","4 0.75 4.0\n","5 0.90 9.0\n","6 0.99 32.0"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":577},"id":"_x1Ogx1x2IGk","executionInfo":{"status":"ok","timestamp":1630826707959,"user_tz":-330,"elapsed":1916,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"69f18dd0-b133-447e-993b-d1b5fab79a3e"},"source":["def plot_interactions_distribution(series, aggregation=\"user\", ylabel=\"Users\", bins=30):\n"," matplotlib.rcParams.update({\"font.size\": 22})\n"," series.plot.hist(bins=bins, rwidth=0.9, logy=True, figsize=(16, 9))\n"," plt.title(f\"Number of interactions per {aggregation}\")\n"," plt.xlabel(\"Interactions\")\n"," plt.ylabel(ylabel)\n"," plt.grid(axis=\"y\", alpha=0.5)\n","\n","\n","plot_interactions_distribution(interactions_per_user, \"user\", \"Users\")"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA9EAAAJECAYAAAAR7Kc3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde5wlZ10n/s9Xwj2hWcUQTJCBncBKWIRkWGBVgr94JQxrJCwrwi54GSUR44rCIIvCqjCIgkogMiJmV4grIKJDuLgiCbdwSbgIgSxR6YBIQC62hGskz++PqmYOJ32p07dzuvv9fr3Oq/qceup5vnXO6XnNp6vqqWqtBQAAAFjdN0y7AAAAANguhGgAAAAYSIgGAACAgYRoAAAAGEiIBgAAgIGEaAAAABhIiAZ2vKp6dFW1kcejJmi/Z2uq3Hij+zHtWmZJVd2sqp5QVe+qqs+NfNa/PUEfi9s8ehNLZUJV9cCd8LsLwGwTooHd6Jer6ibTLoKpeXGSZya5d5Jjp1zLkqpqvg+CT512LbOgqi7s349Lpl0LAAjRwG60N8l/nXYRbL2qumuSh/VPn5/kLkmO6x9PmFZdAMD2ccy0CwDYYn+fLjg9pape3Fq7ftoFsaW+feTnJ7fW/nktnbTWaoPqYQO11i5J4rMBYFM5Eg3sNv+zX945yWOmWQhTcavFH9YaoAGA3U2IBnabS5P8df/zk6vqZpN2MGRSqZUmOKqqPSPrHlhVN6+qJ1bVe6vquqr6ZFW9qqruO7bdGVV1pKo+XlVfqqorq+rxQ6/vrqo7V9UL+uttv9T385KqOmXAtsdU1Y9V1Wur6tqq+kpV/VNV/WVV/WhVLXn0b3xys6q6Y1X9TlV9qKq+0K+77ZD6x/rdX1V/VlX/WFVfrqpPVdUlVXVOVd10ifYX9jVcOPLa6GRz8xOOv+x3oK+jVdWF/fPvqqpX9u/bl6vqw1X13Kq6/Qp13ql/6VfG6lxuzC35fKrq31fV/6iqN/b9X19V/1xVV1TVr1XVNw98/x7Q7+vfVtXnq+pfquoDVfWyqnrE4me4WF+S/9ZvevoS78eFI/0Omlis/0wuqqqP9L8Ln62qt1fVwapa9jr59Xy2I33ctKp+uqreMPIefqaq/l91v98/W1W3G/I+jvX7ddfRV9Vjquotfd+f7z+j86pq1bMQq+qu/X58oLrJ977Q1/e7VfWtE9TwiKr6q6r6RFXdUJNN3vd17/UK7Vb6vaiq+pGqenV1/959paoW+u/d/61ugsGV9mfq7wMwo1prHh4eHjv6keTRSVr/2JPkP448P2e19kusX1z36BXGfOByffQ1LK57SJJ3jjwffXw5yff22zxlmTYtyf8esN//IclnVxjnh1fYl29N8p4Vxm9JXp3k1qvUcL8kn1li29tO8FnePMlLV6nlvUlOHNvuwlW2mZ/wO7XsdyDJJf26C5P89yRfXWbMa9ZQ543G3KrPJ92p8KvV9okkp63wvt0yyUsG9HOvJepb7nHhkN+7fn0lec4q/V2T5O7L1L/mz7bf/tgklw3Yp7PX8O/cfL/tU9NNnrdc35cu9V0Y6efxSa5fYfvPJ3nwKjU8bZnP+bcn2J+vvddr+V1McpMkrxzwXv/CLL8PHh4es/lwJBrYdVprb03yuv7pL1XVzadYzm8nuVuSn093rfbtkvxQko8nuVmSF1TVw9Kdhv6SdGH4m5LcM8mf9308qqq+b5Vx/iTJl9KFkhOTfEu6o3vX9uP8cVXdfXyjqrpNuiP3354uIJ2X5N8l+ca+7if3/f5gkhesUsPLk1zXj3vHJCck2d9vP9Rzc3RisFem+4PI7ZKckuQ304WaeyZ5VX39WQY/lW7ysJ8eee24kceN9n0DPCDJbyX5iyTf2dd5lyS/mu4/0t+a5Flj2yzW+ZH++TPG6jwuXUBKsuWfT0vy+iQ/2+/byem+i6ck+YkkVyU5PsmfVtUtlxnjoiSP6H9+XZIHpfsu3i7dbOn/Pcm7R9q/uN/nl/TP37zE+/FTq+zXqCcl+bmRvs5I8s3pJht8cpIvpvtc/rKq/s0K/azls02SJ6b7Y8VXk/x6un2+fV/DPZP8eJKL+/Vr9d+S/Gi692xfX9tpOfq9eUCS31tqw6o6N93v0TFJXpHke9J9D745yfcneWu6SyJeVlX3WKGGH0/3Of9hun+zbpfud+yl69ivST06yX/qf74g3fv+Lem+s9/W1/cn6f6I+HV22PsAbIZpp3gPDw+PzX5kiSPL6f5Ds/jaeau1H1u/5JGPsTYPXK6PfP2R6OuTfMcS23/PWJvzl2hz03QTpbUkf7zKfl+X5K5LtLlrv64lObLE+t/t1316qfeib/P9I+PsW6GGTyU5aR2f471H+nrJMm3OGWnzsyu9J+v8Ti37HcjRI2gtyeFltl98X7+c5DZLrJ/v1z91lTpm6fM5Nsnf9n392BLr/8vIWCseiUtyzNjzC/vtLlllu5V+727fv9+t/4xuusT2DxrZ/tkb/dkmuaJf95z1fP+WGXd+QG2/P9LmtLF1d0j3B5OW5LeW2f6m6Y5ktySvWqWGZ65zfxbf6wtXabfk72KSP+1f/7MJx52p98HDw2M2H45EA7tSa+0dSV7VPz24wpGzzfYnrbW3LPH669OFmqT7z/gvjTdo3czir+if3nd8/ZjzW2sfWqKPDyV5Xv/0QaPXclbVrdMdSUm6maznl+q4tfa6JG/on/7oCjU8q7X2D6vUuZLFWr6So0cTx2t5frrTuZPu6Og0fSHL3zbrD/vlzfL1M4YPNmufT2vtuhz9Pn7vEk3O65d/l+QXVunrX9daxwoele79Tro/sNxoZv7W2qvTHV1OksfU8vMNrPWzXbwe+WODKl6bL6U74r2UJ+TomQXjEyv+dLrLJf5hue379+wp/dMH1fLzGXw2ya8MLXiTrPW93mnvA7AJhGhgN/vldEcKTkh3BHMaXrfUi621lu4oc5K8rbX2L8ts/3f98oRVxvmzAeu+Id0pj4v+Y47OZv3Gqjp2uUeOBtd9K4xz8So1ruY7++UlrbV/WqHdy/vlPVb4z+1WeFtbfgbw/zfy82qf3XK2/PPpJ2p6WFX9aT+J1uLkY4uTk/1i3/SuY9sdl+Q+/dOXbFJIXs3i9+dDrbW/WaHdy/rlbZMsd6ruWj/bxVPVn1Dd5HiDJgWc0CWttc8utaJ//dL+6XeMrf6efnlpklus8F36YN+u0p0mvpS/bq1NcpnGZlh8r3+sqh41wWU7O+19ADaB+0QDu1Zr7d1V9cokZ6X7T+3vtdY+v8Vl/OMK677YLz8+oM1qR9KvGrjuTiM/323k5ytX6X/RSrMz//0K64ZYrO0Dq7RbrLXSXZs6rVtZLfvZtta+UEcnzL7Vcu1WsaWfTx8c/iLJdw8YZ27s+Z50Ez0l3SRo0zDp92dxm/cu0Watn+1T012n+83p3svPVtWb0l2f/YbW2uWr1DbESr/ri+u/P1//u54c/T79aFY+Y2HUct+n9f6ub4TnpLtc4U5J/neSC6rqLene60uSvLW1ttS15zvtfQA2gSPRwG73K+mORh+f5HFTGH/IBELrmWRo0XUD1x038vN4EBriFsutaK19YQ39jVqsbaV9SZLPLbHNNAz93Ja8/dQAW/35PCdHA/Qfprt++N+mm6hpcZKvQ/368T/S32bk589lOjby+7Omz7Y/5f7UJP8r3ezO/ybdDP2/keSd/a2Xhga35ay2f4vrx/dtI79P6/1dX7fW2kK6uS+em+606lsn+b50kzS+MclH+1t+jf9feEe9D8DmEKKBXa219r4cPX3zF/rTTjfCrJ3ps+y9b8fWjQaI0f+M36q1VgMeeza06q+3WNtK+zK+flqBbSts2efTX3/9qP7podbaj7XWXtNa+/vW2mdaa9f110Qvd0bELPxhYya+P621D7fWHp1uBvXvSHd9+GvSTSD4b5O8uKrOW76HVQ3dv/F9W/w+/cbA71K11i5cR52raas1qFXued1a+2Rr7WfTHSnel+4Ppa9IF27vkO7uCL81ttmsvQ/ADBKiAbpTLG9Id0RtyH9eF69xW+kU6m9ZZ00b7d8NXHfNyM+jpyL+240tZ03m++Vqt6M6pV+2HL1V1E60lZ/P3dJNtpQk/2eFdv9+mdc/nKNHb++1UUVNaL5fDv3+jG6z4VprX2mtvbW19luttQelu0XW4uR/v7zEEdKhVvpdH11/zdjri9+nWfhdTzbw39nW2ldba1e01s5vrT003S3c3tyvflxVfdNI81l7H4AZJEQDu15r7YNJ/rh/+visfjrf4jXKd1uhzQ+st64NdtYK636oX96Q5G0jr1+ao/dQffhmFDWhxf/0nl5Vt1uh3dn98v0rTP406xZnjl5p4qmt/HxGJ2VasqaqOindPYhvpLX2uSRv758+YrUjiEsY8n6sZvH7c9dV7u27+P355yTvX8d4E+lnRl+8f/M3prsl11o8cLkJ9fp7X5/ePx2/K8Bf9svvm/KEfIs27d/Z1tpnkjy7f3qTfP1EeLP2PgAzSIgG6Dwt3ZGy22b1a6MXw8DZVXWjSaGq6vQk/3ljy1u3n6mqu46/2L/2M/3TV7fWPrG4rp8R/IX908dX1fhsvuN93aaq7rBRBS/hD/rlzXP0P8DjNfxUjh7p/P1NrGWzfbpfLnukbYs/n/mRn/cv0fcxSQ5n5csYfrdf7k3yzJUGW2LW6lXfjwFenO72aEnyO0sF+ar6gRz9o9KLWms3rGO8G6mq1Y4SLx79/GqShTUOc4t011gv5Zk5ev3uH46te166P8ocl+SFVXXTlQapqpXC7UZY/Hf226vqRreB62/H98vLbTzBe50c/X4ls/c+ADNIiAZI0lq7Oskf9U9XO43vRf3yxCQXV9V9q+rfVNXJVXUwyauziaeBrtE/Jbmkqv5rVd2hfzwq3Sy1t04XLpa6J+qT092y55ZJ/rqqnlNV96+qb66qb6yqu/W3PHpRuvuqrhjk1qO19p4cDcaP6m+zdN++jn9XVYdy9J7X70nygs2qZQtc0S9/qKq+r6rmquqY/jE6WdWWfD6ttY8neVP/9Jeq6slVddequl1VnZHuvuY/mKO3/Vmqjz9J8sr+6c9X1aur6geq6oS+1ntW1c9U1eW58Wnhi+/HXarq3Ko6fuT9GPR/mf4PRE/rn/5/Sf6qqr67qr6pqu7S/+7+ab/+Y0l+bUi/E/pAVf1VVZ1TVaf1+3G7qjq1qn4zR2+19+frmIhvPslPVtUf9f1+Y1Xdu6r+d5Kf7Nu8uLV2xehG/ZHwxfuvPzTJO/pbQ92l//59S1V9Z1X9YlW9M0ffq83yshy9bvvPq+oh/Wd1YlU9Mt1ZMyvdPuo1VfW2qnp8/zux+D07paqenKOf77taa4un0c/i+wDMotaah4eHx45+pLvNSesfe1Zod+d0YbKt1j5dQGvLPN6U5Mzl+kh3u5/FdQ9coZ5L+jYXDtm3Vfb7P6Q7PXWper+c5KErjHGHdKfCLre/o4+HDK1vjZ/lzZO8dJUa3pvkxEnfrwnrWBzr0Wv53Ab0cY/+c1lq/x491nZLPp901wp/ZoW+n51ufoGWZH6ZPm6Z5E8G1HmvJbb7u2XaXjjS7oEjr9/odzfdbNm/vcrY1yS5+1p/J1f6bAd+Ru9Ocvs1fCfn++2fmuQlK/R/aZJbr9DPgXThdLU637VSDRv0+/6j6Y7KLzX+P6S7vn2593p+wD58OMldZ/198PDwmL2HI9EAvdbah3PjUxyX89NJfiLJO9Ldqua6dP/5/e/pbgO01febXlFr7R1JTkt3SvRH0v2x4NokFyU5tbW27NGU1h2F/K5011W/rN/+S30fH08XLP5nuuDzF5u3F0lr7cuttf+c7l67f97vw/Xpwt0b052afp/W2sc2s47N1lp7f7rri1+R7p7E16/Qdks+n9balem+Qxf2/V6f5BNJXpvkh1prPz+gjy+21h6e7lZD/6ev9cvp/sDzgXSnXO/P2LXIrbUv9vv4e0n+NisfgVxp/NZa+7l07+3/SRfEvpLu1Ol3JHlSklNaa6vdS3qtTkvyhHSzcX8oyb/k6Pv4l+mOFP+HNnJZxRo9su/rben27Qvp/n36uSRntNaW/feptXY43SRnv57uPflMuiD7uRz9jB6Z7vPYVK21l6T79/Q1fR1fTvfHlN9K931e6XP6/iQ/m+7shw+ku83Vvyb5VLp/Kx6f5B5t5Cj02Ngz8z4As6daa9OuAQCAdaiq+SR3SvK01tpTp1sNwM7mSDQAAAAMJEQDAADAQEI0AAAADCREAwAAwEBCNAAAAAxkdu41uN3tbtf27Nkz7TIAAADYBFdcccWnWmvfvNS6Y7a6mJ1gz549ufzyy6ddBgAAAJugqq5Zbp3TuQEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIRoAAAAGEqInUFX7q+rwwsLCtEsBAABgCoToCbTWjrTWDszNzU27FAAAAKZAiAYAAICBhGgAAAAYSIgGAACAgYRoAAAAGEiIBgAAgIGEaAAAABhIiAYAAICBhGgAAAAYSIgGAACAgYRoAAAAGOiYaRcwTVX1qCQ/l+TuSb6Q5F1JfqS19qmpFrYB9hy8eFP7nz905qb2DwAAMIt27ZHoqnpykguSvCLJDyb58SRXJrn5NOsCAABgdu3KI9FVdbckT01yVmvtVSOrXjmdigAAANgOduuR6MckuWYsQAMAAMCKZiZEV9Xdquq8qnpxVV1VVTdUVauqswds+4iqelNVLVTVdVV1eVWdW1XL7d/9kvxNVf2Pqrq2qq6vqndU1ekbu1cAAADsJLN0Ovdjk5w36UZV9bwk5yT5UpLXJ7k+yRlJzk9yRlWd3Vq7YWyzE5KcluTbk/xskn9J8gtJXltV39Zam1/rTgAAALBzzcyR6CTvT/KsJA9PsjfJpattUFUPTRegr01yz9bag1trZyU5OckHk5yV5HFLbPoNSY5N8tDW2ktba69N8pB0YfoXN2BfAAAA2IFm5kh0a+2Fo8+rashmT+qXT2ytXT3S1yeq6rFJLklysKqeO3Y0+rNJPt1ae8/INl+oqrcluccadwEAAIAdbpaORE+kqk5Kd0r2V5K8bHx9a+3SJB9Ld+r2/cZWX7lC17fYqBoBAADYWbZtiE5y7355ZWvti8u0eedY20WvSvJNVXXq4gtVdesk909yxYZWCQAAwI4xM6dzr8Gd++U1K7T5yFjbRa9M8o4kL6+qJyf5XJLHJ7lVkmcv1VFVHUhyIElOPPHEzM/Pr63qLXL/48fnUttYs77/AAAAm2E7h+hj++XnV2hzXb88bvTF1toNVXVmkt9M8vx0p3C/LckDW2t/u1RHrbXDSQ4nyb59+9qePXvWXvkWuOyTK52xvn6zvv8AAACbYTuH6HVprX0qyaOnXQcAAADbx3a+JnrxKPOtV2izeLT6cxsxYFXtr6rDCwsLG9EdAAAA28x2DtHz/fJOK7S541jbdWmtHWmtHZibm9uI7gAAANhmtnOIfne/PKWqbrlMm/uMtQUAAIA127YhurX20STvSnKzJA8bX19Vpyc5Kcm1SS7b2uoAAADYibZtiO49o18+s6r2Lr5YVcenm3U7SQ611jbkfk+uiQYAANjdZiZEV9WpVfW2xUeSU/tVTx97/Wtaay9PckGSE5K8r6qOVNUrklyd5O7p7gd9/kbV6JpoAACA3W2WbnF1myT3XeL1k1faqLV2TlW9Ocm5SU5PcpMkVyV5UZILNuooNAAAAMxMiG6tXZKk1rjtRUku2tCCAAAAYMzMnM69HbgmGgAAYHcToifgmmgAAIDdTYgGAACAgYRoAAAAGEiIBgAAgIGE6AmYWAwAAGB3E6InYGIxAACA3U2IBgAAgIGEaAAAABhIiAYAAICBhGgAAAAYSIiegNm5AQAAdjchegJm5wYAANjdhGgAAAAYSIgGAACAgYRoAAAAGEiIBgAAgIGEaAAAABhIiJ6AW1wBAADsbkL0BNziCgAAYHcTogEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIXoCVbW/qg4vLCxMuxQAAACmQIieQGvtSGvtwNzc3LRLAQAAYAqEaAAAABhIiAYAAICBhGgAAAAYSIgGAACAgYRoAAAAGEiIBgAAgIGEaAAAABhIiAYAAICBhGgAAAAYSIgGAACAgYToCVTV/qo6vLCwMO1SAAAAmAIhegKttSOttQNzc3PTLgUAAIApEKIBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAbatSG6qh5dVW2Jx/nTrg0AAIDZdMy0C5gBP5BkYeT5tdMqBAAAgNkmRCdXtNY+Ne0iAAAAmH279nRuAAAAmNRMheiqultVnVdVL66qq6rqhv465bMHbPuIqnpTVS1U1XVVdXlVnVtVq+3j+6vqq1X14ar6lapydB4AAIAlzVpgfGyS8ybdqKqel+ScJF9K8vok1yc5I8n5Sc6oqrNbazeMbfbxJL+S5B1JvprkB5M8Jcmdkzx6jfUDAACwg81aiH5/kmcluTzJFUn+IMnpK21QVQ9NF6CvTfKA1trV/eu3T/KGJGcleVyS3xndrrX2uiSvG3np/1bVQpKnVtWvttb+bkP2CAAAgB1jpk7nbq29sLX2hNbaSycIsU/ql09cDNB9X59Id2Q7SQ4OOK07SV7aL08dODYAAAC7yEyF6ElV1UlJTkvylSQvG1/fWrs0yceSnJDkfltbHQAAADvNtg7RSe7dL69srX1xmTbvHGu7kv+SpKU7lRwAAAC+zqxdEz2pO/fLa1Zo85GxtkmSqnpdkr9Odx32DekmFjsnyR+01v5+vJOqOpDkQJKceOKJmZ+fX1fhm+3+x4/Po7axZn3/AQAANsN2D9HH9svPr9Dmun553NjrH0zyY0lOSvc+XJ3kiUl+e6lOWmuHkxxOkn379rU9e/asreItctknr9zU/md9/wEAADbDdg/Ra9Za+7kkPzftOnaCPQcv3tT+5w+duan9AwAADLXdr4lePMp86xXaLB6t/twm1wIAAMAOt91D9Hy/vNMKbe441nbNqmp/VR1eWFhYb1cAAABsQ9s9RL+7X55SVbdcps19xtquWWvtSGvtwNzc3Hq7AgAAYBva1iG6tfbRJO9KcrMkDxtfX1Wnp5s47Nokl21tdQAAAOw02zpE957RL59ZVXsXX6yq45M8v396qLW27ns+OZ0bAABgd5upEF1Vp1bV2xYfSU7tVz197PWvaa29PMkFSU5I8r6qOlJVr0h3y6q7J3llkvM3oj6ncwMAAOxus3aLq9skue8Sr5+80kattXOq6s1Jzk1yepKbJLkqyYuSXLARR6EBAABgpkJ0a+2SJLXGbS9KctGGFgQAAAAjZup0bgAAAJhlQvQETCwGAACwuwnREzCxGAAAwO4mRAMAAMBAQjQAAAAMJERPwDXRAAAAu5sQPQHXRAMAAOxuQjQAAAAMJEQDAADAQEI0AAAADCREAwAAwEBC9ATMzg0AALC7CdETMDs3AADA7iZEAwAAwEBCNAAAAAwkRAMAAMBAQjQAAAAMJEQDAADAQEL0BNziCgAAYHcToifgFlcAAAC7mxANAAAAAwnRAAAAMJAQDQAAAAMdM+0CYBJ7Dl686WPMHzpz08cAAAC2J0eiAQAAYCAhGgAAAAYSogEAAGAgIXoCVbW/qg4vLCxMuxQAAACmQIieQGvtSGvtwNzc3LRLAQAAYAqEaAAAABhIiAYAAICBhGgAAAAYSIgGAACAgYRoAAAAGEiIBgAAgIGEaAAAABhIiAYAAICBhGgAAAAYSIgGAACAgYRoAAAAGEiInkBV7a+qwwsLC9MuBQAAgCkQoifQWjvSWjswNzc37VIAAACYAiEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGCgXR+iq+rYqvqHqmpVtW/a9QAAADC7dn2ITvLUJMdMuwgAAABm364Oj1V1jyQ/neTnk7xgyuUwY/YcvHjTx5g/dOamjwEAAGyc3X4k+nlJzk/yoWkXAgAAwOybmRBdVXerqvOq6sVVdVVV3dBfp3z2gG0fUVVvqqqFqrquqi6vqnOratn9q6pHJdmb5Nc2cj8AAADYuWbpdO7HJjlv0o2q6nlJzknypSSvT3J9kjPSHWE+o6rObq3dMLbNXJJnJXl8a+26qlpv7QAAAOwCM3MkOsn70wXbh6c7QnzpahtU1UPTBehrk9yztfbg1tpZSU5O8sEkZyV53BKb/lqSq1trL9mg2gEAANgFZuZIdGvthaPPBx4dflK/fGJr7eqRvj5RVY9NckmSg1X13MWj0VV1SrrJxL63qm7bb3Ls4rKqjmutfW7tewIAAMBONTMhelJVdVKS05J8JcnLxte31i6tqo8lOTHJ/ZK8tV91crr9fsMS3b4hyXuT3GszagYAAGB727YhOsm9++WVrbUvLtPmnelC9L1zNES/Ocl3j7W7V5LnpDtCfcUG1wkAAMAOsZ1D9J375TUrtPnIWNu01j6V7jTvrxk5dfyK1trlS3VUVQeSHEiSE088MfPz8xMXvJXuf/wNqzdah9H930ljjY63lWMBAADbw3YO0YvXMX9+hTbX9cvj1jtYa+1wksNJsm/fvrZnz571drmpLvvklZva/+j+76SxRsfbyrEAAIDtYTuH6A3TWrskiftcAQAAsKJZusXVpBaPMt96hTaLR6s3ZLbtqtpfVYcXFhY2ojsAAAC2me0couf75Z1WaHPHsbbr0lo70lo7MDc3txHdAQAAsM1s5xD97n55SlXdcpk29xlrCwAAAGu2bUN0a+2jSd6V5GZJHja+vqpOT3JSkmuTXLa11QEAALATbdsQ3XtGv3xmVe1dfLGqjk/y/P7podbahtyryDXRAAAAu9vMhOiqOrWq3rb4SHJqv+rpY69/TWvt5UkuSHJCkvdV1ZGqekWSq5PcPckrk5y/UTW6JhoAAGB3m6VbXN0myX2XeP3klYIjav4AACAASURBVDZqrZ1TVW9Ocm6S05PcJMlVSV6U5IKNOgoNm2nPwYs3fYz5Q2du+hgAALDTzUyIXs+9mltrFyW5aEMLAgAAgDEzczr3duCaaAAAgN1NiJ6Aa6IBAAB2NyEaAAAABhKiAQAAYCAhGgAAAAYSoidgYjEAAIDdTYiegInFAAAAdjchGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICF6AmbnBgAA2N2E6AmYnRsAAGB3E6IBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIXoCbnEFAACwuwnRE3CLKwAAgN1NiAYAAICBhGgAAAAY6JhpFwBsvT0HL97U/ucPnbmp/QMAwLQ4Eg0AAAADCdEAAAAwkBANAAAAAwnRAAAAMJAQPYGq2l9VhxcWFqZdCgAAAFMgRE+gtXaktXZgbm5u2qUAAAAwBUI0AAAADLSh94muqu9J8u1JrknyZ621r25k/wAAADBNEx+JrqqfrKoPVNV3jr3++0lel+Q3kvxJkr+qqptuTJkAAAAwfWs5nfuHk5yQ5O2LL1TV/ZP8eJLrkrwkyYeTPCDJIzagRgAAAJgJawnRd0/y/tba9SOv/ZckLcmPtNb+a5L7JvlCksesv0QAAACYDWu5Jvp2Sd469toDkny2tfbqJGmtfbqq3pTk36+zPmCb23Pw4k3tf/7QmZvaPwAAjFrLkehvSHLzxSdVdask90jylrF2n04XuAEAAGBHWEuI/ock9xp5/r1JbpIbh+jbJvnsGusCAACAmbOWEP26JHeqqudV1UOSPDPd9dCvGmt3ryQfWWd9AAAAMDPWEqJ/Pcknkzw2yZ8luWuSi1prH1hsUFX3TnJibnztNAAAAGxbE08s1lr7eB+SfzLJ7ZO8I8kfjTW7R5I/T/Kn665whlTV/iT79+7dO+1SAAAAmIKJQ3RV3SbJ51trv7pcm9baH+XGwXrba60dSXJk3759PzntWgAAANh6azmd+5+T/NVGFwIAAACzbi0h+nNJrt7oQgAAAGDWrSVEfzDJSRtdCAAAAMy6tYTo30/ynVV12kYXAwAAALNs4hDdWvuDJM9P8n+r6olVddequvnGlwYAAACzZS2zc3915OnT+0eqaqnmrbU28RgAAAAwi9YScJdMyxvQFgAAAGbaxCG6tbaW66gBAABg2xOIAQAAYCAhGgAAAAZac4iuqr1V9ayqenNV/b+q+o2RdfetqgNVdduNKXPjVdUP97V/qqq+VFV/V1W/WVVz064NAACA2bSmmbOr6seTPC/JzfqXWpLbjTS5VZILklyf5A/XU+Am+sYkb0zy7CSfSXLPJE/tl983vbIAAACYVWu5xdV3JHlBkuuSPDldEH37WLNLkywkeUhmNES31l449tIlVfWlJC+oqm9prf3jNOoC1m7PwYs3tf/5Q2duav8AAMy+tRyJfkK6I88/2Fq7LLnxPaJbazdU1buTfNu6K9xan+qXN1uxFQAAALvSWq6Jvn+SdywG6BVcm+QOk3RcVXerqvOq6sVVdVVV3VBVrarOHrDtI6rqTVW1UFXXVdXlVXVuVa24j1V1k6q6RVWdluSXk/xFa21+kroBAADYHdZyJHouyT8MaHfsGvp/bJLzJi2oqp6X5JwkX0ry+nTXYp+R5PwkZ1TV2a21G5bZ/NPp9ilJXpvkEZOODwAAwO6wliPRn0xy5wHt7pbkYxP2/f4kz0ry8CR7011bvaKqemi6AH1tknu21h7cWjsryclJPpjkrCSPW6GLByb5jiQ/leSUJEeq6iYT1g0AAMAusJYj0W9JcnZV7WutXb5Ug6r63iR3TTI+edeKxif7Gr/WehlP6pdPbK1dPdLXJ6rqsUkuSXKwqp671NHo1tp7+h/fWlVXJLk8XfB++SS1AwAAsPOt5Uj0c5JUkldU1feNX3NcVQ9I8qIk/5rkuesvcXlVdVKS05J8JcnLxte31i5NdzT8hCT3G9Dle5LckO4oOAAAAHydiUN0a+3t6WboPinJa9JdU9yS/FBVfSLJG5KcmOQJrbX3bWCtS7l3v7yytfbFZdq8c6ztSu6f7j35+/UWBgAAwM6zltO501r7rar6QJKnJrlP//Jt++X7kjyltfYX6y9vVYvXZl+zQpuPjLVNklTV69JNQnZlugnJ7pXkF5P8TZJXjndSVQeSHEiSE088MfPz8+upe9Pd//jl5lHbGKP7v5PGGh1vp461FePthrEAANid1hSik6S19pokr6mqb0oXUG+S5KOttX/cqOIGOLZffn6FNtf1y+PGXn9HkkfmaLieT/J7SZ7dWvvKeCettcNJDifJvn372p49e9ZW8Ra57JNXbmr/o/u/k8YaHW+njrUV4+2GsQAA2J3WHKIXtdY+ne6U7m2ltfaUJE+Zdh0AAABsH+sO0aOq6uQk90xyzXIzd2+wxaPMt16hzeLR6s9tci0AAADscBOH6Kr64SQ/keRp/SRji68/JcmvpJu5O1X1x621R25UocuY75d3WqHNHcfarllV7U+yf+9ek3fDbrfn4MWb2v/8oTM3tX8AANZmLbe4emSSB6SbQCxJUlX3SPK0dLeHekuSf07yI33g3kzv7penVNUtl2lzn7G2a9ZaO9JaOzA3N7fergAAANiG1hKi753kva21L4y89sh0t7n6idbaA9IF1+uT/OT6S1xea+2jSd6V5GZJHja+vqpOT3crrmuTXLaZtQAAALDzrSVEf1OSj429dnq665MvSpLW2t8neXOSb1tXdcM8o18+s6q+dp51VR2f5Pn900OttXXf+6aq9lfV4YWFhfV2BQAAwDa0lhB98/TXPSdJVd0s3T2WL2ut/etIu2uT3H6Sjqvq1Kp62+Ijyan9qqePvf41rbWXJ7kgyQlJ3ldVR6rqFUmuTnL3dPd8Pn+yXVya07kBAAB2t7XMzv3xdOF00QPSBeu3jLU7Nsm/TNj3bZLcd4nXT15po9baOVX15iTnpjsqfpMkVyV5UZILNuIoNAAAAKwlRF+a5JFV9YQkr03yq+muh37tWLt7JPmHSTpurV2SkaPcE257UfrTyQEAAGAzrOV07l9Pd/3zM9LNeH3fJK9vrb1zsUFV3TXJXZK8fcketinXRAMAAOxuE4fo1tqHknxHkv+V5DVJnprkP401OyPJe5O8ap31zRTXRAMAAOxuazmdO6219yf5sRXWX5Busi8AAADYMVYN0VX1rQP6aUmua619dv0lAQAAwGwaciT6w0M7q6rr0k089puttTeuuSoAAACYQUNC9CSzZR+X5MFJHlRVT2itPXttZc2mqtqfZP/evXunXQqwi+w5ePGmjzF/6MxNHwMAYCdYdWKx1to3DHmku8fzaUkOJflKkt+oqn2bXP+WMrEYAADA7raWW1wtqbV2XWvt3a21X0ry8L7vczeqfwAAAJi2DQvRo1prR5JcmeS7NqN/AAAAmIZNCdG9K5PcYRP7BwAAgC21mSEaAAAAdpTNDNGnJPn4Jva/5apqf1UdXlhYmHYpAAAATMGmhOiqelC6EP3mzeh/WszODQAAsLsNuU/0IFV1qyR7kzw0yeOT3JDk/I3qHwAAAKZt1RBdVV9dY99PaK1dvsZtAQAAYOYMORJdE/T3+SRvTPKbrbU3rK0kAKZlz8GLN32M+UNnbvoYAACbZUiIvvOANi3JF5J8prV2w/pKAgAAgNm0aohurV2zFYUAAADArHOf6Am4xRUAAMDuJkRPwC2uAAAAdjchGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSoidQVfur6vDCwsK0SwEAAGAKhOgJtNaOtNYOzM3NTbsUAAAApkCIBgAAgIGOmXYBAOxOew5evOljzB86c9PHAAB2F0eiAQAAYCAhGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAY6ZtoFAMBW2HPw4k3tf/7QmZvaPwAwGxyJnkBV7a+qwwsLC9MuBQAAgCkQoifQWjvSWjswNzc37VIAAACYAiEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgoGOmXQAA7DR7Dl68qf3PHzpzU/sHAJbnSDQAAAAMJEQDAADAQEI0AAAADLRrQ3RVPayqXllVH62qz1fV31TVY6tq174nAAAArGw3Tyz2+CTXJPnFJJ9I8t1JfjfJXfrXAAAA4Ovs5hC9v7X2TyPP31BVxyb5mar6H621L0+rMAAAAGbTrj11eSxAL3p3klsk+cYtLgcAAIBtYKZCdFXdrarOq6oXV9VVVXVDVbWqOnvAto+oqjdV1UJVXVdVl1fVuRNe4/xdST6T5JNr3gkAAAB2rFk7nfuxSc6bdKOqel6Sc5J8Kcnrk1yf5Iwk5yc5o6rObq3dsEof+5I8JsnTWmtfnbQGAAAAdr6ZOhKd5P1JnpXk4Un2Jrl0tQ2q6qHpAvS1Se7ZWntwa+2sJCcn+WCSs5I8bpU+Tkjyp0nekeSZ69kBAAAAdq6ZOhLdWnvh6POqGrLZk/rlE1trV4/09YmqemySS5IcrKrnLnU0uqrmkrwmyReSPKS1dv0ayweALbfn4MWb2v/8oTM3tX8A2G5m7Uj0RKrqpCSnJflKkpeNr2+tXZrkY0lOSHK/Jba/RZK/SHJ8kh9orX16UwsGAABgW9vWITrJvfvlla21Ly7T5p1jbZMkVXVMkpcmuWeSH2ytXbM5JQIAALBTzNTp3Gtw5365UgD+yFjbRc9Lsj/JE5LcqqpGj1R/oLX2L6ONq+pAkgNJcuKJJ2Z+fn6tNW+J+x+/4jxq6za6/ztprNHxdupYWzGesbbXWKPj7dSxtmK83TAWALD9Q/Sx/fLzK7S5rl8eN/b69/fL31him+9Ody3117TWDic5nCT79u1re/bsmaTOLXfZJ6/c1P5H938njTU63k4dayvGM9b2Gmt0vJ061laMtxvGAgC2f4hes9banmnXAAAAwPay3a+JXjzKfOsV2iwerf7cegerqv1VdXhhYWG9XQEAALANbfcQPd8v77RCmzuOtV2z1tqR1tqBubm59XYFAADANrTdQ/S7++UpVXXLZdrcZ6wtAAAArMm2DtGttY8meVeSmyV52Pj6qjo9yUlJrk1y2dZWBwAAwE6zrUN07xn98plVtXfxxao6Psnz+6eHWmvrvgeIa6IBAAB2t5kK0VV1alW9bfGR5NR+1dPHXv+a1trLk1yQ5IQk76uqI1X1iiRXJ7l7klcmOX8j6nNNNAAAwO42a7e4uk2S+y7x+skrbdRaO6eq3pzk3CSnJ7lJkquSvCjJBRtxFBoAAABmKkS31i5JUmvc9qIkF21oQQAAADBipk7nnnWuiQYAANjdhOgJuCYaAABgd5up07kBgNm15+DFm9r//KEzN7V/ANgIjkQDAADAQEI0AAAADCRET8DEYgAAALubED0BE4sBAADsbkI0AAAADCREAwAAwEBucQUAzJzNvp1W4pZaAKyNI9ETMLEYAADA7iZET8DEYgAAALubEA0AAAADCdEAAAAwkBANAAAAAwnRAAAAMJAQDQAAAAMJ0RNwiysAAIDdTYiegFtcAQAA7G5CNAAAAAwkRAMAAMBAQjQAAAAMJEQDAADAQEI0AAAADCREAwAAwEDHTLsAAIBp2nPw4k0fY/7QmZs+BgBbw5FoAAAAGEiInkBV7a+qwwsLC9MuBQAAgCkQoifQWjvSWjswNzc37VIAAACYAiEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICEaAAAABhKiAQAAYCAhGgAAAAYSogEAAGCgY6ZdwHZSVfuT7N+7d++0SwEAtqE9By/e9DHmD5256WMA7GaORE+gtXaktXZgbm5u2qUAAAAwBUI0AAAADCREAwAAwEBCNAAAAAwkRAMAAMBAQjQAAAAMJEQDAADAQEI0AAAADCREAwAAwEBCNAAAAAwkRAMAAMBAuzZEV9Xeqvq9qnpPVf1rVb1/2jUBAAAw246ZdgFTdEqSM5O8Pd0fE3btHxQAAAAYZjcHxyOttTu21s5O8q5pFwMAAMDs27UhurV2w7RrAAAAYHuZqRBdVXerqvOq6sVVdVVV3VBVrarOHrDtI6rqTVW1UFXXVdXlVXVuVc3UPgIAALB9zdo10Y9Nct6kG1XV85Kck+RLSV6f5PokZyQ5P8kZVXW2I88AAACs16wdpX1/kmcleXiSvUkuXW2DqnpougB9bZJ7ttYe3Fo7K8nJST6Y5Kwkj9u0igEAANg1ZupIdGvthaPPq2rIZk/ql09srV090tcnquqxSS5JcrCqnutoNAAAAOsxa0eiJ1JVJyU5LclXkrxsfH1r7dIkH0tyQpL7bW11AAAA7DTbOkQnuXe/vLK19sVl2rxzrC0AAACsyUydzr0Gd+6X16zQ5iNjbZMkVXWrJA/qn94pyW1GZgF/Z2vtmrH2B5IcSJITTzwx8/Pz6yh7893/+M09c310/3fSWKPj7dSxtmI8Y22vsUbH26ljbcV4xtpeY42Ot1PHAmBzbPcQfWy//PwKba7rl8eNvX58bnwK+OLzxyS5cHRFa+1wksNJsm/fvrZnz54JS91al33yyk3tf3T/d9JYo+Pt1LG2Yjxjba+xRsfbqWNtxXjG2l5jjY63U8cCYHNs9xC9Zq21+SSDZi4DAACAZPuH6MWjzLdeoc3i0erPrXewqtqfZP/evXvX2xUAwKbbc/DiTe1//tCZm9o/wCza7hOLzffLO63Q5o5jbdestXaktXZgbm5uvV0BAACwDW33EP3ufnlKVd1ymTb3GWsLAAAAa7KtQ3Rr7aNJ3pXkZkkeNr6+qk5PclKSa5NctrXVAQAAsNNs6xDde0a/fGZVfe1i5ao6Psnz+6eHWmvrvqdEVe2vqsMLCwvr7QoAAIBtaKZCdFWdWlVvW3wkObVf9fSx17+mtfbyJBckOSHJ+6rqSFW9IsnVSe6e5JVJzt+I+lwTDQAAsLvN2uzct0ly3yVeP3mljVpr51TVm5Ocm+T0JDdJclWSFyW5YCOOQgMAAMBMhejW2iVZ472bW2sXJbloQwsCAACAETMVomed+0QDACzNPamB3WKmromeda6JBgAA2N2EaAAAABhIiAYAAICBhGgAAAAYyMRiEzCxGADA9JnEDJgmR6InYGIxAACA3U2IBgAAgIGEaAAAABhIiAYAAICBhGgAAAAYSIieQFXtr6rDCwsL0y4FAACAKRCiJ2B2bgAAgN1NiAYAAICBhGgAAAAYSIgGAACAgYRoAAAAGEiIBgAAgIGE6Am4xRUAAMDuJkRPwC2uAAAAdjchGgAAAAYSogEAAGAgIRoAAAAGEqIBAABgICEaAAD4/9u7+2jJqvLO498foMhri1EkggNI9xjfBlRANDGgOAk6IDE2YjDj4DLBAIlkdAZwuVZCQiaCjG8jSNJxKZOlTCL4ymjUiAptgllg1CjYE4Q0AopAgIbmXfuZP84prBR1b5+q+1K37v1+1qq17zl777P3qdpr133qvEnqyCBakiRJkqSODKIlSZIkSerIIHoESY5Ksm7Tpk2T7ookSZIkaQIMokdQVZdU1QmrVq2adFckSZIkSRNgEC1JkiRJUkcG0ZIkSZIkdWQQLUmSJElSRwbRkiRJkiR1ZBAtSZIkSVJHBtGSJEmSJHVkEC1JkiRJUkcG0ZIkSZIkdWQQLUmSJElSRwbRkiRJkiR1ZBAtSZIkSVJH2026A9MkyVHAUatXr550VyRJkrQI9jn9swu6/Y1n/acF3b6k+eeR6BFU1SVVdcKqVasm3RVJkiRJ0gQYREuSJEmS1JFBtCRJkiRJHRlES5IkSZLUkUG0JEmSJEkdGURLkiRJktSRQbQkSZIkSR0ZREuSJEmS1JFBtCRJkiRJHRlES5IkSZLUkUG0JEmSJEkdrdggOsmaJJ9PsjnJbUnen2THSfdLkiRJkrR0bTfpDkxCkscDXwFuANYCuwPvBp4EvHaCXZMkSZIkLWErMogG3gTsBhxQVbcDJPkJ8NEkZ1bV1RPtnSRJkiRpSVqpp3O/Ari0F0C3Pg48CLx8Ml2SJEmSJC11SyaITvL0JKck+UiSDUm2JKkkazvUPS7J+iSb2mucr0pycpKZ9u8ZwDX9K6rqQeA64BfmvjeSJEmSpOVoKZ3OfSJwyqiVkpwHnAQ8AFwKPAwcDpwLHJ5kbVVtGai2G3DXkM3dCTxh1D5IkiRJklaGJXMkGvgucA5wLLAauGxrFZK8miaAvgX4D1V1ZFW9ClgDfA94FfB7C9ZjSZIkSdKKsmSORFfVB/uXk3Sp9rY2Pa2qru3b1o+TnAh8FTg9yfsHjkbfCTx+yPZ2AzaM0m9JkiRJ0sqxlI5EjyTJXsDzgYeAiwbzq+oy4GZgD+CQgezv0VwX3b+97YH9MIiWJEmSJM1gaoNo4LltenVV3T9DmSsHyvZ8juZ66Z/rW/cqYPs2T5IkSZKkR1kyp3OPYd82vWGWMj8YKNvz5zTXSn86yZnA7sC7gb+uqmsYIskJwAkAe+65Jxs3bhyz24vjhbsP3kttfvXv/3Jqq7+95drWYrRnW9PVVn97y7WtxWjPtqarrf72lmtbi9Gebc1vW4vpbZ/4zoK38Y5ff86Ct7GSLPRn1v95LWZb02iag+id2/TeWcpsbtNd+ldW1V1JXgr8L+ATwP3AXwGnzrShqloHrAM48MADa5999hmv14vkiluvXtDt9+//cmqrv73l2tZitGdb09VWf3vLta3FaM+2pqut/vaWa1uL0Z5tzW9bi2mxx6LmznG/dExzED0nVfXPwBGT7ockSZIkaXpM8zXRvaPMO81Spne0+p75aDDJUUnWbdq0aT42J0mSJEmaMtMcRG9s071nKfPUgbJzUlWXVNUJq1atmo/NSZIkSZKmzDQH0d9s02cl2WGGMgcNlJUkSZIkaWxTG0RX1Y3APwKPBY4ZzE9yKLAXcAtwxeL2TpIkSZK0HE1tEN16R5uenWR1b2WS3YEPtItnVdW8PJvAa6IlSZIkaWVbMkF0kucl+XrvBTyvzfrTgfWPqKqLgfOBPYDvJLkkySeAa4FnAp8Czp2vPnpNtCRJkiStbEvpEVe7Ai8Ysn7NbJWq6qQkXwNOBg4FtgU2AB8Czp+vo9CSJEmSJC2ZILqqvgpkzLoXAhfOa4ckSZIkSRqwZE7nngZeEy1JkiRJK5tB9Ai8JlqSJEmSVjaDaEmSJEmSOjKIliRJkiSpI4NoSZIkSZI6MogegTcWkyRJkqSVzSB6BN5YTJIkSZJWNoNoSZIkSZI6MoiWJEmSJKkjg2hJkiRJkjoyiB6BNxaTJEmSpJUtVTXpPkydJLcBN0y6H32eCNw+6U5I88gxreXGMa3lxjGt5cYxrUF7V9WThmUYRC8DSa6qqgMn3Q9pvjimtdw4prXcOKa13DimNQpP55YkSZIkqSODaEmSJEmSOjKIXh7WTboD0jxzTGu5cUxruXFMa7lxTKszr4mWJEmSJKkjj0RLkiRJktSRQfSUSnJckvVJNiXZnOSqJCcn8TPVkpPkgiQ1y2vDDPW2acf1Ve0439SO+99Y7H3QypPk6UlOSfKRJBuSbGnH69oOdceao5MckeSLSe5Icl+S7yZ5e5Lt52/PtFKNM6bHnb/bus7hWjBJHpPk8CTvasfY3UkeSnJzkouTHLaV+s7TGtt2k+6ARpfkPOAk4AHgUuBh4HDgXODwJGurassEuyjN5O+A7w9Z/6PBFUm2BT4BvBK4G/gisD3NWL8wySFVdcoC9lU6ERh5jI07Ryc5FTgb+CnwVeBO4FDgT4AjkxxeVfeNtysSMOaYbnWev8E5XIviUOBv279vAS4H7gWeCbwaeHWSM6vqDwYrOk9rzqrK1xS9aCaFovnSWtO3/snANW3eKZPupy9f/S/ggnZsHj9Cnbe2da4Gnty3fg3Nl2UBR09633wt3xfwW8A7gdcA+9H8w1TA2lnqjDVHAwcCW2j+AXxB3/qdgcvaeu+Z9Hvia7pfY47pkefvtp5zuK8FfQEvBS4GXjwk71jgJ+04e8lAnvO0rzm/PPV3+rytTU+rqmt7K6vqxzS/MAOc7mndmmbtEYxT28UT2/ENQDvuT2sX377YfdPKUVUfrKpTq+pjVXVdx2rjztGnAwHOrqp/6Ku3GXgDzT9uJyV5/Dj7IsHYY3pkzuFaDFX15apaW1Xrh+T9Nc0PQAC/OZDtPK05M9CaIkn2Ap4PPARcNJhfVZcBNwN7AIcsbu+kefVCYHfgpqq6fEj+RTSnXh2UZM9F7Zk0g3Hn6CSPBV7eLn50SL3rgSuAxwKvmPeOS/PPOVxLwTfbdK/eCudpzReD6Ony3Da9uqrun6HMlQNlpaXkJUnenWRdkjOT/OoMZ030xu+VQ/Ko5nqjq9vFAxaio9IYxp2jnw7sCNwxy9FB53ZNWtf5G5zDtTSsadP+6/adpzUvvLHYdNm3TW+YpcwPBspKS8nrh6y7Jslrq+o7feu6jvUDcKxr6Rh3jt53IK9rPWkxdZ2/wTlcE5ZkD+D4dvHjfVnO05oXHomeLju36b2zlNncprsscF+kUXwLeDPNHTN3Bp4CHAl8u133pYFT+hzrmkbjjlvHu5ayUedvcExrgpJsB3wEWAVcWlWX9GU7T2teeCRa0oKrqvcOrLoX+GySv6W5o+UhNDf6+N3F7pskaWbO35pCf0bzuKobefRNxaR54ZHo6dL7hWunWcr0fim7Z4H7Is1ZVT0EvKNd7L8Rh2Nd02jccet419SZZf4Gx7QmJMn7gDfSPEbt8Kq6ZaCI87TmhUH0dNnYpnvPUuapA2WlpW5Dm/afDrixTR3rmiYb23TUcdv7+9+NWE+atGHzNziHawKSvIvm0oPbaALoa4cU29imztOaE4Po6dK7Vf+zkuwwQ5mDBspKS93Ptenmw4K4ZwAACe5JREFUvnX/2KYHMUSSHYFnt4uOdS0V487RG4D7gSck2W+GegcPqSdN2rD5G5zDtciSvBN4C/CvwMuq6poZijpPa14YRE+RqrqR5ovpscAxg/lJDqV5Ft4tNM+qk6bBa9q0/1EoV9D8krxXkl8eUucY4DHAlVV18wL3T+pk3Dm6PS32b9rF1w2p9zSa5+4+BHx23jsujW/Y/A3O4VpESc4C/jtwJ/Afq+qfZirrPK35YhA9fXrXH52dZHVvZZLdgQ+0i2dV1ZZF75k0RJIDkhyZZNuB9dsleSvNqVcA7+nlVdVPgXe2i+e347tXbw1wVrv4Pxau59JYxp2jzwIKOC3JwX31dgY+RPN9/YGqumvBei4NGGf+BudwLZ4kfwKcBtxFE0B3OQrsPK05S1VNug8aUZIPACcCDwBfAh6muQvhrsCngLXtF5g0cUl+DfgkcAfNr7+30pwC+ByaR6VsAU6vqnMG6m3b1jsKuBu4lObIxcuAxwHvr6o3Iy2QJM/jZ/9QQfM4n12Aa2nGMwBVdchAvbHm6CSnAmcDPwW+TPNP4aHA7sA/AC+tqvvmafe0Ao06psedv9u6zuFaUEleCXy6XbwKuHqGohuq6qz+Fc7TmiuD6CmV5DjgZJovsm1prtX4EHC+R6G1lCTZFziF5lqhvWn+ASvgJmA9cF5VfWOGutsAJwFvAH6B5kvrn2h+6b1w4XuvlSzJYcBXtlauqjKk7lhzdJIjgLcCB9IEGtcDFwL/s6oeHH0vpJ8ZdUzPZf5u6zuHa8EkOR74cIeil1XVYUPqO09rbAbRkiRJkiR15DXRkiRJkiR1ZBAtSZIkSVJHBtGSJEmSJHVkEC1JkiRJUkcG0ZIkSZIkdWQQLUmSJElSRwbRkiRJkiR1ZBAtSVIHSTYmqSSHTbovy0X7ftak+yFJ0igMoiVJWmRJzmgDyDMm3ZeFkuSCdh+Pn3RfJEmaT9tNugOSJGnFesakOyBJ0qgMoiVJ0kRU1YZJ90GSpFF5OrckSWPqP2U5yeokFyb5cZIHk2xIclqSbQbqFPCH7eIf9q4LHnZ6d5Kdkpya5Mokdye5P8nV7engOw/pzyOniSfZO8mHk9yU5CdJ3tuWeUyS/5zk/yT5f0nuSXJfkmuSnJ3kCbPs72OSnJDkK0nuaPfzB0n+b5LXtWX2affxv7TVPjywj8f3vxczXROd5Iltfza0+313kq8nOSnJow4CtJ9BtZ/JLknOSfIvbR9vTnL+TPuW5LVJvtzu08NJbk/ynSTnJdlvpvdDkrQyeSRakqS5OwB4H3A78BVgd+DFwFnAXsDv9ZX93235/YFvA9/qy3vk7yR7AV8AngncBlwBPAAcRBOEvyrJYVV155D+rAG+2Zb/O5rv+7vavCcDfwncCWxo29wVOBA4FVib5AVVdXv/BpPsBnwWeCHwYLvdW4GnAL8IPBv4KLC53cdfAvZry32/b1P9fw+VZDXwZeCpwC3AJcCOwEuA89p9P7KqHhxSfVXb5p7A5cB32778DnBwkkOq6uG+ts6geT8fBv4e+CHweGAf4CRgPXDd1vosSVo5DKIlSZq7U4A/Av64qrYAJPllmoD6pCTvrKobAarq+DZw2x/4VFWdMbixJAE+RhNAnwucWlX3t3k7AOuA3wTeAxw/pD/HARcAb6qqhwbyNgGvBD4/EEzuQBOgvgE4EzhxoN6HaQLoK4C1VfXDvrqPowlwaYPv45NcQBNEf7CqLhjSx9lcSBNAXwS8vqoeaNt5KvAl4GXAGcDbhtT9NeBzwIuqanNb7ynA14HnAa+hCfZJsj3NDwebgedX1T/3byjJGuAnI/ZdkrTMeTq3JElzdyXwR70AGqCqLqc5krwNbYA5giNoAtavA6f0Auh2u/fTHFW9FXhde4R40L8Cbx4SQFNV91TVJf0BdN92f5cmaHx1f16SA4CjgXuAo/sD6LbuA1X1NyPu41BJXkxztP0e4Hd6AXTbzo00P1gAnNwG74M2A2/sBdBtvR/S/BgBcHhf2V2BHYDrBgPott61VfUvc9kfSdLy45FoSZLm7nNVNeza3g3Ay2lOeR7FK9r04/2BeU9V3ZvkqrbcQcAXB4p8qaruma2BJM+lCSj3AXYC0mY9BDwpyW59p4of0aafqarbRtyXUR3appdU1R2DmVX1+SQ/An4eeD7Nqdv9vlFVtwzZbu8mZo98FlV1W5KNwP5J3gX8hTc7kyRtjUG0JElz94MZ1t/dpsOOmM7maW16TpJztlL2SUPW3TBT4faGZB+lOaV7NrvSXDcNsHebLkaAuWebznYE+HqaIHrPIXmjfhavBy4G3gK8JcltNGcAfAH4SFVt6tJpSdLKYRAtSdLcPepo8Rxt26aXARu3UnZYwHz/kHU976AJoK8BTgeuAm7vnd6d5Ic0AWr66gy9g/YCG7fNkT6LqlqfZF/gSOAw4EXt30cBZyT5lar65ph9kSQtQwbRkiQtPTe26UVVdd48b/uYNj22qr7bn5FkJ2CPIXV6R3efPs99GebmNn3aLGV6eTfPUqazqrqP5kZuHwNI8vM0N207luZmay+aj3YkScuDNxaTJGnx9W74NdOP2b2bdB0zQ/5c9J6VfOOQvOP4t0ege77QpkcneWLHdra2jzO5rE2PGnbTtCS/SnOkfDPwjRG33UlV/Qh4e7u4/0K0IUmaXgbRkiQtvt4R1GfMkP8pmgDx0CR/luQJgwWS7JHkt8dou3dd80kD2zuQ5lTvR2lPZ74E2AX4ZHuktr/u45K8fKDa1vZxqKpaT3O3812A89rHUPXa2RN4b7t4bv+du8eRZO8kv5Vk1yHZR7XpjNeXS5JWJk/nliRp8X0BuA/49SSXA9cBP6W5+/VnqmpLkt7zjt8EHJfk2zRHjx8H/HuaZ0jfCvzFiG3/Mc3zl/80ybHA92juWP1LwF8Bv8jPbiTW73jg822565N8Dbitrbs/zfOn9+kr/2ngD4DfT/Js4Caa65w/VFV/v5U+HkfzjO3fAA5Lsh7YkeZRYTsBl9I8J3qudqN5/85L8i2am5ltQ/PePgt4mOY50pIkPcIgWpKkRVZVtyQ5kibIfC5NYBqaQPMzbZmbkhwMvBF4DfAc4AU0z4C+GXgX8Mkx2r44yUvatvcHVgPXAr9Pc/3v9TPUu6N9hvNv0wS3BwPbAz8G1gMXDpT/Vhuk/zeaa4p3brO+BswaRFfV99tHcJ1K83zqo2kC2quBvwTWDT7nekzXAf+V5oZiz2pfW2je33XA+6rqmnloR5K0jGT4Yy0lSZIkSdIgr4mWJEmSJKkjg2hJkiRJkjoyiJYkSZIkqSODaEmSJEmSOjKIliRJkiSpI4NoSZIkSZI6MoiWJEmSJKkjg2hJkiRJkjoyiJYkSZIkqSODaEmSJEmSOvr/p2gaUe2TRD8AAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":283},"id":"l2hCfy8j2LFx","executionInfo":{"status":"ok","timestamp":1630826722962,"user_tz":-330,"elapsed":725,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"8883efd4-11c0-4d07-ec34-241427548bb6"},"source":["print(\"Interactions distribution per item:\")\n","compute_quantiles(interactions_per_item)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Interactions distribution per item:\n"]},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
quantilevalue
00.011.00
10.101.00
20.253.00
30.50134.00
40.75423.25
50.90958.00
60.992858.99
\n","
"],"text/plain":[" quantile value\n","0 0.01 1.00\n","1 0.10 1.00\n","2 0.25 3.00\n","3 0.50 134.00\n","4 0.75 423.25\n","5 0.90 958.00\n","6 0.99 2858.99"]},"metadata":{},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":576},"id":"BkgMuktb2PDG","executionInfo":{"status":"ok","timestamp":1630826723781,"user_tz":-330,"elapsed":833,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"dfae38be-f4d0-4674-9290-1465a915bb78"},"source":["plot_interactions_distribution(interactions_per_item, \"item\", \"Items\")"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA9MAAAJECAYAAAAVGXcKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde7xtZV0v/s83UbyAS7yQBeZSN1haBrJJtDpolKm4+0lC9qM8Rzu2E8ywNMU8GZpHt1l6UhTbeYxKrdTK2mJaXsAbpXhLEdLUBYqCqbgUAcF4zh9zrPZ0um5jrTXXXJf3+/Uar7HmHM8Yz3fMOdfa+zPHGM+o1loAAACA5fuuSRcAAAAAm40wDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTwJZWVY+pqjY0PbpH++n1qXLtDe/HpGvZSKrqFlX11Kr6YFV9fei9/j89tjG3zmPGWCo9VdUDt8LvLklVndW9jzOTrgVgMcI0sN08s6puNukimJhXJXl+kqOTHDThWuZVVTNdkDhr0rVsBFV1bvd6nD/pWpi8qpoe+tLkgZOuB9jehGlgu9mR5L9PugjWX1UdmeSU7uHLktw9ycHd9NRJ1QUAbE7CNLCdfLqb/3ZV3XyilTAJPzz08zNaa59prV3TTTcsdyOtteqmc9e+RFaqtXb+0HszM+l6WLnW2lnd+zg96VoAFiNMA9vJs7v53ZI8dpKFMBG3nvuhtfbVSRYCAGx+wjSwnVyQ5O3dz8+oqlv03cByBp9abCCk0ev9qurAqnpaVX2kqq6pqi9W1Rur6n4j651QVfuq6gtVdX1VXVxVT17u9d9Vdbeq+qPuetzru+28uqruvYx1D6iqX6qqN1fVlVV1Q1X9R1X9Y1X9QlXVAut92yBoVXWXqvrDqvpEVV3bLbvdcuof2e6uqvrbqvp8VX2zqr5UVedX1enznXEwd81tknOHnhselG6mZ/8Lfga6OlpVnds9/vGqekP3un2zqj5TVS+pqu9epM67dk/9zkidC/W5Lu9PVf1QVf2vqnpnt/0bq+qrVfWBqnpOVd1pma/ff+v29d+r6htV9bWq+nhVva6qTp17D+fqS/I/ulWPn+f1OHdou8sagKx7T15TVZd3vwtXV9W/VNWZVbXgdfSreW+HtnHzqnp8Vb1j6DX8SlX9Ww1+v3+tqu64nNdxZLvfdp19VT22qt7Tbfsb3Xt0RlUdsIxtHdntx8drMEjftV19L66q7+tRw6lV9daquqqqbqp+g/zNOwBZ9/gzQ0+9Y57PxPQ82zuk++z+S1V9uXu/PldVf1FV91+kjm+7Xr+qjq2q19bgb891VXVJVT2jqm410tdZNfgb/Y2uv7+pqnstd/+BTaS1ZjKZTFt2SvKYJK2bppM8YOjx6Uu1n2f53LLHLNLnAxfaRlfD3LKfSfL+ocfD0zeT/FS3zm8v0KYl+bNl7PePJLl6kX5+dpF9+b4kH16k/5bkTUlus0QNxyX5yjzr3q7He3lgktcuUctHkhw2st65S6wz0/MzteBnIMn53bJzk/x6kv9coM/LVlDnd/S5Xu9PBqfIL1XbVUmOWeR1u1WSVy9jO0fNU99C07nL+b3rlleSFy2xvcuS3GuB+lf83nbrH5TkwmXs08kr+Ds30617VgaD7C207Qvm+ywMbefJSW5cZP1vJHn4EjU8a4H3+f/02J+zMs/v5lAfi02jf3MflOTLS6zz7AXqOLdbfn4GX+os9Nq8JcnNMzjr6RMLtPlakh/u+96aTKaNPU28AJPJZBrnlHnCcZI3d48/l+TApdqPLJ9b9phF+nzgIv+xmx5a9unuP1i/3v0n7A5J/r8knx9afkr386uSHJvk9kl+KMkbhrbz4CX2+zNJvtD9Z/B7k3xPBoOwfaFb/s3MEyCS3DbJv3dtrkzya0numeSQJEcm+a0k183Vt0QNn0tyedfv4Um+O8nDk9yyx3u5d2h7f5vk/t1rdq8kL0jyrW7Zh5LcYmi9AzMIMr8ytP5BQ9Ote36mFvwMZH/g+nSSm7o6f7Sr824ZXGpwU9fmNSPrztV5Wbf8uSN1HpTkgEm8P0nuk+StSZ6Y5MczGMjv9t1r/z+TXNJtZybJrRZ43f52qL83J3lo91m8Q5KjkjwpyQezP0wf0O3zXDh81zyvx4FD23/g0Pbn+939raHl70ryE0numOQe3bJrh16LQ9byve3W/91u2beSPKfb50O7Gn4oyS8leWOSk1bwd24m+3/X5/5eHNPVdt8kfz6073++wDaeMNTmr5Oc0H0O7pjkwUne0y27LskPLlLD57r5KzP4m3WHJD+Q5AE99uesuc/TyPO37j5zc3U+dJ7PRA21v2+S67u2H0zyqAy+gDqke33+ZGhbj5unjnO7ZVdk8Hfyn5L8t26fjkzyx0PrPz7JRRn8/X5sBr9HhyZ5dAZ/51uS9/Z9b00m08aeJl6AyWQyjXPK/GH6R4aeO2Op9iPL1zJM35jkR+dZ/ydH2pw9T5ubZ/Cf+pbkL5bY72uSHDlPmyO7ZS3JvnmWv7hb9uX5XouuzU8P9bNzkRq+lOTwVbyPRw9t69ULtDl9qM2vLfaarPIztZww3ZLsXWD9udf1m0luO8/ymW75WUvUsZHen4OyP9j/0jzLf36or0WPUGboC4Pu8bndeucvsd5iv3ff3b3erXuPbj7P+g8bWv+Fa/3eJvlAt+xFq/n8LdDvzDJqGw5+x4ws+57sD51/sMD6N8/gyHZL8sYlanj+KvfnrMwTprtl00P9PHCJ7Xyka3dhhr5gG2nz3K7NFzPyRVC+/WyRNya52Tzrvzv7/1bPJrnHPG0eN7Sde671+28ymSY3uWYa2HZaa+/L4D9GSXLm8PVu6+yvWmvvmef5t2UQbpLBf8p/a7RBa+3GJH/TPbzf6PIRZ7fWPjHPNj6R5KXdw4cNX+tZVbfJ4IhjMhj5ema+DbfW3pLkHd3DX1ikhhe01j63RJ2LmavlhgyOYM5Xy8sy+M9zMvjP6yRdm4Vvt/Un3fwW+fYRxpdto70/rbVrsv/z+FPzNDmjm38qyVOW2Na3VlrHIh6dweudDL5ouXGeft+U5O+7h4+thccjWOl7O3e98hXLqnhlrk/ytAWWPbVbnnznAIyPz+DMiM8ttH73mv129/BhtfB4B1cn+Z3lFjwuVfWgDM6oSJLHtoVH7P/dDE5fv1MGR+AX8uuttf+c5/m/7OYHJHlxa+1T87T5qwyCdLL032tgExGmge3qmRn85+bOGRzRnIS3zPdka61l/228/rm19rUF1p/7T9udl+jnb5ex7LsyuG52zgOyf/Trd1bVQQtN2R9gdy7Sz3lL1LiUH+vm57fW/mORdq/v5j+4yH/218M/t4VHDP+3oZ+Xeu8Wsu7vTw2cUlV/3Q22NTdI2dwgZr/ZNT1yZL2DMzjdNxmcVTCOsLyUuc/PJ1pr/7pIu9d189sl+cEF2qz0vf1QN39qDQbRW9bggT2d31q7er4F3fMXdA9/dGTxT3bzC5LccpHP0iVdu8rgNOn5vL21dv0Cy9bT3D5dnuRzi+zTzZJc2rVd6Hfk0621Ty6wbDg8L/Q3/etJ5v5urfR3HtiAlhzVEWAraq19qKrekOSkDP5z+/LW2jfWuYzPL7Lsum7+hWW0WerI+qXLXHbXoZ/vOfTzxUtsf85iozl/epFlyzFX28eXaDdXa2VwbeSkboG14HvbWru29g+wfeuF2i1hXd+fLnT8fQaDOS1lauTxdAaBJRkMljYJfT8/c+t8ZJ42K31vz8pgTIQ7ZfBaXl1V78rgNOF3tNYuWqK25Vjsd31u+U/n23/Xk/2fp1/I4mcwDFvo87Ta3/W1MrdP35fk68tcZ6F9Ws7f6mRt/l4Dm4gj08B29jsZHJ0+NIOBldbbfKcMrqTNUq5Z5rKDh34eDUTLccuFFrTWrl3B9obN1bbYviTf/p/mgxdsNX7Lfd/mvW3VMqz3+/Oi7A/Sf5LB9cX3yGAgpoO7aU+3fPSL+tsO/bzcULPW1vLzs6L3tjsV/75J/jSD04oPyWBE/99L8v4a3CpsuUF2IUvt39zy0X1by8/Tan/X18pa7tNy3/PltFvp7zywATkyDWxbrbWPVtXrkvxckqdU1UuXWmeZNtrf1oMyGBhnoWVzhoPE8H/Kb91aGz76Mglfz+DU2wXvA9xZaH+2mnV7f7rrsx/dPdzTWnv6Au0WOuK2Eb7gmKthop+f1tpnkjymqnZncErx/TMYNfsnM/hy4lVVdcfW2h+usIvl7t/ovl2Twe/X77XWFrrmerOZ+x15X2vNdcrAWDgyDWx3Z2VwO5s7ZP8gSYuZuxZwsVP1vneVNa2171/mssuGfh4+VfMea1vOisx083st0e7e3bxlcK3kVrWe7889MxicKtk/2NJ8fmiB5z+T/Ufsjlqronqa6ebL/fwMr7PmWms3tNbe21r7g9baw5LcPYP7EyfJM6tqpf8/W+x3fXj5ZSPPz32eNsLv+lqZ26e719C59wBrSZgGtrXW2iVJ/qJ7+OQsfWrg3DVx91ykzUNWW9caO2mRZY/o5jcl+eeh5y/IYCTxZHBv1kl7dzc/vqruuEi7k7v5xxYZJGqjmxtperEBqtbz/Tlw6Od5a6qqwzO4/+536AZf+pfu4alV1ffMjeW8HkuZ+/wcWVULDSyW7P/8fDXJx1bRXy/dSOov7x7ePoNbea3EAxcaeK+qDklyfPdw9C4C/9jNHzzhgfuWY3gk9sU+E3P7dMcM7ikOsOaEaYDkWRkcObtdlr52ei4UnFxV3zF4VFUdn8Fp4xvJr1bVkaNPds/9avfwTa21q+aWdSOIv6J7+OSqGh39d3Rbt62q71mrgufxf7v5gUleuEANv5L9Rz7/eIy1jNuXu/mCZzis8/szM/Tzrnm2fUCSvVn88oYXd/MdSZ6/WGfzjHK95OuxDK/K4LZqSfKH8wX6qnpI9n+59MrW2k2r6O87VNVSR43njgr/Zxa+LGMpt8zgGuz5PD/7rwn+k5FlL83gy5mDk7yiqm6+WCdVtdiXieN2dfbfZmqxz8Q/Zv8XIucM3/pvPlU1XVUHLtYGYJQwDWx73S1P/rx7uNRpjq/s5oclOa+q7ldVh1TVEVV1ZpI3ZYynh67QfyQ5v6r+e1V9Tzc9Osn5SW6TQciY7zrJZ2Rwq59bJXl7Vb2oqu5fVXeqqttX1T27WyW9MoP70y4a6Fajtfbh7A/Ij+5uz3S/ro7vr6o92X/P7A8n+aNx1bIOPtDNH1FVD66qqao6oJuGT1ddl/entfaFJO/qHv5WVT2jqo6sqjtW1QkZ3Bf9odl/26T5tvFXSd7QPfyNqnpTVT2kqu7c1XqfqvrVqroo33m6+NzrcfeqekJVHTr0eizr/zHdF0XP6h7+RJK3VtWDquoOVXX37nf3r7vlVyR5znK229PHq+qtVXV6VR3T7ccdq+q+VfX72X+Lvr9bxYB9M0l+uar+vNvu7avq6Kr6syS/3LV5VWvtA8MrdUfG5+7f/sgk76uqR3evzVRVfW9V/VhV/WZVvT/7X6t11702c5+1J1bVD1fVrec+E0PtWpL/kcEo2kck+UhVPaWqfrD7m31oVR1VVY+rqn1J/j2THbQQ2IQ22iA5AJPy7AxuCbPoEZnW2j9V1d4ku5M8MN9+anQyOJ10T5I3jqHGlXpUBkdp/nSeZTckObW19h23DGqtzVbVgzK49+6PZvCf7SeNthvZ1jg9MYOzB05J8rPdNOpfkzy8tTbuWsbpnCSPy+B039H71j42ybnJur8/p2UQqA/JIGiOhs0XJflaBiPkL+TUDGr/uQzC90OX2fcbM7j+9e5Jzu6mOX+a5DHL3M7zMhi5/4wMTnd++zxtLk/y0IXu1bxKlcFgYycs0ubDWd197/80g+D4i9006p1JHj/fiq21l1fVTRmcRXBUkj9bpJ8PLbJsPfxhBl+YHZuR261V1d26kdPTWvtgVf1kktdm8AXoC7ppPv+Ztbl7ArCNODINkP8aZXf01MeFPD6DsPO+DG5xc00G/7n89QxuH7Te96teVGvtfUmOyeBU6cszCFVXJnlNkvu21hY8ytQdlfzxDK67fl23/vXdNr6QwdHtZyc5qrX29+Pbi6S19s3W2s9lcK/ev+v24cYkX8kgJPxqkmNba1eMs45xa619LIPrj/8mg/vb3rhI23V5f1prF2fwGTq32+6NSa5K8uYkj2it/cYytnFda+1RSR6cwUBml2dwavFXM7j/86syOI38Y6Prdfv48gyOHl6fFWgDT8rgtf3LDI7W35DBKdXvS/L0JPee74ulNXJMkqcm+YcMBhv7Wva/jv+YwZHjHxm+3GKFfrHb1j9nsG/XZvD36UlJTmitLfj3qbW2N4MvLf53Bq/JVzIImF/P/vfoFzN4Pyamq/PUDD7fV2cw5sNCbd+b5MgMvoz7pwxe7xszOGL9mST7Mnhtvm9MX6IAW1gNzoIBAGAzqqqZJHdN8qzW2lmTrQZg+3BkGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAejKa9wrc8Y53bNPT05MuAwAAgDH4wAc+8KXW2p0Wa3PAehWzlUxPT+eiiy6adBkAAACMQVVdtlQbp3kDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPQkTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPQkTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPQkTAMAAEBPwjQAAAD0JEwDAABAT9s+TFfVQVX1uapqVbVz0vUAAACw8R0w6QI2gLOyBV+H6TPPG+v2Z/acONbtAwAAbGTb+sh0Vf1gkscneeakawEAAGDz2NZhOslLk5yd5BOTLgQAAIDNY8OE6aq6Z1WdUVWvqqpLq+qm7jrmk5ex7qlV9a6qmq2qa6rqoqp6QlUtuH9V9egkO5I8Zy33AwAAgK1vI10rfFqSM/quVFUvTXJ6kuuTvC3JjUlOyOCI8wlVdXJr7aaRdaaSvCDJk1tr11TVamsHAABgG9kwR6aTfCyDgPuoDI4YX7DUClX1yAyC9JVJ7tNae3hr7aQkRyS5JMlJSZ44z6rPSfLJ1tqr16h2AAAAtpENc2S6tfaK4cfLPFr89G7+tNbaJ4e2dVVVnZbk/CRnVtVL5o5OV9W9Mxh07Keq6nbdKgfNzavq4Nba11e+JwAAAGx1GyZM91VVhyc5JskNSV43ury1dkFVXZHksCTHJXlvt+iIDPb7HfNs9h1JPpLkqHHUDAAAwNawacN0kqO7+cWttesWaPP+DML00dkfpt+d5EEj7Y5K8qIMjlh/YI3rBAAAYIvZzGH6bt38skXaXD7SNq21L2Vw+vd/GTql/AOttYvm21BV7U6yO0kOO+ywzMzM9C54Pd3/0JuWbrQKG33/AQAAxmkzh+m565y/sUiba7r5wavtrLW2N8neJNm5c2ebnp5e7SbH6sIvXjzW7W/0/QcAABinzRym10xr7fwk7o8FAADAsmykW2P1NXfU+TaLtJk7em10bgAAANbMZg7TM938rou0uctIWwAAAFi1zRymP9TN711Vt1qgzbEjbVelqnZV1d7Z2dm12BwAAACb1KYN0621zyb5YJJbJDlldHlVHZ/k8CRXJrlwjfrc11rbPTU1tRabAwAAYJPatGG687xu/vyq2jH3ZFUdmuRl3cM9rbXx3icKAACAbWXDjOZdVffN/gCcJPfq5s+tqqfMPdlaO27o59dX1TlJTkvy0ap6a5Ibk5yQ5LZJ3pDk7HHXDgAAwPayYcJ0BuH3fvM8f8RiK7XWTq+qdyd5QpLjk9wsyaVJXpnkHEelAQAAWGsbJkyv5l7PrbXXJHnNmhYEAAAAC9js10yvK6N5AwAAkAjTvRjNGwAAgESYBgAAgN6EaQAAAOhJmAYAAICehGkAAADoSZjuwWjeAAAAJMJ0L0bzBgAAIBGmAQAAoDdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKme3CfaQAAABJhuhf3mQYAACARpgEAAKA3YRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpntwn2kAAAASYboX95kGAAAgEaYBAACgN2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhuoeq2lVVe2dnZyddCgAAABMkTPfQWtvXWts9NTU16VIAAACYIGEaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqZ7qKpdVbV3dnZ20qUAAAAwQcJ0D621fa213VNTU5MuBQAAgAkSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoKcDJl0Am9/0meeNdfsze04c6/YBAAD6cmQaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqZ7qKpdVbV3dnZ20qUAAAAwQcJ0D621fa213VNTU5MuBQAAgAkSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgp20bpqvqZ6vq3VX1paq6vqo+VVW/X1VTk64NAACAje2ASRcwQbdP8s4kL0zylST3SXJWN3/w5MoCAABgo9u2Ybq19oqRp86vquuT/FFVfW9r7fOTqAsAAICNb9ue5r2AL3XzW0y0CgAAADa0DXVkuqrumeQhSY5NsjPJkUkqySmttdcvse6pSU7L4DTtmyW5NMmfJDmntXbTIuvdLMnNk9w7yTOT/H1rbWbVO8NYTJ953tj7mNlz4tj7AAAANrcNFaYzCMNn9F2pql6a5PQk1yd5W5Ibk5yQ5OwkJ1TVyYsE6i8nmRt07M1JTu3bPwAAANvLRjvN+2NJXpDkUUl2JLlgqRWq6pEZBOkrk9yntfbw1tpJSY5IckmSk5I8cZFNPDDJjyb5lQyOTu/rjlYDAADAvDbUkenRQcGqajmrPb2bP6219smhbV1VVaclOT/JmVX1kvmOTrfWPtz9+N6q+kCSizII4IueVg4AAMD2tdGOTPdSVYcnOSbJDUleN7q8tXZBkiuS3DnJccvY5IeT3JTBUXEAAACY16YO00mO7uYXt9auW6DN+0faLub+Gbwmn15tYQAAAGxdG+o07xW4Wze/bJE2l4+0TZJU1VsyGKzs4gwGLjsqyW8m+dckbxjdSFXtTrI7SQ477LDMzMyspu6xu/+hCw5gviaG938r9TXaHwAAwHw2e5g+qJt/Y5E213Tzg0eef1+SX8z+kD2T5OVJXthau2F0I621vUn2JsnOnTvb9PT0yipeJxd+8eKxbn94/7dSX6P9AQAAzGezh+kVa639dpLfnnQdAAAAbD6b/ZrpuaPOt1mkzdzR66+PuRYAAAC2ic0epme6+V0XaXOXkbYAAACwKps9TH+om9+7qm61QJtjR9quWFXtqqq9s7Ozq90UAAAAm9imDtOttc8m+WCSWyQ5ZXR5VR2f5PAkVya5cA3629da2z01NbXaTQEAALCJbeow3XleN39+Ve2Ye7KqDk3ysu7hntba+O+pBAAAwLawoUbzrqr7Zn8ATpJ7dfPnVtVT5p5srR039PPrq+qcJKcl+WhVvTXJjUlOSHLbDO4Zffa4awcAAGD72FBhOoPwe795nj9isZVaa6dX1buTPCHJ8UluluTSJK9Mco6j0gAAAKylDRWmW2vnJ6kVrvuaJK9Z04IAAABgHlvhmul1YzRvAAAAEmG6F6N5AwAAkAjTAAAA0JswDQAAAD0J0wAAANCTMA0AAAA9CdM9GM0bAACARJjuxWjeAAAAJMI0AAAA9CZMAwAAQE/CNAAAAPQkTAMAAEBPB0y6ANiops88b+x9zOw5cex9AAAAa8+R6R7cGgsAAIBEmO7FrbEAAABIhGkAAADoTZgGAACAnoRpAAAA6EmYBgAAgJ6EaQAAAOhJmAYAAICehGkAAADoSZjuoap2VdXe2dnZSZcCAADABAnTPbTW9rXWdk9NTU26FAAAACZImAYAAICehGkAAADoSZgGAACAnoRpAAAA6EmYBgAAgJ6EaQAAAOhJmAYAAICehGkAAADoSZjuoap2VdXe2dnZSZcCAADABB0w6QI2k9baviT7du7c+cuTroWtZfrM88bex8yeE8feBwAAbBeOTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPQkTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPQkTPdQVbuqau/s7OykSwEAAGCChOkeWmv7Wmu7p6amJl0KAAAAEyRMAwAAQE/CNAAAAPQkTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPQkTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9CRMAwAAQE/CNAAAAPR0wKQLANbf9JnnjXX7M3tOHOv2AQBg0hyZBgAAgJ6EaQAAAOhJmO6hqnZV1d7Z2dlJlwIAAMAECdM9tNb2tdZ2T01NTboUAAAAJkiYBgAAgJ7WNExX1UFVdUxVHbqW2wUAAICNpHeYrqoHVdXLqurokecfk+SqJO9LckVVPWdtSgQAAICNZSVHph+X5JeSzMw9UVV3S7I3ya2SXNE9/fSqOmG1BQIAAMBGs5Iw/SNJPtJau3rouUcnOSDJ01pr35fk/klaktNXXyIAAABsLCsJ03dK8rmR534iyfVJzk6S1tpFSd6b5IdXVR0AAABsQCsJ07dOcuPcg6r6riQ7k7yvtXbdULvPJvme1ZUHAAAAG89KwvQXk+wYenxcBgH7PSPtDkxyXQAAAGCLWUmYvjDJ0VX1c1V1207TuA4AACAASURBVCTPyOD66H8aafcDST6/yvoAAABgw1lJmH5Bkm8l+YskVyd5aJIPtdbOn2tQVYdnEKYvWoMaAQAAYEPpHaZba+9L8vAkFyS5JMm5SU4cafaoJLP5zqPVAAAAsOkdsJKVWmv/lEWCcmvtD5L8wUqLAgAAgI1sJad5AwAAwLYmTAMAAEBPKwrTVbWjqv64qv69qq6tqv9cYPrWWhcMAAAAk9b7mumq2pnk7Uluk6SWar6SogAAAGAjW8mR6d9LclCS1ya5b5KDW2vftdC0ptUCAADABrCS0bzvl+SS1tr/v9bFAFvP9JnnjXX7M3tG78wHAADjt5Ijx9cl+chaFwIAAACbxUrC9PuS3H2tCwEAAIDNYiVh+n8nObqqfnatiwEAAIDNoPc1062191TVzyf546o6KclbknwuyU0LtH/n6kocj6o6JckvJDkmye2TfCrJOUn+qLU2774AAABAsrIByJLkFkmuTXJqNy2kraKPcXtyksuS/GaSq5I8KMmLMziF/TcnWBcAAAAb3EruM/3IJK/O4BTxLyeZSXLN2pa1Lna11v5j6PE7quqgJL9aVf+rtfbNSRUGAADAxraSo8a/laSSnJ5k72Y9JXokSM/5UJJbZnDa9xfWtyIAAAA2i5UMQPb9Sd7TWnv5WgfpqrpnVZ1RVa+qqkur6qaqalV18jLWPbWq3lVVs1V1TVVdVFVPqKo++/jjSb6S5Isr3gkAAAC2vJUcmZ7NYMCxcTgtyRl9V6qql2ZwpPz6JG9LcmOSE5KcneSEqjp5qeBfVTuTPDbJs1pr/9m3BgAAALaPlRyZ/sckx1ZVrXUxST6W5AVJHpVkR5ILllqhu4b79CRXJrlPa+3hrbWTkhyR5JIkJyV54hLbuHOSv87gHtrPX80OAAAAsPWtJEw/I8nBSX6/qtZ0pO7W2itaa09trb22tfapZa729G7+tNbaJ4e2dVUGR7qT5MyFTveuqqkk/5DB6OQ/01q7cYXlAwAAsE2sJAz/zwzC55OSPKKq3pGF7zPdWmu/u4r6FlVVh2dwn+gbkrxuns4vqKorkhyW5Lgk7x1Z/5ZJ/j7JoUke0Fr78rhqBQAAYOtYSZg+K4P7R1eSu3XTqLnlLcnYwnSSo7v5xa216xZo8/4MwvTRGQrT3VH11ya5T5LjW2uXjbFOAAAAtpCVhOlnZxCSN4K5IL9YEL58pO2clybZleSpSW5dVccNLft4a+1rw42raneS3Uly2GGHZWZmZqU1r4v7HzreO5YN7/9W6mu4v63a13r0N6m+AABgvfQO0621s8ZQx0od1M2/sUiba7r5wSPP/3Q3/7151nlQkvOHn2it7U2yN0l27tzZpqen+9S57i784sVj3f7w/m+lvob726p9rUd/k+oLAADWy5oOILaZtNamJ10DAAAAm9OqwnQ3EvaxSe6U5LLW2nuXWGWtzR11vs0ibeaOXn99zLUAAACwTazk1lipqqmqemWSLyZ5S5JXJXnc0PLHVdXnR65DHoeZbn7XRdrcZaQtAAAArErvMF1Vt8ngeuLHJLk6g9tk1UizNyb57iSPWF15S/pQN793Vd1qgTbHjrRdsaraVVV7Z2dnV7spAAAANrGVHJl+SpIfzuBo9N1baw8fbdBauzLJx5P8xOrKW1xr7bNJPpjkFklOGV1eVccnOTzJlUkuXIP+9rXWdk9NTa12UwAAAGxiKwnTpyT5fJJfbq1du0i7T2Rwf+dxe143f35V7Zh7sqoOTfKy7uGe1tr47z0EAADAtrCSAcjunuQtrbVvLtHu+iR36LPhqrpv9gfgJLlXN39uVT1l7snW2nFDP7++qs5JclqSj1bVW5PcmOSEJLdN8oYkZ/epAwAAABazkjB9Y5JbLqPdXbJ/tO3lum2S+83z/BGLrdRaO72q3p3kCUmOT3KzJJcmeWWScxyVBgAAYC2tJEz/W5Kjq+rAhY5OV9UhGVxX/cE+G26tnZ/vHMxsueu+JslrVrIuAAAA9LGSMP36JHuSPD/JkxZo89wM7u/82hXWtSFV1a4ku3bs2LFkW2D9TZ953li3P7PnxLFuHwCAzWMlA5CdneSSJE+sqndX1W90z09X1WlV9fYku5N8NMn/XaM6NwSjeQMAAJCs4Mh0a+3aqnpwktcleUCS+3eLju+mSvKBJI9ord2wVoUCAADARrGS07zTWrsiyQOq6iFJHpbBCN83S/LZJP+Q5A2ttbZmVQIAAMAGsqIwPae19uYkb16jWgAAAGBT6H3NdFU9s6p+ZhntdlXVM1dWFgAAAGxcKxmA7Kwkj1hGu59J8jsr2P6G1X1BsHd2dnbSpQAAADBBKwnTy3WzJFvqummjeQMAAJCMN0zfI8nXxrh9AAAAmIhlDUA2z7XPRy1yPfQBSX4gyY8lOX/lpQEAAMDGtNzRvM/K4JTt6h4f1U2LuTbJs1dWFgAAAGxcyw3Tz87+MP3MJB9O8ncLtL0hyRVJ3tJau2rVFQIAAMAGs6ww3Vo7a+7n7vTuD7fWnjWuogAAAGAjW+6R6f/SWhvnoGUbWlXtSrJrx44dky4FAACACVoyTFfVf1tNB621d65m/Y2ktbYvyb6dO3f+8qRrAQAAYHKWc2T6/Kz8ftFtmX0AAADAprGcoHt5Vh6mAbak6TPPG+v2Z/acONbtAwCwOkuG6dba9DrUAQAAAJvGth1MDAAAAFZKmAYAAICehGkAAADoyUjbABvcuAc7Swx4BgDQlyPTPVTVrqraOzs7O+lSAAAAmCBhuofW2r7W2u6pqalJlwIAAMAECdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTPVTVrqraOzs7O+lSAAAAmCBhuofW2r7W2u6pqalJlwIAAMAECdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdM9VNWuqto7Ozs76VIAAACYIGG6h9bavtba7qmpqUmXAgAAwAQJ0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPR0w6QIA2J6mzzxv7H3M7Dlx7H0AANuTI9MAAADQkzANAAAAPTnNG4D/4tRrAIDlcWQaAAAAehKmAQAAoCdhGgAAAHoSpnuoql1VtXd2dnbSpQAAADBBwnQPrbV9rbXdU1NTky4FAACACRKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAejpg0gUAwHqYPvO8sW5/Zs+JY90+ALCxODINAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTAAAA0NO2DdNVtaOqXl5VH66qb1XVxyZdEwAAAJvDAZMuYILuneTEJP+SwZcK2/aLBQAAAPrZzgFyX2vtLq21k5N8cNLFAAAAsHls2zDdWrtp0jUAAACwOW2oMF1V96yqM6rqVVV1aVXdVFWtqk5exrqnVtW7qmq2qq6pqouq6glVtaH2EQAAgM1vo10zfVqSM/quVFUvTXJ6kuuTvC3JjUlOSHJ2khOq6mRHogEAAFgrG+2o7ceSvCDJo5LsSHLBUitU1SMzCNJXJrlPa+3hrbWTkhyR5JIkJyV54tgqBgAAYNvZUEemW2uvGH5cVctZ7end/GmttU8ObeuqqjotyflJzqyqlzg6DQAAwFrYaEeme6mqw5Mck+SGJK8bXd5auyDJFUnunOS49a0OAACArWpTh+kkR3fzi1tr1y3Q5v0jbQEAAGBVNtRp3itwt25+2SJtLh9pmySpqlsneVj38K5Jbjs0avj7W2uXjbTfnWR3khx22GGZmZlZRdnjd/9Dx3tG+/D+b6W+hvvbqn2tR3/62lx9Dfe3Vftaj/42+r8LAMDa2uxh+qBu/o1F2lzTzQ8eef7QfOep4XOPH5vk3OEFrbW9SfYmyc6dO9v09HTPUtfXhV+8eKzbH97/rdTXcH9bta/16E9fm6uv4f62al/r0d9G/3cBAFhbmz1Mr1hrbSbJskY4AwAAgGGb/ZrpuaPOt1mkzdzR66+PuRYAAAC2ic0epme6+V0XaXOXkbYAAACwKps9TH+om9+7qm61QJtjR9quWFXtqqq9s7Ozq90UAAAAm9imDtOttc8m+WCSWyQ5ZXR5VR2f5PAkVya5cA3629da2z01NbXaTQEAALCJbeow3XleN39+Ve2Ye7KqDk3ysu7hntba+O/BAgAAwLawoUbzrqr7Zn8ATpJ7dfPnVtVT5p5srR039PPrq+qcJKcl+WhVvTXJjUlOSHLbJG9Icva4awcAAGD72FBhOoPwe795nj9isZVaa6dX1buTPCHJ8UluluTSJK9Mco6j0gAAAKylDRWmW2vnZ4X3fm6tvSbJa9a0oBFVtSvJrh07dizZFgDWw/SZ5411+zN7Thzr9gFgs9oK10yvGwOQAQAAkAjTAAAA0JswDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQ04a6NdZG59ZYALD1jPv2YolbjAFsRY5M9+DWWAAAACTCNAAAAPQmTAMAAEBPwjQAAAD0JEwDAABAT8I0AAAA9OTWWD24NRYAyzHuWy1N6jZLW3W/AGAlHJnuwa2xAAAASIRpAAAA6E2YBgAAgJ6EaQAAAOhJmAYAAICehGkAAADoSZgGAACAnoRpAAAA6EmY7qGqdlXV3tnZ2UmXAgAAwAQJ0z201va11nZPTU1NuhQAAAAmSJgGAACAnoRpAAAA6EmYBgAAgJ6EaQAAAOhJmAYAAICehGkAAADoSZgGAACAnoRpAAAA6OmASRewmVTVriS7duzYMelSAIBNaPrM88bex8yeE8feBwCOTPfSWtvXWts9NTU16VIAAACYIGEaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgJ2EaAAAAehKmAQAAoCdhGgAAAHoSpgEAAKAnYRoAAAB6EqYBAACgpwMmXcBmUlW7kuzasWPHpEsBAFjS9JnnjXX7M3tOHOv2t5txv1+J9wzWkiPTPbTW9rXWdk9NTU26FAAAACZImAYAAICehGkAAADoSZgGAACAnoRpAAAA6EmYBgAAgJ6EaQAAAOhJmAYAAICehGkAAADoSZgGAACAnoRpAAAA6EmYBgAAgJ6EaQAAAOhJmAYAAICehGkAAADoSZgGAACAnoRpAAAA6EmYBgAAgJ4OmHQBm0lV7Uqya8eOHZMuBQC2tOkzzxt7HzN7Thx7H9vJuN8z7xfL5bO4el7D5XFkuofW2r7W2u6pqalJlwIAAMAECdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTAAAA0JMwDQAAAD0J0wAAANCTMA0AAAA9CdMAAADQkzANAAAAPQnTAAAA0NO2DdNVdURVvbmqrqmq/6iql1TVrSddFwAAABvfAZMuYBKq6nZJ3pHksiQnJzk0yQuT3CnJz0+wNAAAADaBbRmmk/xKkkOSHNVa+1KSVNW3kry6qn63tXbxRKsDAABgQ9uup3k/LMnb5oJ056+TfDPJQydTEgAAAJvFhgnTVXXPqjqjql5VVZdW1U1V1arq5GWse2pVvauqZrtroC+qqidU1UL79wNJPj78RGvtm0k+leT7V783AAAAbGUb6TTv05Kc0XelqnppktOTXJ/kbUluTHJCkrOTnFBVJ7fWbhpZ7ZAkX51nc1cnuX3fGgAAANheNsyR6SQfS/KCJI9KsiPJBUutUFWPzCBIX5nkPq21h7fWTkpyRJJLkpyU5IljqxgAAIBtacMcmW6tvWL4cVUtZ7Wnd/OntdY+ObStq6rqtCTnJzmzql4ycnT66iS3m2d7hyS5tE/dAAAAbD8b6ch0L1V1eJJjktyQ5HWjy1trFyS5Ismdkxw3sviSDK6bHt7egUnuEWEaAACAJWzaMJ3k6G5+cWvtugXavH+k7Zw3ZXA99R2GnjspyYHdMgAAAFjQhjnNewXu1s0vW6TN5SNt5/xRBtdS/11V/W6SQ5O8MMlftdY+nnlU1e4ku5PksMMOy8zMzArLXh/3P3R0zLW1Nbz/W6mv4f62al/r0Z++Nldfw/1t1b7Woz99ba6+hvvbqn2tR3+T6ms9Pf1vPjrW7T/vZ3/ov35ez8/HuPcr2b9v69lXsnU/i+vJa7g8mzlMH9TNv7FIm2u6+cHDT7bWvlpVP5HkxUn+Jsl1Sf4yyVMX2lBrbW+SvUmyc+fONj09vbKq18mFX7x4rNsf3v+t1Ndwf1u1r/XoT1+bq6/h/rZqX+vRn742V1/D/W3Vvtajv0n1tZ620ms43N9W7Ws9+tvoOWAteA2XZzOH6VVprX0iyUMmXQcAAACbz2a+ZnruqPNtFmkzd/T662OuBQAAgG1kM4fpmW5+10Xa3GWkLQAAAKzaZg7TH+rm966qWy3Q5tiRtqtSVbuqau/s7OxabA4AAIBNatOG6dbaZ5N8MMktkpwyuryqjk9yeJIrk1y4Rn3ua63tnpqaWovNAQAAsElt2jDdeV43f35V7Zh7sv5fe3ceNUdV5nH8+0uIQICwyCph2BI5oh6WEIK4gIMzA44xqCASFzJHHRUYUQajcxwV0UEFGWEEcRAQPYALaIC4gIMIBIWRuI2AUQiEJQmQEMi+QZ754942lU51v939prc3v88599Tbde/tut31nHr76aq+Je0KfC0//GJEtP8+A2ZmZmZmZrbZ6JnZvCUdyvoEGODAvDxX0lmVlRFxROHv6yVdCnwI+KOkW4G1wDHAKOAG4OJ2j93MzMzMzMw2Lz2TTJOS3wkl68fW6xQRp0q6CzgNOAoYDswCrgQu9VlpMzMzMzMz29R6JpmOiNsBtdj3WuDaTTqgEpImAhPHjBkzYFszMzMzMzMbuvr9N9Md5QnIzMzMzMzMDJxMm5mZmZmZmTXNybSZmZmZmZlZk5xMm5mZmZmZmTXJybSZmZmZmZlZk5xMN0HSREmXLV68uNtDMTMzMzMzsy5yMt0Ez+ZtZmZmZmZm4GTazMzMzMzMrGlOps3MzMzMzMya5GTazMzMzMzMrElOps3MzMzMzMyapIjo9hj6jqQFwKPdHkcNOwMLuz0Is8zxaL3GMWm9xPFovcTxaL2kF+Jx74jYpV4DJ9NDjKSZEXFYt8dhBo5H6z2OSesljkfrJY5H6yX9Eo++zNvMzMzMzMysSU6mzczMzMzMzJrkZHrouazbAzArcDxar3FMWi9xPFovcTxaL+mLePRvps3MzMzMzMya5DPTZmZmZmZmZk1yMj1ESJosaYakxZKWSZop6TRJ3se2EUkjJB0j6YIcK0skrZE0V9L1ko4eoH9L8SbpWEk/k7RI0gpJ90n6pKQtB+g3QdI0SU9LWiXpQUnnSdq+hZdvfUDSuZIil7PqtHMsWttI2lrSVEn3Snoux8ojkq6T9OqS9sNy/M3M8bg4x+fJDWyro7Fs/UXSaElflfRnSSsLx5+vS9qvTj8fI61pkg6QdIakqyXNkrQu/z8+oYG+fRFz+TVeLWmepNWSHpV0qaQ9BnqNG4gIlz4vwCVAACuBHwHTgCV53Q+BYd0eo0tvFeANOT4CmJ/j5nvAHwvrz6nRt6V4A6bmNs8DtwLXAU/ndXcDI2v0Ozn3CeCuPM5H8+MHgV27/X66bPL4HJ/3+bq8n8+q0c6x6NLOONw379cA5uX4ug74NbAW+Peq9sOBG3P7xTkGfwysyusuqrOtjsayS38V4BDg2bxfHwduyOWJvG4pcGS348rHyKFTgAtZ/3mwWE4YoF9fxBxwFLAit/sN8F3gT/nx08BLG36vur2zXAZXgLexPiEaW1i/G/BArjuj2+N06a0C/C1wPfDakrqTCgem11fVtRRvwGGkxGg5MKGwflvgjtzvKyX9RueD3QvApML6LfKBL4Bp3X4/XTZpbG6ZY2lu/idcmkw7Fl3aWYBtgIdyrHwcGF5V/+LqD1vAv+Y4uB/YrbB+LPBkrptUsq2OxrJL/xXgV3l/XgaMKKwfAVyR6/7QzbjyMXJoFeB9wHnA24H9gdsZIJnul5jLx/f5uf70qrovsz7BVkPvVbd3lsvgCjAz7/T3lNQdVQhqn512abgAl+fYuaJqfUvxRkrcA/h0Sb/98oFwNbBDVV3loHZlSb9RpLM/ARzY7ffMZZPF3pfyPp0IXEXtZNqx6NLOOPxC3p9fbbD9cOCp3Od1JfWn5Lpfl9R1NJZd+qsAW7H+rOAeJfV7FOpHFtb7GOmyKePwdgZOpvsi5oDT8/rbSvoNJ32RGsAbG3lv/HvaPiZpNDAOWEO6HGIDEXEH6ezO7sARnR2d9bnf5eXoyopW403Si4Dj8sNrSvo9TLqE50XAG6uqj6/Tbwkwvaqd9TFJE0hn966NiOl12jkWrW1ynLw/P/zPBru9CtgVeCIi7iypv450afh4SXsWttWNWLb+8gLparGBLCddWutjpHVcn8VcvX4vkM5ql/Ur5WS6vx2Sl/dHxMoabe6tamvWiLF5Ob+wrtV4OwAYCSyKiNmN9pM0inRpUbG+ke1ZH5K0FfAtYBFwxgDNHYvWTuNIl3HPjYhHJB0q6XOS/lvSOZJeU9Knst9L4yMiVpAu/wY4uKRfR2LZ+k9ErAV+nh9+VtKISl3++3P54RWRT63hY6R1Xj/FXN3jdZ1+pbZopJH1rH3z8tE6bR6ramtWl6TdgSn54Q8KVa3G275VdY322ycvn8vfMDbaz/rTf5D+qb4jIhYO0NaxaO30yrycK+nLpKslij4l6QbgXRGxPK9rNCYPpjwmOxXL1p9OBW4mXTFxnKSZef14YEfSZFFTC+19jLRO64uYy0n4TgOMtalY9Znp/rZtXi6v02ZZXm7X5rHYECBpC+BqYHvg51WX2rYab53uZ31G0pHAR4AbIuJ7DXRxLFo7VT5oHUJKpC8ExpCSlkmkSxWPB75W6OOYtLbJl7oeCfyU9POr43PZkzSx04x8BrvC8Wid1i8xt23h71p9m4pVJ9NmVvR14BjSrTfe1eWx2GZA0takicaWkM6+mHVb5bPRCODqiPhoRMyOiOci4iZSEhPAuyXtX/NZzDaR/IXjfaQvdSYBu+RyPOlLnh9I+nT3Rmi2+XIy3d8q35xsU6dN5RuYpW0ei/U5SRcB7yXdwuWYiHiyqkmr8dbpftZfziX9Rv/MiJg/UOPMsWjtVNyH36iujIiZ5NumkGaoBcektYmkHUj3lN4OODYiboqIhbncCBxLmnjsU5Iq8504Hq3T+iXmlhX+rtW3qVh1Mt3f5uTl3nXa7FXV1mwjki4APgwsICXSD5Y0m5OXzcZb5e+/abJf5bcsO+TfuDTaz/rLW0j3lzxF0u3FQvqQCPChvO7y/HhOXjoWrR0eqfF3WZvd83JOXrYak52KZes//0g6C31Pvtx7AxHxEPC/pHmQjs6r5+Slj5HWKXPysqdjLv+++tn8sNZYm4pVJ9P9rXL7opfnSyXLjK9qa7YBSecBZwLPAG+IiAdqNG013maRvjXfqc4lkYdX94uIxUBlZsfxG/Wo0c/60jDSGb7qsluu3y8/Piw/dixaOxX34YtrtNk5LytnOX6bl6XxIWkk8IqS5+9oLFtfqiQZi+u0eS4vK7/39zHSOq2fYq7u8bpOv1JOpvtYRDxOCogXASdW10s6ijRRxZOke7SZbUDSF4GPkb6l+7uI+L9abVuNt4hYQ5o0BeCdJf32I92jdQ3w46rqG+v0GwVMzA+n1Rq39baI2CciVFZIt8oC+Fhed3Du41i0tomIuaQzfZDmkNiApB2BQ/PDyqzKd5Ou7Bkt6XUlT3si6TfY9+bnr2yrG7Fs/WVeXo4r3harIq8blx8+Aj5GWuf1WczV6zcceEeNfuUiwqWPC3ACaSKU+cCYwvpdSfe0DOCMbo/TpfcK8PkcH88C4xrs01K8kb79W0eaOfHwwvptgdtzv6+U9NsLWAG8ALy5sH4L4Du537Ruv5cu7SmkickCOKukzrHo0s7Ym5j36TPAYYX1WwHfzXUzARXqzsrr7wd2Lawfm+M0gEkl2+poLLv0V8lxsDzvz4uBLQt1WwKX5rpFwPbdiisfI4d2Kez7E+q06YuYy89bOSafVlV3fl7/2+Lxve570+2d4zL4Qro9R5Auk5gO/JB0OVCQvlUZ3u0xuvRWAd6c4yNIN6e/qkb5REnfluKNdA/MAJ4HfgZ8H3gqr7sHGFmj38m5zzrgTtIH2Tm534MUPrS6DK1CnWQ61zsWXdoZf1/O+3ZN3t/TSLfFCuAJYGxV++HATbl+cY7H6Tk+A/ivOtvqaCy79FcBTsn7OHIMTs9lXl63Cji+23HlY+TQKaSrb+4plCV5P/6luL5fY47007EVrP9i9Duk28wF6SqjAxp+r7q9s1w2TQEmA7/Mwb6cNNPoacCwbo/NpfcKMIX1yXS9cnuN/i3FG2lCqf8hnQ1fSfqm8pMUvmmv0W8CaTbTBcBq4CHgPArfwrsMvcIAyXRu41h0aVsB3grcluNkdf5wdgGwS432w4DTcxwuz3F5FzC5gW11NJZd+quQkptvky7lXpXLbOBy4MBeiSsfI4dGIU1mN+DnxH6OOeAA4BrSpeergcdIt4jdo5n3SvnJzMzMzMzMzKxBnoDMzMzMzMzMrElOps3MzMzMzMya5GTazMzMzMzMrElOps3MzMzMzMya5GTazMzMzMzMrElOps3MzMzMzMya5GTazMzMzMzMbJyh7AAAB1VJREFUrElOps3MzBogaY6kkHR0t8cyVOT3M7o9DjMzs1Y4mTYzM+swSWfnRPLsbo+lXSRdlV/jlG6PxczMrB226PYAzMzMbLP1sm4PwMzMrFVOps3MzKwrImJWt8dgZmbWKl/mbWZm1qLipcySxki6VtJTklZLmiXp45KGVfUJ4DP54Wcqvxsuu+xb0jaSpkq6V9ISSSsl3Z8vE9+2ZDx/vXxc0t6SvinpCUnPS7owtxkh6d2SviPpz5KWSloh6QFJX5K0U53XO0LSP0v6haRF+XU+JulHkt6Z2+yTX+Mpuds3q17jlOJ7Ues305J2zuOZlV/3Ekn3SDpV0kYnA/I+iLxPtpN0vqRH8hjnSrq01muT9A5Jt+XXtFbSQkl/lHSJpP1rvR9mZrZ585lpMzOzwTsYuAhYCPwC2BV4LfBFYDTwL4W238rtDwL+APy+UPfXvyWNBm4BDgQWAHcDq4DxpGT8LZKOjohnS8YzFvhdbv9L0v/753LdbsC3gWeBWXmbo4DDgKnACZImRMTC4hNK2hH4MfAqYHV+3qeBlwCvBl4BXAMsy6/xNcD+ud1Dhacq/l1K0hjgNmAv4ElgOjASeD1wSX7tb4qI1SXdt8/b3BO4E7gvj+WDwOGSjoiItYVtnU16P9cCvwLmATsA+wCnAjOA2QON2czMNj9Ops3MzAbvDOCzwDkRsQ5A0utIifWpks6LiMcBImJKTuAOAm6IiLOrn0ySgO+TEumLgakRsTLXbQ1cBrwL+AowpWQ8k4GrgA9ExJqqusXAm4Gbq5LKrUmJ6j8BnwM+VNXvm6RE+m7ghIiYV+i7FSnRJSfhUyRdRUqmL4+Iq0rGWM+1pET6OuA9EbEqb2cv4FbgDcDZwL+V9D0e+AlwZEQsy/1eAtwDHAq8nZT0I2lL0hcIy4BxEfGX4hNJGgs83+TYzcxsM+HLvM3MzAbvXuCzlUQaICLuJJ1ZHkZONJtwLClxvQc4o5JI5+ddSTrL+jTwznzGuNozwIdLEmkiYmlETC8m0oXnPZ2UPL6tWCfpYGASsBSYVEykc99VEfHTJl9jKUmvJZ19Xwp8sJJI5+08TvriAuC0nMRXWwa8t5JI537zSF9KABxTaDsK2BqYXZ1I534PRsQjg3k9ZmY2dPnMtJmZ2eD9JCLKfvs7CziOdCl0M96Ylz8oJugVEbFc0szcbjzws6omt0bE0nobkHQIKbHcB9gGUK5aA+wiacfCJeTH5uVNEbGgydfSrKPycnpELKqujIibJc0H9gDGkS7pLvpNRDxZ8ryVyc7+ui8iYoGkOcBBki4AvuFJ0czMrFFOps3MzAbvsRrrl+Rl2RnUevbLy/MlnT9A211K1j1aq3GeuOwa0qXe9Ywi/a4aYO+87ESiuWde1jsj/DApmd6zpK7ZffEe4HrgTOBMSQtIVwTcAlwdEYsbGbSZmW1+nEybmZkN3kZnjwdpeF7eAcwZoG1Z4ryyZF3FF0iJ9APAJ4CZwMLKZd+S5pESVRX6lM643WatbrOpfRERMyTtC7wJOBo4Mv89EThb0t9HxO9aHIuZmQ1hTqbNzMx6z+N5eV1EXLKJn/vEvDwpIu4rVkjaBti9pE/lbO8Bm3gsZebm5X512lTq5tZp07CIWEGa8O37AJL2IE3udhJpUrYjN8V2zMxsaPEEZGZmZp1XmRis1pfalcm8TqxRPxiVey0/XlI3mQ3PSFfckpeTJO3c4HYGeo213JGXE8smV5P0D6Qz58uA3zT53A2JiPnAJ/PDg9qxDTMz639Ops3MzDqvckb1ZTXqbyAlikdJ+rqknaobSNpd0vtb2Hbld8+nVj3fYaRLwDeSL3OeDmwHTMtnbot9t5J0XFW3gV5jqYiYQZodfTvgknz7qsp29gQuzA8vLs703QpJe0t6n6RRJdUT87Lm78/NzGzz5su8zczMOu8WYAXwVkl3ArOBF0izZd8UEeskVe6X/AFgsqQ/kM4mbwW8lHQP6qeBbzS57XNI928+V9JJwJ9IM1y/Bvgu8GrWTzhWNAW4Obd7WNJdwILc9yDS/av3KbS/Efg08BFJrwCeIP0O+sqI+NUAY5xMukf3ycDRkmYAI0m3GNsG+DnpPtODtSPp/btE0u9Jk54NI723LwfWku5DbWZmthEn02ZmZh0WEU9KehMp2TyElKCKlHDelNs8Ielw4L3A24FXAhNI95CeC1wATGth29dLen3e9kHAGOBB4COk3wc/XKPfonwP6PeTktzDgS2Bp4AZwLVV7X+fk/WzSL853jZX3QXUTaYj4qF8666ppPtbTyIltvcD3wYuq75PdotmAx8lTTz28lzWkd7fy4CLIuKBTbAdMzMbglR+W0wzMzMzMzMzq8W/mTYzMzMzMzNrkpNpMzMzMzMzsyY5mTYzMzMzMzNrkpNpMzMzMzMzsyY5mTYzMzMzMzNrkpNpMzMzMzMzsyY5mTYzMzMzMzNrkpNpMzMzMzMzsyY5mTYzMzMzMzNrkpNpMzMzMzMzsyb9P+/OKnHP6Js8AAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":297},"id":"2EbnersE2PM8","executionInfo":{"status":"ok","timestamp":1630826756032,"user_tz":-330,"elapsed":19,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"ab019d42-7123-4d42-9f9c-e01f9c1e1f44"},"source":["event_frequency = pd.DataFrame(\n"," interactions[\"event\"].value_counts() / len(interactions)\n",").rename(columns={\"event\": \"frequency\"})\n","\n","event_frequency[\"frequency\"] = event_frequency[\"frequency\"].apply(\n"," lambda x: f\"{round(100*x,3)}%\"\n",")\n","event_frequency"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
frequency
click89.895%
contact_phone_click_12.582%
bookmark2.487%
chat_click2.138%
contact_chat1.451%
contact_partner_click0.682%
contact_phone_click_20.666%
contact_phone_click_30.1%
\n","
"],"text/plain":[" frequency\n","click 89.895%\n","contact_phone_click_1 2.582%\n","bookmark 2.487%\n","chat_click 2.138%\n","contact_chat 1.451%\n","contact_partner_click 0.682%\n","contact_phone_click_2 0.666%\n","contact_phone_click_3 0.1%"]},"metadata":{},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"Nzgv964U2W2q"},"source":["def unix_to_day(timestamps):\n"," min_timestamp = timestamps.min()\n"," seconds_in_day = 60*60*24\n"," return (timestamps - min_timestamp)//seconds_in_day + 1\n","\n","\n","interactions[\"day\"] = unix_to_day(interactions[\"timestamp\"])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":369},"id":"t_exVXo52f_e","executionInfo":{"status":"ok","timestamp":1630826800115,"user_tz":-330,"elapsed":657,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"6088a712-f753-4e17-e030-204c2af97a31"},"source":["def plot_interactions_over_time(series):\n"," freq = series.value_counts()\n"," labels, counts = freq.index, freq.values/10**6\n"," \n"," matplotlib.rcParams.update({\"font.size\": 22})\n"," plt.figure(figsize=(16,5))\n"," plt.bar(labels, counts, align='center')\n"," plt.gca().set_xticks(labels)\n"," plt.title(f\"Data split\")\n"," plt.xlabel(\"Day\")\n"," plt.ylabel(\"Interactions [mln]\")\n"," plt.grid(axis=\"y\")\n"," \n","plot_interactions_over_time(interactions[\"day\"])"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA80AAAFrCAYAAADrbGsvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde5hkVXnv8e8vIIqAeOGiAgKCwUAUCYNBURhE4wX1QAATwQhqxHDLiRoVTDSjBsXERFGQhERFTkSiaFCDN1RACUa5eEEQxcsAjo6KCsod4T1/7N1OW3T19O6u7urq+n6ep55dtfeqtd81XdO931prr5WqQpIkSZIk3dPvDDsASZIkSZIWK5NmSZIkSZL6MGmWJEmSJKkPk2ZJkiRJkvowaZYkSZIkqQ+TZkmSJEmS+jBpliRpGkkOS1JTPG5NsjrJFUnen+RlSbYadrzjIsn57c/htCmO/eZnNoTQJElLjEmzJEmzcx9gc2BH4E+BfwK+n+RDSR46XydNss2kxH35fJ1nKZv073fYsGORJC1+Js2SJM3cM4CN2sf9gW2AJwLHAlcB6wB/DFye5HFDilGSJA2QSbMkSTN3a1Xd1D5urKprqurCqnozTY/zy4C7gAcCZyfZcqjRjqmqOq2qUlUZdiySpNFn0ixJ0gBU4600vc4AmwErhheRJEkaBJNmSZIG65+Ab7fP/yzJpr0FkmzfThz26SQ/THJHkl8m+UaStyXZZqqKk6wEvj9p13lTTFC2zaTy90myb5J/TXJ5kl+151qd5ONJDk4yp2uBJFsneWtb/01t/T9M8tUkpybZf4r3nNbGen77es8kZ7dx3ZbkO0nekuSBs4xpyonAJiYPm7TrPVP8+y2fzTklSUvXusMOQJKkpaSqKsm7gROA9YA9gQ9NHE+yMXD1FG+9F7BT+3hRkoOq6pNzDOdNwF9NsX9z4Ont45Akf1xVt3etPMlewDnABj2HHtI+dgYOBjacpo6XAO/kt7/I3w54OfDcJE+qqm91jU2SpEGxp1mSpMG7aNLzqSYE+zLwSmBv4JHAJsAONLNwf5kmyTwzyUN63rcjTVI9YfLEZBOPayYdvxH4N+A5wDJgC5pk9rE0PeK3tnW8oWsD2x7q02kS5u8Czwe2p7mf+6E0Xxa8Fpgu4d0eeAdwKfBHwKbAI9r33dHW87Ek9+kaXx9Pp/k3mvAX3PPf7wsDOpckaYmwp1mSpMH79qTnv5X4VtWNwB9O8Z6fAd9O8iHgfGAP4AiaBHLivbckuWXSe26tqpv6BVFVK/ocWg1cnOQzwCeAI5O8oap+1bdF97QT8LD2+QFV9bWe4z+iSUCnS8i3AL4GLK+qiXZdD7whyfeA/6BJoo+iSfLnpKpuBUh+Mz/Y7dP9+0mSBPY0S5I0H26Y9LzTfblV9WvgjPblUwYW0dTn+iTwU5re4sd3fPvkL95XzSGMYyclzJNjex9NrzvAC+ZQvyRJc2LSLEnS4E1e6qimLJA8Ncn7kny7nUCrJk1edXJb7HfnHEiyaZK/SfKFJD9NcmfPuSYmKut6rm/RDO8GeG+S2cR6M3DuNMf/q93umOT+s6hfkqQ5c3i2JEmDt/Gk57+YfCDJujT3Aj+3Yz2dJXkicDYz6+3udK52qPhxwNto7ot+RpJv0QzJ/jzwmar60Vqqubqq7prm+FXtNjRDwW+YpqwkSfPCnmZJkgZvcq9rb+L4KtYkzGcD+7flN2HNZFRHtMfXmW0A7SzdH6ZJmH/Snnd3msm1Np50ruvat3T+Ir2qTmzj/1+aHvUdgD+n+VLgB0k+tpYe6LXdTzz5+EZ9S0mSNI/saZYkafAm3x98Uc+xv2i3Z1bVlL3NA5ot+kCaRPxuYO+qurLPue43l5NU1dnA2e161I8HnkDT87wj8ExgjyR/UFUrp3h736WopjjeZZIySZIGxp5mSZIGKM3UzC9sX95BM1R54tgDgS3bl2dOU82jBhDKzu3269MkzFsxxyHgE6rqp1X1kap6RVXtRNObfjfwAKZeKxrgEUmm601/5ET1wLWDiFOSpK5MmiVJGqyXsmZ49nur6vpJx+496fmUyWKSDYD9pqn/zrXV0XOu6cocMs2xOamqM4FvtC9/r0+xDZh+hvCJf4crq2qQ9zP/ut3Oevi7JGl8mDRLkjQAafwl8OZ212rgdT3FfkozYzTAs/pU9Vamn7jrF6yZkfuh05T7frt9ZJLtp4j3kcCrp3n/tJJskaTv8Ook60+K72fTVHVCkvtO8f6DWbOe9XtmG2cfE/FM9+8nSRJg0ixJUhfrJ9mwfWyc5GFJ9kjyCuAK4ESa+UJ+BuxXVb+1fnG7BvOH25eHJfnnJDsleVCSxyf5MPBi4Jv9AmjXNJ44fkySnZPcN8m67czcEz5EMzz6XsA5SZ6d5MFtzEfQzHJ9C/DzWf5bPIVmsq9/T/LHSbZP8oAkWyXZl2YpqU3asu/vU8cPae59Pj/Jk9t/h+2SvIY1ifLVrFmCa1AubbeHJnlc+/Nct31k2ndKksaOE4FJkjRzH1/L8btoZsQ+Zprlll4J7AlsTTOU+6U9xz/Unudd05znROBfgd2Ar04+kGTbqlpZVVcn+VvgjTTDxT/SU8eNwAE0M13PZEmqqWwMvKh99HNCVX2sz7Grgb8HTmLq9Zp/CDyrqm6bZXz9vJ1msrLtuOdEbXsD5w/4fJKkEWZPsyRJs3M7zXDrq4D/BF4ObFtVB063PnFVraZJdt8OXENzj/L1wAXAC6rqQJoe4r6q6lTgYJrk7hf9ylfVm2iWhLqAZvbp24DvAqcAu1TV56d63wx9gGZ27LfSLDl1Hc2/ya3At4HTgMdV1XFracspwJOB/6ZZGuv2NsZ/Bh5VVd+aQ4z9zvkp4GnAJ2l+hr+e/h2SpHGWqlp7KUmSpAFJchpwKHBBVS0fbjSSJE3PnmZJkiRJkvowaZYkSZIkqQ+TZkmSJEmS+jBpliRJkiSpD5NmSZIkSZL6cPbsGdhkk01qm222GXYYkiRJkqR5cOmll15fVZtOdWzdhQ5mFG2zzTZccsklww5DkiRJkjQPklzT75jDsyVJkiRJ6sOkWZIkSZKkPkyaJUmSJEnqw6RZkiRJkqQ+TJolSZIkSerDpFmSJEmSpD5MmiVJkiRJ6sOkWZIkSZKkPkyaJUmSJEnqw6RZkiRJkqQ+TJolSZIkSepj3WEHIKm/bY49Z9ghzNrKE/YddgiSJEnSnJk0S5IkjTG/oJWk6Tk8W5IkSZKkPkyaJUmSJEnqw6RZkiRJkqQ+TJolSZIkSerDpFmSJEmSpD5MmiVJkiRJ6sOkWZIkSZKkPkyaJUmSJEnqw6RZkiRJkqQ+TJolSZIkSepj3WEHIHW1zbHnDDuEWVt5wr7DDkGSJElSB/Y0S5IkSZLUh0mzJEmSJEl9ODxb0tCN8pB7cNi9JEnSUmbSLEmSpLEwyl/S+gWtNDwOz5YkSZIkqQ+TZkmSJEmS+lh0SXOSg5N8IcmNSW5KckmSo5J0ijXJiiQ1zeO2+WqDJEmSJGlpWFT3NCc5GTgSuA34LHAnsA9wErBPkgOr6u6O1X4N+OoU+++cS6ySJEmSpKVv0STNSQ6gSZhXA3tW1dXt/s2B84D9gWOAEztWfXZVrRhgqJIkSZKkMbGYhmcf125fNZEwA1TVj4Ej2pfHdh2mLUmSJEnSbC2KBDTJlsCuwB3AB3uPV9UFwCrgwcDuCxudJEmSJGlcLZbh2bu02yuq6tY+ZS4GtmjLXtSh7j9I8mbgAcDPgS8B51TVHbMNVpIkSZI0HhZL0rxtu71mmjLX9pSdqWe1j8l+kOR5bQ+2JEnSPWxz7DnDDmHWVp6w77BDkKQlY1EMzwY2bLc3T1Pmpna70Qzr/C7NfdKPATYGNgWeBFwAbAl8PMmju4cqSZIkSRoXqaqpDyQPG9RJqura6Y4neTVwPPC+qnpenzLHA68GTq2ql8wlniRnAQfQDNN+Zp8yhwOHA2y++ea7nnnmmXM5pQbo8lU3DjuEWXvUFht3Kj8ubR3ldkL3n6uk0TDKv5vG5Xewf1clDcree+99aVUtm+rYdMOzVwJTZ9Td1FrOA2t6kTeYpsxEb/Sv5hwRvJ4maX5KkntV1T3WbK6qU4FTAZYtW1bLly8fwGk1CIeN8nC5Q5Z3Kj8ubR3ldkL3n6uk0TDKv5vG5Xewf1clLYS1JbO3ANfPof5NgPvOoNzKdrv1NGW26ik7F1e12/VoYvzRAOqUpLXyHklJkqTRsrak+YNV9cLZVp7kPcDzZ1D0K+12pyTr95lBe7eesnPxoEnPb+pbSpIkSZI01hbFRGBVdR1wGU3P70G9x5PsRTN512rgiwM45XPa7beqahDDvSVJkiRJS9B0SfNLgffPsf4zgJfNsOyb2u2bk2w/sTPJZsA725cnVNXdk44dneSqJKdPrijJw5IcnOTePfuT5M8mneutHdoiSZIkSRozfYdnV9WJc628qs4Fzp1h2bOSnAIcAVye5DPAncA+wP2As4GTet62CbADTQ/0ZA8E3gf8S5LLgB/SLFW1E2vWeT6pqv61c6MkSZIkSWNjbfc0L6iqOjLJhcBRwF7AOjSTdr0bOGVyL/NaXAf8I8190NsDj6XpVV8N/CfNslWfG3D4kiRJkqQlZlElzQBVdQbNsO6ZlF0BrJhi/8+AVw40MEmSJEnS2OmcNCfZAtgbeChwnz7FqqreMJfAJEmSJEkathknzUkCvA04kjUTiKWnWLX7CjBpliRJkiSNtC49za8AjgHuBj5Jc6/xL+cjKEmSJEmSFoMuSfMLaGezrqoL5ykeSZIkSZIWjenWae61LfAFE2ZJkiRJ0rjokjTfAPxkvgKRJEmSJGmx6TI8+3M06x5LkiRJWsS2OfacYYcwaytP2HfYIUi/pUtP82uATZO8Zr6CkSRJkiRpMenS07wH8B5gRZJnAJ8ArqWZTfsequr0uYcnSZIkSdLwdEmaT2PNOsx/CDx2LeVNmiVJkiRJI61L0nw6TdIsSZIkSdJYmHHSXFWHzWMckiRJkiQtOl0mApMkSZIkaayYNEuSJEmS1Eff4dlJ9pxLxVX1+bm8X5IkSZKkYZvunubzmf3EX7WWuiVJS9w2x54z7BBmbeUJ+w47BEmStEhMl9h+HmfLliRJkiSNsb5Jc1UtX8A4JEmSJEladJwITJIkSZKkPkyaJUmSJEnqY1aTdSXZEngocJ9+ZZw9W5IkSZI06jolzUmeC7wO2G4tRZ09W5IkSZI08mac2CY5BDgdCPBz4PvATfMUlyRJkiRJQ9elN/hV7fYo4NSqumse4pEkSZIkadHokjQ/Ariwqk6Zr2AkSZIkSVpMusye/TNg1XwFIkmSJEnSYtMlaf4U8Ngkma9gJEmSJElaTLokzX8HbAC8JYkzY0uSJEmSlrwZJ79V9YMkewAfA/ZLch7wA+DuqYvXGwYUoyRJkiRJQ9FlyanfAV4O7EDTQ73tFMWKZkmqAmaVNCc5GDgCeDSwDnAV8B7glKqaKkHvUvfhwL+2L0+uqqPnUp8kSZIkaWnrMsz6OJpk9k6a3ubvMOB1mpOcDBwJ3AZ8tj3XPsBJwD5JDpxt4pxka+AtrEnsJUmSJEmaVpek+YXAzcAeVfX1QQeS5ACahHk1sGdVXd3u3xw4D9gfOAY4cRZ1B3gXTQ/56cChAwpbkiRJkrSEdZkI7CHA5+cjYW4d125fNZEwA1TVj2l6uAGObYeJd/UXND3WxwEr5xKkJEmSJGl8dElAVwG3zkcQSbYEdgXuAD7Ye7yqLmjP/2Bg9451bwv8A3AhzTBvSZIkSZJmpEvSfCawPMmG8xDHLu32iqrql5hf3FN2rdph2e+mGYb+oqqq2YcoSZIkSRo3XZLmvweuBP47ye8OOI6JmbivmabMtT1lZ+JoYDmwoqq+PYu4JEmSJEljrMtEYJ+gSbKfAFyR5BqmX6d5nw51T/Re3zxNmYmZujeaSYVJtgNOAC6hmTVbkiRJkqROuiTNyyc9Xwd4ePuYylCHQU8aln0vmmHZd82ijsOBwwE233xzzj///IHGqNl7+aN+PewQZq3r52hc2jrK7YTxaaufX42bcfkMj0s7wbaOCn8Ha7HpkjTvPW9RrOlF3mCaMhO90b+aQX1/CewJvH62s31X1anAqQDLli2r5cuXz6YazYPDjj1n2CHM2spDlncqPy5tHeV2wvi01c+vxs24fIbHpZ1gW0eFv4O12Mw4aW5nsJ4vK9vt1tOU2aqn7HT2b7dPSbJXz7FtJsok+X3gpqp65gzqlCRJkiSNmS49zfPpK+12pyTr95lBe7eesjPxuGmOPbR93NihPkmSJEnSGOkye/a8qarrgMuA9YCDeo+3vcVbAquBL86gvuVVlakewOvaYie3++4/uJZIkiRJkpaSvklzkrcnOXgulSc5JMnbZ1j8Te32zUm2n1THZsA725cnVNXdk44dneSqJKfPJU5JkiRJkqYyXU/z0cCT51j/k4GjZlKwqs4CTgEeDFye5GNJPgxcDewInA2c1PO2TYAdgIfNMU5JkiRJku5hsdzTDEBVHZnkQppEey+apa2uolk+6pTJvcySJEmSJM23tSXNhyY5dEEiaVXVGcAZMyy7AljRsf7O75EkSZK0OG0zystrnbDvsEPQDKwtac4AzlEDqEOSJEmSpAXXN2muqkUxs7YkSZIkScNiYixJkiRJUh8mzZIkSZIk9WHSLEmSJElSHybNkiRJkiT1YdIsSZIkSVIfJs2SJEmSJPVh0ixJkiRJUh8mzZIkSZIk9WHSLEmSJElSHzNOmpNsm+Q5Sbbu2b9zkguT/CrJ5UmePvgwJUmSJElaeF16ml8OvB9Yd2JHkvsB5wKPBzYAdgL+K8kjBxmkJEmSJEnD0CVp3hP4ZlV9d9K+5wGbAGcC2wMvA9YD/nJgEUqSJEmSNCRdkuaHACt79j0VuBt4aVV9r6reBlwB7DWY8CRJkiRJGp4uSfPGwA09+3YHvl5VP56070pgi7kGJkmSJEnSsHVJmn8FPHTiRZIdgE2Bi3rK3d2xXkmSJEmSFqUuye3XgMcn2a59/WKggPN7ym0L/GjuoUmSJEmSNFxdkuZTaSb5uizJZcBLgZ8A/z1RIMlGwGNo7muWJEmSJGmkzThprqozgdfTLDn1GOAa4KCqum1SsefQJNbnDzBGSZIkSZKGYt21F1mjqlYkeSNwv6q6fooi5wK7AN+d4pgkSZIkSSOlU9IMUFV3AFMlzFTVtcC1cw1KkiRJkqTFwFmuJUmSJEnqo1NPc5J1aO5b3odm+an79ClaVbXPHGOTJEmSJGmoZpw0J3kA8GngD4CspXjNJShJkiRJkhaDLj3NxwO7AtcBJwFXAb+cj6AkSZIkSVoMuiTNzwZ+AfxhVa2ep3gkSZIkSVo0ukwEtglwoQmzJEmSJGlcdEmafwj8er4CkSRJkiRpsemSNH8I2DPJ+vMVjCRJkiRJi0mXpPl1NL3N/5lks3mKhyQHJ/lCkhuT3JTkkiRHJem0pnSSQ5L8vySXJ/lpkjuT/CLJhUmOTnKv+WqDJEmSJGlp6DIR2NuB7wD7A1cnuRS4Frh7irJVVS/qGkySk4EjgduAzwJ30qwJfRKwT5IDq2qq803lCOBxwJXAxcCNNGtLPw7YAzgkyZOr6uaucUqSJEmSxkOXpPkw1qy/vBGwfJqyBXRKmpMcQJMwrwb2rKqr2/2bA+fRJOvHACfOsMqXAd+uqht6zrMlcC6wO/BK4O+6xClJkiRJGh9dkuYXzFsUjePa7asmEmaAqvpxkiOA84Fjk7xjJr3NVfXlPvt/kOSNwOnAUzBpliRJkiT1MeOkuareO19BtL2/uwJ3AB+c4twXJFkFbEHTQ3zRHE85MQv47XOsR5Iktjn2nGGHMGsrT9h32CFIkrSodZpcax7t0m6vqKpb+5S5uKfsrCTZBHhF+/Kjc6lLkiRJkrS0dRme/RtJ1qPpGd6i3bUKuLSq7phlHNu222umKXNtT9kZSfIs4ABgHeAhNJOA3Qc4jWaCMUmSJEmSptQpaW6XaVoBHEUzGdhkNyV5B/C6qrqzYxwbttvpZrK+qd32nndtdgYO7dn3NmDFLOKUJEmSJI2RVNXaSwFJ1gE+DjwZCPAj4Hvt4YfT9OIW8BngGVV114yDSF4NHA+8r6qe16fM8cCrgVOr6iUzrXvS+9cDtgaeAxwL/KyN88o+5Q8HDgfYfPPNdz3zzDO7nnJBXb7qxmGHMGuP2mLjTuVt62jo0tZRbieMT1v9/PY3Tm0dJ+Pycx2XdoJtHRW2VcOw9957X1pVy6Y61qWn+XCa2aa/DfzfqvrU5INJnkrTg/tk4MXAv3Soe6IXeYNpykz0Rv+qQ72/0Q4dvxo4PslVwFnA6Ul2qym+OaiqU4FTAZYtW1bLly+fzWkXzGGjPAnNIcs7lbeto6FLW0e5nTA+bfXz2984tXWcJj0bl5/ruLQTbOuosK1abLpMBPZ8muHT+/QmzADtvicDt3DP4dBrs7Ldbj1Nma16ys7Fh4Ff0tyXvc0A6pMkSZIkLUFdkuYdgfOqalW/Au2x89qyXXyl3e6UZP0+ZXbrKTtrbc/yz9qXm821PkmSJEnS0tQlab4XTS/y2tzSlp2xqroOuAxYDzio93iSvYAtgdXAF7vUPZUkD6fpYb6bNfdlS5IkSZL0W7okzdcAT2wn1JpSe+yJTL90VD9vardvTrL9pDo3A97Zvjyhqu6edOzoJFclOb0njh2THJzkPlPE+PvAB2gmM/uvqvrpLGKVJEmSJI2BLhOBfRR4BfDeJEdU1Q2TDybZGDgZeDDw/7oGUlVnJTkFOAK4PMlngDuBfYD7AWdzz3WVNwF2oOmBnmwz4H3AzUkuo1lH+t40vcuPoUmYvwx0noVbkiRJkjQ+uiTN/wA8l2bJpqcn+RjwfZplph4OPItmDeUftGU7q6ojk1xIsw70XsA6wFXAu4FTJvcyr8UVwN/S9Ho/kmbCr3WB64FP0PQ0/0eXZbEkSZIkSeNnxklzVf0syZOAM4BlwCE0CTM0PbcAFwMHV9XPZxtQVZ3RnmMmZVcAK6bY/1OadZ8lSZIkSZq1Lj3NVNV3gMcmeQJNT/AW7aFVwAVVdeGA45MkSZIkaWg6Jc0T2uTYBFmSJEmStKR1mT1bkiRJkqSxYtIsSZIkSVIffYdnJ7mLZqKvHavq2+3rmaqqmtXQb0mSJEmSFovpEtuwZlZsep6vTZeykiRJkiQtSn2T5qr6neleS5IkSZK01JkIS5IkSZLUx4zvO07yfOA7VXXRWsrtDvxuVZ0+1+AkSZIkSY1tjj1n2CHM2soT9h12CLPWpaf5NODPZ1DuRcB7ZhWNJEmSJEmLyHwMz3YSMEmSJEnSkjAfSfOWwE3zUK8kSZIkSQtq2nua2/uYJ9t+in2T6/o9YB/g4gHEJkmSJEnSUK1tIrDTgJr0eo/20U+Au4G3zC0sSZIkSZKGb21J8+msSZoPBb4L/E+fsncAq4CPVNXXBhOeJEmSJEnDM23SXFWHTTxPcihwYVW9cL6DkiRJkiRpMZjxOs3AtjjBlyRJkiRpjMw4aa6qa+YzEEmSJEmSFpsZLzmV5OAk30vy1GnKPK0tc9BgwpMkSZIkaXi6rNP8XGBj4LxpypwH3B84ZC5BSZIkSZK0GHRJmh8NfL2q7uhXoKpuB74G7DzXwCRJkiRJGrYuSfPmwA9nUO6HbVlJkiRJkkZal6T5ZmCzGZTbFLh9duFIkiRJkrR4dEmavw7skaRvL3KSBwNPAL4x18AkSZIkSRq2Lknz+4H7AGcleWDvwXbfB4B7t2UlSZIkSRppM16nGXg3cBiwB/C9JB8FrmqP7QD8H+B+wJeBfxtgjJIkSZIkDcWMk+aq+nWSZwCnAc8GngdUezjt9mPAYVV15yCDlCRJkiRpGLr0NFNVNwD7JdkZeBqwNU3ifC3wqar66uBDlCRJkiRpODolzROq6ms06zFLkiRJkrRkdZkIbEEkOTjJF5LcmOSmJJckOSrJjGNN8jtJHp/k75NclOQXSe5M8uMkH0+y33y2QZIkSZK0NMyqpxkgycY0E39lquNVde0s6jwZOBK4DfgscCewD3ASsE+SA6vq7hlU9XDgf9rnP6eZnOwX7f6nA09PchrwwqqqKWuQJEmSJI29Tklzu6zUG4ADgE2nKVqzqPsAmoR5NbBnVV3d7t8cOA/YHzgGOHEG1RXwOeAfgXOr6q5J59kLOIdmJvDPA+/pEqckSZIkaXx0GfL8AOBLwF8ADwRupellXj1RpN1eC1w3i1iOa7evmkiYAarqx8AR7ctjZzJMu6q+W1X7VNUnJyfM7bELgBPal8+bRZySJEmSpDHR5Z7mVwHb0fTMbgycBVRVbQFsBLyEZij0hVW1bZcgkmwJ7ArcAXyw93ib6K4CHgzs3qXuPr7SbrccQF2SJEmSpCWqS9L8LOCnwFFVdStr1mimqm6pqn+juV/4uUmO7BjHLu32irbuqVzcU3YuHtFufzSAuiRJkiRJS1SXpHkb4JKqur19XQBJ1pkoUFWXABcCL+oYx0TP9DXTlJmYWKxTL3avJPcF/rJ9+aG51CVJkiRJWtq6JM13Ab+c9PrmdrtJT7kfsqYnd6Y27KlzKje124061t3rnTSJ95XAqXOsS5IkSZK0hGWmKy4luQq4vqqe0L5+OfAPwLOq6uOTyn0F2LKqpptdu7fuVwPHA++rqikn50pyPPBq4NSqeslM6+6p4zXA64EbgT2q6oppyh4OHA6w+eab73rmmWfO5pQL5vJVNw47hFl71BYbdypvW0dDl7aOcjthfNrq57c/2zoabOvUxqWdYFtHhW3tb5zautD23nvvS6tq2VTHuiwLdRnwR0nWaWek/izNjNknJPk+8AOaJaN2plnuqYuJXuQNpikz0Rv9q451A5DkZTQJ803A06dLmAGq6lTanuhly5bV8uXLZ3PaBXPYsecMO4RZW3nI8k7lbeto6NLWUW4njE9b/fz2Z1tHg22d2ri0E2zrqLCt/Y1TWxeTLsOzP0Gz1NTTAKrqq8DHgN8HvgHcALyR5l7n13eMY2W73XqaMlv1lJ2xJMcA/0SzTNYzq+qLXeuQJEmSJI2fLoZ0jzsAABwmSURBVEnz+2kS1wsm7TsYOBn4CfBrmuT5OVX1+Y5xTCwBtVOS9fuU2a2n7IwkOQp4O3Ab8Ox2+SpJkiRJktZqxklzVf26qlZV1U2T9t1cVcdU1UOq6t5VtXNVdZ6Ruqquoxn+vR5wUO/xJHvRrKm8GphxL3GSvwBOAm4H9quqz3SNTZIkSZI0vmacNCf55ySvncdY3tRu35xk+0nn3YxmxmuAE6rq7knHjk5yVZLTp4j3xe37bgf2r6pPzV/okiRJkqSlqMtEYMcAH52vQKrqrCSnAEcAlyf5DHAnsA9wP+Bsml7jyTYBdqDpgf6NJI8B/pVmorLvA3+S5E+mOO31VfXXA22IJEmSJGnJ6JI0r6a5b3neVNWRSS4EjgL2AtYBrgLeDZwyuZd5Le5PkzADPLJ9TOUawKRZkiRJkjSlLknzZ4CnJFm3quYtea6qM4AzZlh2BbBiiv3nsyZpliRJkiRpVrrMnv13wPrAvySZbj1lSZIkSZKWhC49zYfRrNX8AuDZ7T3H19CsfdyrquoNcw9PkiRJkqTh6ZI0rwCKZtjzJsCfTlFm4ngBJs2SJEmSpJHWJWl+PU0yLEmSJEnSWJhx0txOuiVJkiRJ0tjoMhGYJEmSJEljpcvw7N9IsjGwG7ApcE1VXTTQqCRJkiRJWgQ69TQn2TjJu4GfAJ8C/gP480nH/zzJD5PsPtgwJUmSJElaeDNOmtu1mc+nWXrqFzTLT6Wn2H8DmwP7DSY8SZIkSZKGp0tP818DO9P0Lj+8qp7ZW6CqVgNXAk8aTHiSJEmSJA1Pl6T5IOCHwIur6pZpyn0b2GJOUUmSJEmStAh0SZofDlxcVbevpdxtwINmH5IkSZIkSYtDl6T5TuA+Myi3FXDT7MKRJEmSJGnx6JI0fwvYJcm9+xVI8gCa+54vn2tgkiRJkiQNW5ek+SxgM+DN05R5I7Ah8IG5BCVJkiRJ0mKwboeyJwGHAsckWQZ8uN2/TZIjaCYK24uml/ldA41SkiRJkqQhmHHSXFW3JPkj4IPA44HHtYf2ah8BLgX2q6o7Bh2oJEmSJEkLrUtPM1W1Cnh8kqcBz6CZUXsd4DrgE8DZVVUDj1KSJEmSpCHolDRPqKpPAp8ccCySJEmSJC0qM54ILMlrkzx7BuWeleS1cwtLkiRJkqTh6zJ79gpgvxmUezbwd7OKRpIkSZKkRaRL0jxT6wDe1yxJkiRJGnnzkTRvB/xyHuqVJEmSJGlBTTsR2BT3Jj9mmvuV1wV+D3gCcP7cQ5MkSZIkabjWNnv2Cpqh1mlfP6Z9TOcW4PVzC0uSJEmSpOFbW9L8etYkza8Fvgp8pE/ZO4BVwKeq6scDi1CSJEmSpCGZNmmuqhUTz9th2V+tqtfNd1CSJEmSJC0Ga+tp/o2qmo9JwyRJkiRJWrRMhCVJkiRJ6mPGPc0TkmwB7A08FLhPn2JVVW+YS2CSJEmSJA3bjJPmJAHeBhzJmh7q9BSbmDSsgFklzUkOBo4AHg2sA1wFvAc4paru7lDPVsAzgWXAbsCObX2vqKq3zCY2SZIkSdJ46dLT/ArgGOBu4JM0yewvBxlMkpNpkvLbgM8CdwL7ACcB+yQ5sEPifADw1kHGJ0mSJEkaL12S5hfQJrFVdeGgA0lyAE3CvBrYs6qubvdvDpwH7E+TtJ84wyq/35a9FLgEOA74swGHLUmSJElawrpMBLYt8IX5SJhbx7XbV00kzADtms9HtC+PTTKjmKvqI1X1V1X1/6rqmzQ95JIkSZIkzViXpPkG4CfzEUSSLYFdgTuAD/Yer6oLgFXAg4Hd5yMGSZIkSZJ6dUmaP0czodZ82KXdXlFVt/Ypc3FPWUmSJEmS5lWXpPk1wKZJXjMPcWzbbq+Zpsy1PWUlSZIkSZpXqaqZFUyeT9PL+5fAl4FP0CSyU94rXFWnzziI5NXA8cD7qup5fcocD7waOLWqXjLTuie9/zTgUGa45FSSw4HDATbffPNdzzzzzK6nXFCXr7px2CHM2qO22LhTeds6Grq0dZTbCePTVj+//dnW0WBbpzYu7QTbOipsa3/j1NaFtvfee19aVcumOtZl9uzTWLMO8x8Cj11L+RknzYtRVZ0KnAqwbNmyWr58+XADWovDjj1n2CHM2spDlncqb1tHQ5e2jnI7YXza6ue3P9s6Gmzr1MalnWBbR4Vt7W+c2rqYdEmaT6dJmufDTe12g2nKbNhufzVPMUiSJEmS9FtmnDRX1WHzGMfKdrv1NGW26ikrSZIkSdK86jIR2Hz6SrvdKcn6fcrs1lNWkiRJkqR5tSiS5qq6DrgMWA84qPd4kr2ALYHVwBcXNjpJkiRJ0rjqOzw7yZ5zqbiqPt/xLW8CPgi8OclFVfWdNo7NgHe2ZU6oqt/M1p3kaOBo4MtV9fy5xCtJkiRJUq/p7mk+n9lP/FVrqfueb6g6K8kpwBHA5Uk+A9wJ7APcDzgbOKnnbZsAO9D0QP+WJA8B/mvSru3a7TFJDpy0f/+q+lGXWCVJkiRJ42G6xPZa5m+27ClV1ZFJLgSOAvYC1gGuAt4NnDK5l3kG7k2zNFavh7WPyeUkSZIkSbqHvklzVW2zgHFMPu8ZwBkzLLsCWNHn2EqaNaUlSZIkSZqVRTERmCRJkiRJi5FJsyRJkiRJfZg0S5IkSZLUh0mzJEmSJEl9mDRLkiRJktSHSbMkSZIkSX2YNEuSJEmS1IdJsyRJkiRJfZg0S5IkSZLUh0mzJEmSJEl9mDRLkiRJktSHSbMkSZIkSX2YNEuSJEmS1IdJsyRJkiRJfZg0S5IkSZLUh0mzJEmSJEl9mDRLkiRJktSHSbMkSZIkSX2YNEuSJEmS1IdJsyRJkiRJfZg0S5IkSZLUh0mzJEmSJEl9mDRLkiRJktSHSbMkSZIkSX2YNEuSJEmS1IdJsyRJkiRJfZg0S5IkSZLUh0mzJEmSJEl9mDRLkiRJktTHokuakxyc5AtJbkxyU5JLkhyVZFaxJnlakk8n+XmSW5J8I8nfJLn3oGOXJEmSJC0tiyppTnIy8D5gGfAF4Fzgd4GTgLO6Js5JXgl8AngScBlwDrAZ8PfA+UnuO7joJUmSJElLzaJJmpMcABwJrAYeXVXPrKr9gUcA3wT2B47pUN8y4ATgFmCPqnpyVR0EPBz4PLA7cPxgWyFJkiRJWkoWTdIMHNduX1VVV0/srKofA0e0L4/t0Nt8LBDgzVX1pUn13QS8ALgbODLJ/eccuSRJkiRpSVoUSXOSLYFdgTuAD/Yer6oLgFXAg2l6iNdW33rA09uX75uivu8BXwTWA54x68AlSZIkSUvaokiagV3a7RVVdWufMhf3lJ3ODsB9gZ9X1XcHUJ8kSZIkaQwtlqR523Z7zTRlru0pO5P6rp2mTJf6JEmSJEljaLEkzRu225unKXNTu91oCPVJkiRJksZQqmrYMZDk1TQzWb+vqp7Xp8zxwKuBU6vqJWup72Cae5n/p6qe0KfMi4FTgU9X1VOnOH44cHj7cgfgWzNszlK1CXD9sINYAOPSTrCtS5VtXXrGpZ1gW5cq27o0jUtbx6WdMF5tncrWVbXpVAfWXehI+pjo9d1gmjITvce/Woj6qupUmqRaQJJLqmrZsOOYb+PSTrCtS5VtXXrGpZ1gW5cq27o0jUtbx6WdMF5t7WqxDM9e2W63nqbMVj1lZ1LfwwZUnyRJkiRpDC2WpPkr7XanJOv3KbNbT9npXAXcCjwwyXZ9yjy2Q32SJEmSpDG0KJLmqroOuIxm3eSDeo8n2QvYElhNs77y2uq7A/hE+/KQKep7OPA4mnWhz5l14ONlXIaqj0s7wbYuVbZ16RmXdoJtXaps69I0Lm0dl3bCeLW1k0UxERhAkgOBD9Ikxk+squ+0+zcDzgN2BP6qqk6c9J6jgaOBL1fV83vq2w34Ek2P895V9eV2/4bAfwN7AW+rqpfOd9skSZIkSaNpUfQ0A1TVWcApwIOBy5N8LMmHgatpEuazgZN63rYJzczW97h3uaouBo4F7gtclOTTST4AfJcmYf4S8Dfz1BxJkiRJ0hKwaJJmgKo6kmY49WU0ie1Tge/Q9CYfUFV3dazvH4Cn0/RU7wY8i2Ya9b8F9qqqWwYX/dKRZIck/zfJfyS5KsndSaodDbBkJLlXkn2S/FOSS5L8MskdSVYlOSvJ8mHHOEhJjknygSTfTPKzJHcm+WmSzyR5XpIMO8b5kuSN7We4kvz1sOMZpCSnTWrbVI+rhh3jICVZP8krk1yc5IYktyT5fpIPJtlj2PHNVZLla/l5Tn5MN9nlyEiyZZJ3JPlWkluT3Jbk6iT/0t5OtSQk2SrJSUm+m+T2JNcn+VSSfYcdW1dzuU5IcnCSLyS5MclN7d/fo5IsqmvSCbNp66heR3WNe5Svo2b5cx3J66hBfR6X8rXUTCyWJad+o6rOAM6YYdkVwIq1lPkk8Mk5BzZejgD+77CDWAB7Aee2z1cDnwduphnZcABwQJI3VNVrhxTfoL0K2Az4BnARTVu3Bp4E7AMcmOSPq+ru4YU4eGlu1XglUMCi/IM2IP9D8yVjrx8tdCDzJcm2wKeB7WnadR7wa5rP8X7A12j+HUbZauC90xx/LPB7NKOmrluQiOZRkl2AzwH3B34AfKo9tAx4CXBIkqdW1UVDCnEg2t9DnwQeCFxDM5/Kg2l+//5RktdX1d8NMcSuZnWdkORk4EjgNuCzwJ00f39OAvZJcuAi/Bs0m7aO6nVU17hH+TpqNj+jUb2OmvPncYyupfpadEmzFoVvAP8IXAJcCryL5hfjUnM38CHgxKr6wuQDSf4EeB/wmiTnVdV5wwhwwP4U+EpV3Tx5Z5KdaC5e/g9wKPCeIcQ2L5LcmyYB+THwZZrEaqn696o6bdhBzJckG9BcnD2c5tabt0wefZTkQcCDhhTewFTVVcBh/Y4nubJ9+u5aLJOSzM3JNAnzvwFHVdWd0PRgAf8CvJDm1q2dhxbhHCW5D83fmgcC7wBeVlW/bo89niaBfm2SC6vq3P41LSqdrxOSHECTMK8G9qyqq9v9m9N8AbY/cAxwYt9KhmM210Sjeh3VNe5Rvo6azc9oVK+j5vR5HLNrqb5MmnUPVfXvk18v0tEmc1ZVn6Pp4Zjq2H8meQrwIuB5NH/QR1pVXdhn/xXtt/+vB57C4vtlPxevp+mVezbNt94aXX8LbAecVFVv7j1YVT8DfrbgUS2gJI+j+TzfBZw23Gjmrk0mH9e+/LuJhBmgqu5M8rc0SfOjk9x3hG+p2h/YimZ0wMsnEmaAqrooyfE0F7SvZU2v3aI2y+uE49rtqyYS5rauHyc5AjgfODbJOxZTT91s2jqq11Fd4x7l66hZ/lxH8jpqAJ9Hr6VYZPc0S4vMxBreWw41ioUxcRF3+1CjGKAkfwi8HDijqj427Hg0e0nWA17cvvznYcYyZC9st5+sqh8ONZLBuIs1v3umczPNShijard2e8HkLwYm+XS73SPJgxcopgWVZEtgV5qlPj/Ye7yqLgBW0QxZ331ho9M88jpqxHkttYY9zVJ/j2i3S+ae0Km094n+Rfvyo8OMZVDaHqz3Aj9nNO8rm429kzwa2JBmCNWFwLmLqcdmDnalGXq9qqq+n+QPaHrvNqNp66f79QAsFUnuC/xJ+/Jdw4xlUNre5M/STPr5uiS9w7Pf0BZ914gPRd+w3V7f5/jE/gB/AHx83iNaeLu02yuqqt8XIBcDW7RlR/oedv2G11EjbEyvpfoyaZam0H7bf1j78kNDDGXgkryA5l6We9F8+/t4mlEnb6yq/xpmbAN0PM1ydH9aVf0uVJea50+x78okf1pVly94NIP1qHa7KslbaL71nuw1Sc4Gntd7r9kSchCwEfAT4L+HHMsgHUkzQdaLgacnuaTdvxvwAOBtNJPPjLKftNt+M4FvN+n5tvMcy7BMtOuaacpc21NWI8zrqCVhHK+l+nJ4ttQjybrAfwAbA59dgsNR9qCZqOJgYM9232tY06sz0tqJdf4KOLuq/nPY8SyArwJ/STNb6YbAQ4Fn0swkvSPwmSRbDC+8gXhgu92FJmF+G80M2g+gmXhlFc3EJO8cSnQLY2Jo9ul9hviOpKr6Hs0F5ydoLj73ax9bAFcCX1gC7Z2453PfdphyryMmPb/fAsQzDBO97dN9qXVTu91onmPRPPM6avSN4bXUWpk0S/f0LzRLB1xHM3nFklJVf15VAe4L7ESTgKwA/jfJQ4cZ21wlWZ9mgqRf0vRgLXlV9baqekdVfbOqbq6qH1XVOTRLE/0vzRDm46avZdGb+Ft1L+A/quqlVfXdqrqhqj5Kk2QV8GdJtutby4hKsj1rLszePcxYBq29MPsGzZcg/wfYtH3sR/OlyIeSLMblamasnSzp88D6wLnturYbJdk+yTuA59IsvQTNbMTSqPM6aoSN47XUTJg0S5MkOZFmpsfVwD5VtXrIIc2bqrq1qq6sqlfQJFU706yVOcreSHMP1cuqaknfQ7U2VXUH8Kb25TOGGcsA/GrS83/rPVhVE8tohNFY1qWriV7mL1bVN4cayQAluT9wNk3P4tOq6qNVdX37+AjwNJoJwF6T5BHT1TUCDqJZQ/yRwGdoLkavBo6mueD+Wlvu50OJbv5N9CJvME2Zid7oX01TRouc11Ejfx0FXktNyXuapVaSf6IZ5vpTml/0V6/lLUvJacBbgGcludcID4fcn6an5tAkh/Yce2S7PSLJM4HvVNWfL2h0C++qdjvqw7O/3+d5b5llNLPvLhlJ1mHN/epLYgKwSfal6VX+XDtM+7dU1XeSfAlY3j5G9ndyVf0kyROBJwNPopnY7sfAR6rqkiQTs6GP+vwD/axst1tPU2arnrIaMV5HLYnrKPBaakomzRKQ5B+Al9Gs8/rkqrpyyCEttF/QLJewLs39oz8ebjhz8jtM39v48PZx/4UJZ6ge1G5vmrbU4veVSc8fRDPkr9cm7XbU29rrqTRfetwELLX7yh7Wbm+cpswN7faB05QZCe0M4OfSsxZze0vBQ2j+/lw2hNAWwsT/4Z2SrN9nBu3despqhHgdtaSuo8BrqXtweLbGXpITgFfQ/MJ7SlV9fcghDcOeNL/ob6D/siiLXlVtU1WZ6kGzbALAK9p9jxlmrAvkOe324qFGMUdVtQr4Uvtyn97jSR5As1QPwCW9x0fci9rtB6pqqX0hMNG7umu7xNRvafft2r7sN8JgKfjrdntqe1vFklNV19F8IbAezVD135JkL5qJ4FYDX1zY6DRXXkcBS+Q6CryW6sekWWMtyd8Dr6L5JfeUqlqS33AneUKSZ7YzWvYe24M1wz7fVVV3LWx0mq0kj2l/ruv07F83yctphskBvHXhoxu449vtq5Msm9jZriN5Cs0srZeyhC64k2wCPKt9udSGZkMzY/YtND3Ob01y74kD7fO30wzZ/QXwqaFEOCBJHpVkg5596yb5G+AlwHdY8xlfqibmWHhzO7kdAEk2Y83M9ycskbXlx4bXUV5HjYs0o4WkNZL8Ab+9dMuONBO1XM2kSUqqavcFDm2gkjwb+Ej78hLgij5Fr6qqExYmqvmR5DDgPTR/1C6j+TZ/I5r1QXdsi50DHNRn2NzIS3IazRIRr6iqtww5nIFIsh/wXzT/Ly+jWQ/2QTTrGj+U5p6kY6vqH4cW5ABNWqP5TpqZwX9GM0v4Q2mWndp7Kd1Dl+SlwD/T/A76vWHHMx/a++XeBaxD0/M8MTx5V5ohy7fTrBF69nAiHIz2989BNO1bRTOT9u40s9tfDfxRVa0cVnxdzfY6Ick7aZbYuo1mQrQ7aUaP3I9mUrgDF1vCMZu2jup1VNe4R/k6ahZtPYwRvY4a5OdxKV5LzZT3NGsq9wP+cIr9oz57aa/J98gtax9TuQBYVL/sZ+ECmvUDn0jzc3w8zUzDq4EP0SzjM9IXpWPqa8CJNInjjjQ/3wJ+QPPH/eSqunR44Q1WVf11kotoZhzehWa5j2tpEssTquqnw4xvHryg3S6pZaYmq6r3JrmcZj3QJwJPaQ+tokmm/3mJ3Bt5Ns199zvT3Lt7G/At4B9o/p/eNsTYZmNW1wlVdWSSC4GjaO6XXIdmwsJ3A6cs0l7m2bR1VK+jusY9ytdRXds6ytdRo/p5XFTsaZYkSZIkqQ/vaZYkSZIkqQ+TZkmSJEmS+jBpliRJkiSpD5NmSZIkSZL6MGmWJEmSJKkPk2ZJkiRJkvowaZYkSZIkqY91hx2AJEmauyQrga0n7SrgZuAG4FvAxcD7q+rrCx+dJEmjK1U17BgkSdIcTUqaPwWsbnffF/j/7d1LqJVVGIfx528UWmCZGQ4sqESyy8AMulCGUDQQIwmjoKwou9HAQWEIRU0qDDMItUQii6hmJ4QKIhAHQWiBRSQhljQoIo9Fgy6ab4Pv27g7nl2hx3M8Zz+/ydprrW9dvtl+97rsGcA84PS2bAtwf1X9MLQPSZJ0JINmSZImgK6geWFVbR1SNwlYDLwAnA98DVxdVftGeZqSJI07nmmWJGmCq6pDVfUucDmwG5gDrBnbWUmSND4YNEuS1Ceqaj+wos3ekWRmpy7J9UnWJdmZZF+SP5LsTbI5ydyhfSX5KEklua3XeEnWtM+sHvm3kSRpdBg0S5LUX94DBoGTgIVd5S8D9wIHgW3tc38Cy4AdSa4Z0s9LbfrwcIMkmQLcAxwCNozU5CVJGm0GzZIk9ZFqLjP5rM1e3FX1KDCzquZX1ZKqWkKzjftBmgvFNiZJ1/NbgL3AtUkuGWao24FpwPtV9c1Iv4ckSaPFoFmSpP7zU5tO7xRU1UBV/dz9UDVeAT4G5gIXddX9Baxvs8OtNnfK1g9TJ0nSuOH/NEuS1H86P5of6i5MMgtYBFwITKXZwg3QOfs8B/iyq8km4Cma89Erq+rXtp8rgPnAHuCD4zB/SZJGjUGzJEn956w2HewUJHkaWMW/fzeY2p2pqsEkbwL3AXdy5Mrzhqr6R2AuSdJ44/ZsSZL6SHsueV6b/aItuwV4EvgNWA5cAJxaVamqAG91mg/TZedCsIfavqYDtwK/A68ej3eQJGk0udIsSVJ/WURzQdcBYGtbtrRNV1XVpmHazO7VWVV9nmQbsCDJAuBKYDLwWlUN9monSdJ44UqzJEl9Isk0YG2bfb2qfmw/n9mm3w3TZi6HV6Z76aw2P0Jz2zbAumOYqiRJJwyDZkmSJrgkk5LcBGynWTXeBTzW9ciuNl2e5JSudmcDm/nvnWkDNAH3UuA8YHtV7Rih6UuSNKbcni1J0sTyeJK728+TgRnAZcAZbdkA8EBV7e9q8yKwjGbr9u4knwBTgOtoguEB4OZeA1bVwSQbgGfaIleZJUkThivNkiRNLDcCd3E4CJ4DfAo8C1xaVUu6tmUDUFV7aLZgv01z2ddimv9l3ghcBfzyP8b9sE33Ae8c+2tIknRiSFWN9RwkSdI4l2QtsAJYXVUrx3o+kiSNFINmSZJ0TJKcA3wFnAzMrqojLhSTJGm88kyzJEk6KkmeA2YBNwCnAc8bMEuSJhpXmiVJ0lFJ8i1wLvA98AbwRFUdGNNJSZI0wgyaJUmSJEnqwduzJUmSJEnqwaBZkiRJkqQeDJolSZIkSerBoFmSJEmSpB4MmiVJkiRJ6sGgWZIkSZKkHv4GZBjGWnPqD2QAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"fiCjNxb-6X8N"},"source":["## Preprocessing"]},{"cell_type":"markdown","metadata":{"id":"phtxBLx6-t42"},"source":["### Splitting"]},{"cell_type":"code","metadata":{"id":"ai2-ca5W-26O"},"source":["random_seed=10\n","validation_target_users_size = 10000\n","validation_fraction_users = 0.2\n","validation_fraction_items = 0.2"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"vOd93OF3-vB4"},"source":["# split into train_and_validation and test\n","train_and_validation, test = split(interactions)\n","train_and_validation.to_csv('train_valid.gzip', compression=\"gzip\", index=None)\n","test.to_csv('test.gzip', compression=\"gzip\", index=None)\n","\n","# split into train and validation\n","interactions_subset = get_interactions_subset(\n"," interactions=train_and_validation,\n"," fraction_users=validation_fraction_users,\n"," fraction_items=validation_fraction_items,\n",")\n","train, validation = split(interactions_subset)\n","train.to_csv('train.gzip', compression=\"gzip\", index=None)\n","validation.to_csv('validation.gzip', compression=\"gzip\", index=None)\n","\n","# prepare target_users\n","test[\"user\"].drop_duplicates().to_csv('target_users_all.gzip',\n"," header=None,\n"," index=None,\n"," compression=\"gzip\"\n",")\n","\n","# prepare target_users for validation\n","np.random.seed(random_seed)\n","validation_users = validation[\"user\"].drop_duplicates()\n","validation_users.sample(\n"," n=min(validation_target_users_size, len(validation_users))\n",").to_csv('target_users_subset_validation.gzip',\n"," header=None,\n"," index=None,\n"," compression=\"gzip\",\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"eLD76yNF-sK7"},"source":["### Encoding"]},{"cell_type":"code","metadata":{"id":"oSIC-V4P6ZhB"},"source":["def dataprep(interactions):\n"," \"\"\"\n"," Prepare interactions dataset for training model\n"," \"\"\"\n","\n"," data = interactions.copy()\n","\n"," user_code_id = dict(enumerate(data[\"user\"].unique()))\n"," user_id_code = {v: k for k, v in user_code_id.items()}\n"," data[\"user_code\"] = data[\"user\"].apply(user_id_code.get)\n","\n"," item_code_id = dict(enumerate(data[\"item\"].unique()))\n"," item_id_code = {v: k for k, v in item_code_id.items()}\n"," data[\"item_code\"] = data[\"item\"].apply(item_id_code.get)\n","\n"," train_ui = sparse.csr_matrix(\n"," (np.ones(len(data)), (data[\"user_code\"], data[\"item_code\"]))\n"," )\n","\n"," return train_ui, {'user_code_id':user_code_id,\n"," 'user_id_code':user_id_code,\n"," 'item_code_id':item_code_id,\n"," 'item_id_code':item_id_code}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"e3O2SVR52h6w"},"source":["## Models"]},{"cell_type":"markdown","metadata":{"id":"1IDWdpsN5Dgq"},"source":["### Base Class"]},{"cell_type":"code","metadata":{"id":"AvsGAdei5JdN"},"source":["class BaseRecommender:\n"," \"\"\"Base recommender interface\"\"\"\n","\n"," def preprocess(self):\n"," \"\"\"Implement any needed input data preprocessing\"\"\"\n"," raise NotImplementedError\n","\n"," def fit(self):\n"," \"\"\"Implement model fitter\"\"\"\n"," raise NotImplementedError\n","\n"," def recommend(self, *args, **kwargs):\n"," \"\"\"Implement recommend method\n"," Should return a DataFrame containing\n"," * user_id: id of the user for whom we provide recommendations\n"," * n columns containing item recommendations (or None if missing)\n"," \"\"\"\n"," raise NotImplementedError"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LAg8STYJ4Isz"},"source":["### TopPop"]},{"cell_type":"code","metadata":{"id":"qJzY1MtP4K1E"},"source":["class TopPop(BaseRecommender):\n"," \"\"\"\n"," TopPop recommender, which recommends the most popular items\n"," \"\"\"\n","\n"," def __init__(self, train_ui, encode_maps, show_progress=True):\n"," super().__init__()\n","\n"," self.popular_items = None\n","\n"," self.train_ui = train_ui\n"," self.user_id_code = encode_maps['user_id_code']\n"," self.user_code_id = encode_maps['user_code_id']\n"," self.item_code_id = encode_maps['item_code_id']\n","\n"," self.show_progress = show_progress\n","\n"," def fit(self):\n"," \"\"\"\n"," Fit the model\n"," \"\"\"\n"," self.popular_items = (-self.train_ui.sum(axis=0).A.ravel()).argsort()\n","\n"," def recommend(\n"," self,\n"," target_users,\n"," n_recommendations,\n"," filter_out_interacted_items=True,\n"," ) -> pd.DataFrame:\n"," \"\"\"\n"," Recommends n_recommendations items for target_users\n"," :return:\n"," pd.DataFrame (user, item_1, item_2, ..., item_n)\n"," \"\"\"\n","\n"," with ThreadPool() as thread_pool:\n"," recommendations = list(\n"," tqdm(\n"," thread_pool.imap(\n"," partial(\n"," self.recommend_per_user,\n"," n_recommendations=n_recommendations,\n"," filter_out_interacted_items=filter_out_interacted_items,\n"," ),\n"," target_users,\n"," ),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," return pd.DataFrame(recommendations)\n","\n"," def recommend_per_user(\n"," self, user, n_recommendations, filter_out_interacted_items=True\n"," ):\n"," \"\"\"\n"," Recommends n items per user\n"," :param user: User id\n"," :param n_recommendations: Number of recommendations\n"," :param filter_out_interacted_items: boolean value to filter interacted items\n"," :return: list of format [user_id, item1, item2 ...]\n"," \"\"\"\n"," u_code = self.user_id_code.get(user)\n"," u_recommended_items = []\n","\n"," if u_code is not None:\n"," exclude_items = []\n"," if filter_out_interacted_items:\n"," exclude_items = self.train_ui.indices[\n"," self.train_ui.indptr[u_code] : self.train_ui.indptr[u_code + 1]\n"," ]\n","\n"," u_recommended_items = self.popular_items[\n"," : n_recommendations + len(exclude_items)\n"," ]\n","\n"," u_recommended_items = [\n"," self.item_code_id[i]\n"," for i in u_recommended_items\n"," if i not in exclude_items\n"," ]\n","\n"," u_recommended_items = u_recommended_items[:n_recommendations]\n","\n"," return (\n"," [user]\n"," + u_recommended_items\n"," + [None] * (n_recommendations - len(u_recommended_items))\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"SPpW-C7sXI6i"},"source":["### Random"]},{"cell_type":"code","metadata":{"id":"Rf4Z9HpXXKK7"},"source":["class Random(BaseRecommender):\n"," \"\"\"\n"," TopPop recommender, which recommends the most popular items\n"," \"\"\"\n","\n"," def __init__(self, train_ui, encode_maps, show_progress=True):\n"," super().__init__()\n","\n"," self.train_ui = train_ui\n"," self.user_id_code = encode_maps['user_id_code']\n"," self.user_code_id = encode_maps['user_code_id']\n"," self.item_code_id = encode_maps['item_code_id']\n","\n"," self.show_progress = show_progress\n","\n"," def fit(self):\n"," \"\"\"\n"," Fit the model\n"," \"\"\"\n"," pass\n","\n"," def recommend(\n"," self,\n"," target_users,\n"," n_recommendations,\n"," filter_out_interacted_items=True,\n"," ) -> pd.DataFrame:\n"," \"\"\"\n"," Recommends n_recommendations items for target_users\n"," :return:\n"," pd.DataFrame (user, item_1, item_2, ..., item_n)\n"," \"\"\"\n","\n"," with ThreadPool() as thread_pool:\n"," recommendations = list(\n"," tqdm(\n"," thread_pool.imap(\n"," partial(\n"," self.recommend_per_user,\n"," n_recommendations=n_recommendations,\n"," filter_out_interacted_items=filter_out_interacted_items,\n"," ),\n"," target_users,\n"," ),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," return pd.DataFrame(recommendations)\n","\n"," def recommend_per_user(\n"," self, user, n_recommendations, filter_out_interacted_items=True\n"," ):\n"," \"\"\"\n"," Recommends n items per user\n"," :param user: User id\n"," :param n_recommendations: Number of recommendations\n"," :param filter_out_interacted_items: boolean value to filter interacted items\n"," :return: list of format [user_id, item1, item2 ...]\n"," \"\"\"\n"," u_code = self.user_id_code.get(user)\n"," u_recommended_items = []\n","\n"," if u_code is not None:\n"," exclude_items = []\n"," if filter_out_interacted_items:\n"," exclude_items = self.train_ui.indices[\n"," self.train_ui.indptr[u_code] : self.train_ui.indptr[u_code + 1]\n"," ]\n","\n"," u_recommended_items = random.sample(\n"," range(self.train_ui.shape[1]), n_recommendations + len(exclude_items)\n"," )\n","\n"," u_recommended_items = [\n"," self.item_code_id[i]\n"," for i in u_recommended_items\n"," if i not in exclude_items\n"," ]\n","\n"," u_recommended_items = u_recommended_items[:n_recommendations]\n","\n"," return (\n"," [user]\n"," + u_recommended_items\n"," + [None] * (n_recommendations - len(u_recommended_items))\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"avfP-s7-aFkq"},"source":["### ALS"]},{"cell_type":"code","metadata":{"id":"n1xuciwUaP0Q"},"source":["class ALS(BaseRecommender):\n"," \"\"\"\n"," Module implementing a wrapper for the ALS model\n"," Wrapper over ALS model\n"," \"\"\"\n","\n"," def __init__(self, train_ui,\n"," encode_maps,\n"," factors=100,\n"," regularization=0.01,\n"," use_gpu=False,\n"," iterations=15,\n"," event_weights_multiplier=100,\n"," show_progress=True,\n"," ):\n"," \"\"\"\n"," Source of descriptions:\n"," https://github.com/benfred/implicit/blob/master/implicit/als.py\n"," Alternating Least Squares\n"," A Recommendation Model based on the algorithms described in the paper\n"," 'Collaborative Filtering for Implicit Feedback Datasets'\n"," with performance optimizations described in 'Applications of the\n"," Conjugate Gradient Method for Implicit Feedback Collaborative Filtering.'\n"," Parameters\n"," ----------\n"," factors : int, optional\n"," The number of latent factors to compute\n"," regularization : float, optional\n"," The regularization factor to use\n"," use_gpu : bool, optional\n"," Fit on the GPU if available, default is to run on CPU\n"," iterations : int, optional\n"," The number of ALS iterations to use when fitting data\n"," event_weights_multiplier: int, optional\n"," The multiplier of weights.\n"," Used to find a tradeoff between the importance of interacted and not interacted items.\n"," \"\"\"\n","\n"," super().__init__()\n","\n"," self.train_ui = train_ui\n"," self.user_id_code = encode_maps['user_id_code']\n"," self.user_code_id = encode_maps['user_code_id']\n"," self.item_code_id = encode_maps['item_code_id']\n"," self.mapping_user_test_items = None\n"," self.similarity_matrix = None\n","\n"," self.show_progress = show_progress\n","\n"," self.model = implicit.als.AlternatingLeastSquares(\n"," factors=factors,\n"," regularization=regularization,\n"," use_gpu=use_gpu,\n"," iterations=iterations,\n"," )\n","\n"," self.event_weights_multiplier = event_weights_multiplier\n","\n"," def fit(self):\n"," \"\"\"\n"," Fit the model\n"," \"\"\"\n"," self.model.fit(self.train_ui.T, show_progress=self.show_progress)\n","\n"," def recommend(\n"," self,\n"," target_users,\n"," n_recommendations,\n"," filter_out_interacted_items=True,\n"," ) -> pd.DataFrame:\n"," \"\"\"\n"," Recommends n_recommendations items for target_users\n"," :return:\n"," pd.DataFrame (user, item_1, item_2, ..., item_n)\n"," \"\"\"\n","\n"," with ThreadPool() as thread_pool:\n"," recommendations = list(\n"," tqdm(\n"," thread_pool.imap(\n"," partial(\n"," self.recommend_per_user,\n"," n_recommendations=n_recommendations,\n"," filter_out_interacted_items=filter_out_interacted_items,\n"," ),\n"," target_users,\n"," ),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," return pd.DataFrame(recommendations)\n","\n"," def recommend_per_user(\n"," self, user, n_recommendations, filter_out_interacted_items=True\n"," ):\n"," \"\"\"\n"," Recommends n items per user\n"," :param user: User id\n"," :param n_recommendations: Number of recommendations\n"," :param filter_out_interacted_items: boolean value to filter interacted items\n"," :return: list of format [user_id, item1, item2 ...]\n"," \"\"\"\n"," u_code = self.user_id_code.get(user)\n"," u_recommended_items = []\n"," if u_code is not None:\n","\n"," u_recommended_items = list(\n"," zip(\n"," *self.model.recommend(\n"," u_code,\n"," self.train_ui,\n"," N=n_recommendations,\n"," filter_already_liked_items=filter_out_interacted_items,\n"," )\n"," )\n"," )[0]\n","\n"," u_recommended_items = [self.item_code_id[i] for i in u_recommended_items]\n","\n"," return (\n"," [user]\n"," + u_recommended_items\n"," + [None] * (n_recommendations - len(u_recommended_items))\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"YswMJNXBiCEX"},"source":["### LightFM"]},{"cell_type":"code","metadata":{"id":"irEAXPlLiCEd"},"source":["class LFM(BaseRecommender):\n"," \"\"\"\n"," Module implementing a wrapper for the ALS model\n"," Wrapper over LightFM model\n"," \"\"\"\n","\n"," def __init__(self, train_ui,\n"," encode_maps,\n"," no_components=30,\n"," k=5,\n"," n=10,\n"," learning_schedule=\"adagrad\",\n"," loss=\"logistic\",\n"," learning_rate=0.05,\n"," rho=0.95,\n"," epsilon=1e-06,\n"," item_alpha=0.0,\n"," user_alpha=0.0,\n"," max_sampled=10,\n"," random_state=42,\n"," epochs=20,\n"," show_progress=True,\n"," ):\n"," \"\"\"\n"," Source of descriptions:\n"," https://making.lyst.com/lightfm/docs/_modules/lightfm/lightfm.html#LightFM\n"," A hybrid latent representation recommender model.\n"," The model learns embeddings (latent representations in a high-dimensional\n"," space) for users and items in a way that encodes user preferences over items.\n"," When multiplied together, these representations produce scores for every item\n"," for a given user; items scored highly are more likely to be interesting to\n"," the user.\n"," The user and item representations are expressed in terms of representations\n"," of their features: an embedding is estimated for every feature, and these\n"," features are then summed together to arrive at representations for users and\n"," items. For example, if the movie 'Wizard of Oz' is described by the following\n"," features: 'musical fantasy', 'Judy Garland', and 'Wizard of Oz', then its\n"," embedding will be given by taking the features' embeddings and adding them\n"," together. The same applies to user features.\n"," The embeddings are learned through `stochastic gradient\n"," descent `_ methods.\n"," Four loss functions are available:\n"," - logistic: useful when both positive (1) and negative (-1) interactions\n"," are present.\n"," - BPR: Bayesian Personalised Ranking [1]_ pairwise loss. Maximises the\n"," prediction difference between a positive example and a randomly\n"," chosen negative example. Useful when only positive interactions\n"," are present and optimising ROC AUC is desired.\n"," - WARP: Weighted Approximate-Rank Pairwise [2]_ loss. Maximises\n"," the rank of positive examples by repeatedly sampling negative\n"," examples until rank violating one is found. Useful when only\n"," positive interactions are present and optimising the top of\n"," the recommendation list (precision@k) is desired.\n"," - k-OS WARP: k-th order statistic loss [3]_. A modification of WARP that\n"," uses the k-th positive example for any given user as a basis for pairwise\n"," updates.\n"," Two learning rate schedules are available:\n"," - adagrad: [4]_\n"," - adadelta: [5]_\n"," Parameters\n"," ----------\n"," no_components: int, optional\n"," the dimensionality of the feature latent embeddings.\n"," k: int, optional\n"," for k-OS training, the k-th positive example will be selected from the\n"," n positive examples sampled for every user.\n"," n: int, optional\n"," for k-OS training, maximum number of positives sampled for each update.\n"," learning_schedule: string, optional\n"," one of ('adagrad', 'adadelta').\n"," loss: string, optional\n"," one of ('logistic', 'bpr', 'warp', 'warp-kos'): the loss function.\n"," learning_rate: float, optional\n"," initial learning rate for the adagrad learning schedule.\n"," rho: float, optional\n"," moving average coefficient for the adadelta learning schedule.\n"," epsilon: float, optional\n"," conditioning parameter for the adadelta learning schedule.\n"," item_alpha: float, optional\n"," L2 penalty on item features. Tip: setting this number too high can slow\n"," down training. One good way to check is if the final weights in the\n"," embeddings turned out to be mostly zero. The same idea applies to\n"," the user_alpha parameter.\n"," user_alpha: float, optional\n"," L2 penalty on user features.\n"," max_sampled: int, optional\n"," maximum number of negative samples used during WARP fitting.\n"," It requires a lot of sampling to find negative triplets for users that\n"," are already well represented by the model; this can lead to very long\n"," training times and overfitting. Setting this to a higher number will\n"," generally lead to longer training times, but may in some cases improve\n"," accuracy.\n"," random_state: int seed, RandomState instance, or None\n"," The seed of the pseudo random number generator to use when shuffling\n"," the data and initializing the parameters.\n"," epochs: (int, optional) number of epochs to run\n"," \"\"\"\n","\n"," super().__init__()\n","\n"," self.model = LightFM(\n"," no_components=no_components,\n"," k=k,\n"," n=n,\n"," learning_schedule=learning_schedule,\n"," loss=loss,\n"," learning_rate=learning_rate,\n"," rho=rho,\n"," epsilon=epsilon,\n"," item_alpha=item_alpha,\n"," user_alpha=user_alpha,\n"," max_sampled=max_sampled,\n"," random_state=random_state,\n"," )\n"," self.epochs = epochs\n","\n"," self.train_ui = train_ui\n"," self.user_id_code = encode_maps['user_id_code']\n"," self.user_code_id = encode_maps['user_code_id']\n"," self.item_code_id = encode_maps['item_code_id']\n"," self.mapping_user_test_items = None\n"," self.similarity_matrix = None\n","\n"," self.show_progress = show_progress\n","\n"," def fit(self):\n"," \"\"\"\n"," Fit the model\n"," \"\"\"\n"," self.model.fit(\n"," self.train_ui,\n"," epochs=self.epochs,\n"," num_threads=multiprocessing.cpu_count(),\n"," verbose=self.show_progress,\n"," )\n","\n"," def recommend(\n"," self,\n"," target_users,\n"," n_recommendations,\n"," filter_out_interacted_items=True,\n"," ) -> pd.DataFrame:\n"," \"\"\"\n"," Recommends n_recommendations items for target_users\n"," :return:\n"," pd.DataFrame (user, item_1, item_2, ..., item_n)\n"," \"\"\"\n","\n"," self.items_to_recommend = np.arange(len(self.item_code_id))\n","\n"," with ThreadPool() as thread_pool:\n"," recommendations = list(\n"," tqdm(\n"," thread_pool.imap(\n"," partial(\n"," self.recommend_per_user,\n"," n_recommendations=n_recommendations,\n"," filter_out_interacted_items=filter_out_interacted_items,\n"," ),\n"," target_users,\n"," ),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," return pd.DataFrame(recommendations)\n","\n"," def recommend_per_user(\n"," self, user, n_recommendations, filter_out_interacted_items=True\n"," ):\n"," \"\"\"\n"," Recommends n items per user\n"," :param user: User id\n"," :param n_recommendations: Number of recommendations\n"," :param filter_out_interacted_items: boolean value to filter interacted items\n"," :return: list of format [user_id, item1, item2 ...]\n"," \"\"\"\n"," u_code = self.user_id_code.get(user)\n","\n"," if u_code is not None:\n"," interacted_items = self.train_ui.indices[\n"," self.train_ui.indptr[u_code] : self.train_ui.indptr[u_code + 1]\n"," ]\n","\n"," scores = self.model.predict(int(u_code), self.items_to_recommend)\n","\n"," item_recommendations = self.items_to_recommend[np.argsort(-scores)][\n"," : n_recommendations + len(interacted_items)\n"," ]\n"," item_recommendations = [\n"," self.item_code_id[item]\n"," for item in item_recommendations\n"," if item not in interacted_items\n"," ]\n","\n"," return (\n"," [user]\n"," + item_recommendations\n"," + [None] * (n_recommendations - len(item_recommendations))\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"luifpLf2l1WM"},"source":["### RP3Beta"]},{"cell_type":"code","metadata":{"id":"EnGCs-iQl2-2"},"source":["class RP3Beta(BaseRecommender):\n"," \"\"\"\n"," Module implementing a RP3Beta model\n"," RP3Beta model proposed in the paper \"Updatable, Accurate, Diverse, and Scalable Recommendations for Interactive\n"," Applications\". In our implementation we perform direct computations on sparse matrices instead of random walks\n"," approximation.\n"," \"\"\"\n","\n"," def __init__(self, train_ui,\n"," encode_maps,\n"," alpha=1,\n"," beta=0,\n"," show_progress=True):\n"," \n"," super().__init__()\n","\n"," self.train_ui = train_ui\n"," self.user_id_code = encode_maps['user_id_code']\n"," self.user_code_id = encode_maps['user_code_id']\n"," self.item_code_id = encode_maps['item_code_id']\n","\n"," self.alpha = alpha\n"," self.beta = beta\n"," self.p_ui = None\n"," self.similarity_matrix = None\n","\n"," self.show_progress = show_progress\n","\n"," def fit(self):\n"," \"\"\"\n"," Fit the model\n"," \"\"\"\n"," # Define Pui\n"," self.p_ui = normalize(self.train_ui, norm=\"l1\", axis=1).power(self.alpha)\n","\n"," # Define Piu\n"," p_iu = normalize(\n"," self.train_ui.transpose(copy=True).tocsr(), norm=\"l1\", axis=1\n"," ).power(self.alpha)\n","\n"," self.similarity_matrix = p_iu * self.p_ui\n"," item_orders = (self.train_ui > 0).sum(axis=0).A.ravel()\n","\n"," self.similarity_matrix *= sparse.diags(1 / item_orders.clip(min=1) ** self.beta)\n","\n"," def recommend(\n"," self,\n"," target_users,\n"," n_recommendations,\n"," filter_out_interacted_items=True,\n"," ) -> pd.DataFrame:\n"," \"\"\"\n"," Recommends n_recommendations items for target_users\n"," :return:\n"," pd.DataFrame (user, item_1, item_2, ..., item_n)\n"," \"\"\"\n","\n"," with ThreadPool() as thread_pool:\n"," recommendations = list(\n"," tqdm(\n"," thread_pool.imap(\n"," partial(\n"," self.recommend_per_user,\n"," n_recommendations=n_recommendations,\n"," filter_out_interacted_items=filter_out_interacted_items,\n"," ),\n"," target_users,\n"," ),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," return pd.DataFrame(recommendations)\n","\n"," def recommend_per_user(\n"," self, user, n_recommendations, filter_out_interacted_items=True\n"," ):\n"," \"\"\"\n"," Recommends n items per user\n"," :param user: User id\n"," :param n_recommendations: Number of recommendations\n"," :param filter_out_interacted_items: boolean value to filter interacted items\n"," :return: list of format [user_id, item1, item2 ...]\n"," \"\"\"\n"," u_code = self.user_id_code.get(user)\n"," u_recommended_items = []\n"," if u_code is not None:\n","\n"," exclude_items = []\n"," if filter_out_interacted_items:\n"," exclude_items = self.train_ui.indices[\n"," self.train_ui.indptr[u_code] : self.train_ui.indptr[u_code + 1]\n"," ]\n","\n"," scores = self.p_ui[u_code] * self.similarity_matrix\n"," u_recommended_items = scores.indices[\n"," (-scores.data).argsort()[: n_recommendations + len(exclude_items)]\n"," ]\n","\n"," u_recommended_items = [\n"," self.item_code_id[i]\n"," for i in u_recommended_items\n"," if i not in exclude_items\n"," ]\n","\n"," u_recommended_items = u_recommended_items[:n_recommendations]\n","\n"," return (\n"," [user]\n"," + u_recommended_items\n"," + [None] * (n_recommendations - len(u_recommended_items))\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Vs3_fXsenw_r"},"source":["### SLIM"]},{"cell_type":"code","metadata":{"id":"URqedoMcnyUB"},"source":["class SLIM(BaseRecommender):\n"," \"\"\"\n"," Module implementing SLIM model\n"," SLIM model proposed in \"SLIM: Sparse Linear Methods for Top-N Recommender Systems\n"," \"\"\"\n","\n"," def __init__(self, train_ui,\n"," encode_maps,\n"," alpha=0.0001,\n"," l1_ratio=0.5,\n"," iterations=3,\n"," show_progress=True):\n"," \n"," super().__init__()\n","\n"," self.train_ui = train_ui\n"," self.user_id_code = encode_maps['user_id_code']\n"," self.user_code_id = encode_maps['user_code_id']\n"," self.item_code_id = encode_maps['item_code_id']\n","\n"," self.alpha = alpha\n"," self.l1_ratio = l1_ratio\n"," self.iterations = iterations\n"," self.similarity_matrix = None\n","\n"," self.show_progress = show_progress\n","\n"," def fit_per_item(self, column_id):\n"," \"\"\"\n"," Fits ElasticNet per item\n"," :param column_id: Id of column to setup as predicted value\n"," :return: coefficients of the ElasticNet model\n"," \"\"\"\n"," model = ElasticNet(\n"," alpha=self.alpha,\n"," l1_ratio=self.l1_ratio,\n"," positive=True,\n"," fit_intercept=False,\n"," copy_X=False,\n"," precompute=True,\n"," selection=\"random\",\n"," max_iter=self.iterations,\n"," )\n"," # set to zeros all entries in the given column of train_ui\n"," y = self.train_ui[:, column_id].A\n"," start_indptr = self.train_ui.indptr[column_id]\n"," end_indptr = self.train_ui.indptr[column_id + 1]\n"," column_ratings = self.train_ui.data[start_indptr:end_indptr].copy()\n"," self.train_ui.data[start_indptr:end_indptr] = 0\n","\n"," # learn item-item similarities\n"," model.fit(self.train_ui, y)\n","\n"," # return original ratings to train_ui\n"," self.train_ui.data[start_indptr:end_indptr] = column_ratings\n","\n"," return model.sparse_coef_.T\n","\n"," @ignore_warnings(category=ConvergenceWarning)\n"," def fit(self):\n"," \"\"\"\n"," Fit the model\n"," \"\"\"\n"," self.train_ui = self.train_ui.tocsc()\n","\n"," with ThreadPool() as thread_pool:\n"," coefs = list(\n"," tqdm(\n"," thread_pool.imap(self.fit_per_item, range(self.train_ui.shape[1])),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," self.similarity_matrix = sparse.hstack(coefs).tocsr()\n","\n"," self.train_ui = self.train_ui.tocsr()\n","\n"," def recommend(\n"," self,\n"," target_users,\n"," n_recommendations,\n"," filter_out_interacted_items=True,\n"," ):\n"," \"\"\"\n"," Recommends n_recommendations items for target_users\n"," :return:\n"," pd.DataFrame (user, item_1, item_2, ..., item_n)\n"," \"\"\"\n","\n"," with ThreadPool() as thread_pool:\n"," recommendations = list(\n"," tqdm(\n"," thread_pool.imap(\n"," partial(\n"," self.recommend_per_user,\n"," n_recommendations=n_recommendations,\n"," filter_out_interacted_items=filter_out_interacted_items,\n"," ),\n"," target_users,\n"," ),\n"," disable=not self.show_progress,\n"," )\n"," )\n","\n"," return pd.DataFrame(recommendations)\n","\n"," def recommend_per_user(\n"," self, user, n_recommendations, filter_out_interacted_items=True\n"," ):\n"," \"\"\"\n"," Recommends n items per user\n"," :param user: User id\n"," :param n_recommendations: Number of recommendations\n"," :param filter_out_interacted_items: boolean value to filter interacted items\n"," :return: list of format [user_id, item1, item2 ...]\n"," \"\"\"\n"," u_code = self.user_id_code.get(user)\n"," if u_code is not None:\n","\n"," exclude_items = []\n"," if filter_out_interacted_items:\n"," exclude_items = self.train_ui.indices[\n"," self.train_ui.indptr[u_code] : self.train_ui.indptr[u_code + 1]\n"," ]\n","\n"," scores = self.train_ui[u_code] * self.similarity_matrix\n"," u_recommended_items = scores.indices[\n"," (-scores.data).argsort()[: n_recommendations + len(exclude_items)]\n"," ]\n","\n"," u_recommended_items = [\n"," self.item_code_id[i]\n"," for i in u_recommended_items\n"," if i not in exclude_items\n"," ][:n_recommendations]\n"," return (\n"," [user]\n"," + u_recommended_items\n"," + [None] * (n_recommendations - len(u_recommended_items))\n"," )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"tQqC5tY7Cdi7"},"source":["## Runs"]},{"cell_type":"code","metadata":{"id":"0nN6nDrXDfzh"},"source":["# load the training data\n","interactions_train = load_interactions('train.gzip')\n","\n","# encode user ids and convert interactions into sparse interaction matrix\n","train_ui, encode_maps = dataprep(interactions_train)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RZ4lDR2bYSHp"},"source":["# # load target users\n","target_users = load_target_users('target_users_subset_validation.gzip')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bEH-nxlMYY-n"},"source":["# models list\n","models = {'itempop': TopPop,\n"," 'random': Random,\n"," 'als': ALS,\n"," 'lightfm': LFM,\n"," 'rp3': RP3Beta,\n"," 'slim': SLIM,\n"," }"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Oq68sqtBhj2d"},"source":["### Training Random Model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-ufV9e1jZurF","executionInfo":{"status":"ok","timestamp":1639132355777,"user_tz":-330,"elapsed":715,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"1d72de53-d235-4833-986d-b5865a4076b5"},"source":["model_name = 'random'\n","\n","Model = models[model_name]\n","\n","# number of recommendations\n","N_RECOMMENDATIONS = 10\n","\n","# initiate the model\n","model = Model(train_ui, encode_maps)\n","\n","# train the model\n","model.fit()\n","\n","# # recommend\n","recommendations = model.recommend(target_users=target_users, n_recommendations=N_RECOMMENDATIONS)\n","\n","# # save the recommendations\n","save_recommendations(recommendations, '{}.gzip'.format(model_name))"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["5373it [00:00, 17610.42it/s]\n"]}]},{"cell_type":"markdown","metadata":{"id":"E4UfwTYthq_N"},"source":["### Training Item Pop Model"]},{"cell_type":"code","metadata":{"id":"dt2CcKauhtJJ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1639132360073,"user_tz":-330,"elapsed":1434,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"d9bf0f0c-2bb5-4bd4-82b3-5d3ff6d8dd0b"},"source":["model_name = 'itempop'\n","\n","Model = models[model_name]\n","\n","# number of recommendations\n","N_RECOMMENDATIONS = 10\n","\n","# initiate the model\n","model = Model(train_ui, encode_maps)\n","\n","# train the model\n","model.fit()\n","\n","# # recommend\n","recommendations = model.recommend(target_users=target_users, n_recommendations=N_RECOMMENDATIONS)\n","\n","# # save the recommendations\n","save_recommendations(recommendations, '{}.gzip'.format(model_name))"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["5373it [00:00, 15331.51it/s]\n"]}]},{"cell_type":"markdown","metadata":{"id":"3zqtj1thhvJi"},"source":["### Training ALS Model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":104,"referenced_widgets":["9968de8625a44cd5845470867cd1e547","cdfb05654b1b4039aba929e38b5e11ab","a7dcc1acdc8c45edaf07863a3bfb1030","7d7e8822c9ce44879d76a786bcd482ab","46fafbbc6545402f891b745e1995a671","6f51152ee8a24d8d82d8b580d049f581","ea5f5db35ee24faabf20a62cabc099a6","b2ec7053720e49319db46cc44bd5a023","32c31fca0efb4eea86fd74daeae46c68","e62cf418ba9b4278a008d24c69033fb0","ffb1a52956204822a6d5481632c587f6"]},"id":"QRie0ql2gCXp","executionInfo":{"status":"ok","timestamp":1639132422453,"user_tz":-330,"elapsed":60022,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"d2cfad16-88d2-4d4b-fcb7-a26add15f375"},"source":["model_name = 'als'\n","\n","Model = models[model_name]\n","\n","FACTORS = 400\n","REGULARIZATION = 0.1\n","ITERATIONS = 6\n","EVENT_WEIGHTS_MULTIPLIER = 100\n","\n","N_RECOMMENDATIONS = 10\n","\n","# initiate the model\n","model = Model(train_ui,\n"," encode_maps,\n"," factors=FACTORS,\n"," regularization=REGULARIZATION,\n"," iterations=ITERATIONS,\n"," event_weights_multiplier=EVENT_WEIGHTS_MULTIPLIER,\n"," )\n","\n","# train the model\n","model.fit()\n","\n","# # recommend\n","recommendations = model.recommend(target_users=target_users, n_recommendations=N_RECOMMENDATIONS)\n","\n","# # save the recommendations\n","save_recommendations(recommendations, '{}.gzip'.format(model_name))"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["WARNING:root:OpenBLAS detected. Its highly recommend to set the environment variable 'export OPENBLAS_NUM_THREADS=1' to disable its internal multithreading\n"]},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9968de8625a44cd5845470867cd1e547","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/6 [00:00\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
precisionrecallF_1F_05ndcgmAPMRRLAUCHRreco_in_testtest_coverageShannonGiniusers_without_recousers_without_k_reco
model_name
rp30.0230040.1873880.0402820.0277460.1248050.1000250.1143620.5903830.2149640.8745770.8964766.4442580.6401940.0029780.030151
lightfm0.0180530.1484990.0316590.0217840.1021870.0835300.0954430.5708790.1697380.8973940.7863446.5587080.6032500.0000000.000000
slim0.0153360.1254980.0268810.0185030.0942530.0800690.0918600.5594000.1457290.4674300.4676955.8743120.7987830.1708540.697190
als0.0111480.0910390.0195230.0134460.0617630.0497160.0577860.5420320.1072030.8412800.7400886.3277650.6902160.0000000.000000
itempop0.0042620.0331630.0074000.0051280.0171450.0112780.0140880.5129250.0418761.0000000.0102792.3399070.9933130.0000000.000000
random0.0005770.0045100.0009940.0006920.0018610.0009660.0012090.4985800.0057700.5653640.9860507.1809560.1287530.0000000.000000
\n",""],"text/plain":[" precision recall ... users_without_reco users_without_k_reco\n","model_name ... \n","rp3 0.023004 0.187388 ... 0.002978 0.030151\n","lightfm 0.018053 0.148499 ... 0.000000 0.000000\n","slim 0.015336 0.125498 ... 0.170854 0.697190\n","als 0.011148 0.091039 ... 0.000000 0.000000\n","itempop 0.004262 0.033163 ... 0.000000 0.000000\n","random 0.000577 0.004510 ... 0.000000 0.000000\n","\n","[6 rows x 15 columns]"]},"metadata":{},"execution_count":51}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":393},"id":"cIMnw6wxE8BP","executionInfo":{"status":"ok","timestamp":1639132487449,"user_tz":-330,"elapsed":3918,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"e5b78bc9-ea2b-420f-b97f-f45c12de8535"},"source":["evaluator = Evaluator(\n"," recommendations_path='/content',\n"," test_path='validation.gzip',\n"," k=10,\n"," models_to_evaluate=list(models.keys()),\n",")\n","\n","evaluator.prepare()\n","\n","evaluator.evaluate_models()\n","\n","evaluator.evaluation_results"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["5373it [00:00, 12328.94it/s]\n","5373it [00:00, 11966.13it/s]\n","5373it [00:00, 11717.03it/s]\n","5373it [00:00, 11618.24it/s]\n","5373it [00:00, 11584.71it/s]\n","5373it [00:00, 11754.46it/s]\n"]},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
precisionrecallF_1F_05ndcgmAPMRRLAUCHRreco_in_testtest_coverageShannonGiniusers_without_recousers_without_k_reco
model_name
rp30.0230040.1873880.0402820.0277460.1248050.1000250.1143620.5903830.2149640.8745770.8964766.4442580.6401940.0029780.030151
lightfm0.0180530.1484990.0316590.0217840.1021870.0835300.0954430.5708790.1697380.8973940.7863446.5587080.6032500.0000000.000000
slim0.0153360.1254980.0268810.0185030.0942530.0800690.0918600.5594000.1457290.4674300.4676955.8743120.7987830.1708540.697190
als0.0111480.0910390.0195230.0134460.0617630.0497160.0577860.5420320.1072030.8412800.7400886.3277650.6902160.0000000.000000
itempop0.0042620.0331630.0074000.0051280.0171450.0112780.0140880.5129250.0418761.0000000.0102792.3399070.9933130.0000000.000000
random0.0005770.0045100.0009940.0006920.0018610.0009660.0012090.4985800.0057700.5653640.9860507.1809560.1287530.0000000.000000
\n","
"],"text/plain":[" precision recall ... users_without_reco users_without_k_reco\n","model_name ... \n","rp3 0.023004 0.187388 ... 0.002978 0.030151\n","lightfm 0.018053 0.148499 ... 0.000000 0.000000\n","slim 0.015336 0.125498 ... 0.170854 0.697190\n","als 0.011148 0.091039 ... 0.000000 0.000000\n","itempop 0.004262 0.033163 ... 0.000000 0.000000\n","random 0.000577 0.004510 ... 0.000000 0.000000\n","\n","[6 rows x 15 columns]"]},"metadata":{},"execution_count":52}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"X4SI7nEl50fW"}},{"cell_type":"code","source":["!apt-get -qq install tree\n","!rm -r sample_data"],"metadata":{"id":"uD0hoTOC8EsL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!tree --du -h -C ."],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tz0gEDLu8MTs","executionInfo":{"status":"ok","timestamp":1639133057885,"user_tz":-330,"elapsed":469,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"65e3d5f1-51ac-46e4-ea93-e96b7f2eec6d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[01;34m.\u001b[00m\n","├── [122K] als.gzip\n","├── [ 69M] df_subset_items.parquet.snappy\n","├── [ 69M] df_subset_users.parquet.snappy\n","├── [2.0G] interactions.csv\n","├── [ 26K] itempop.gzip\n","├── [137K] lightfm.gzip\n","├── [509M] \u001b[01;31molx-jobs-interactions.zip\u001b[00m\n","├── [151K] random.gzip\n","├── [4.5K] README.txt\n","├── [121K] rp3.gzip\n","├── [ 61K] slim.gzip\n","├── [524K] target_users_all.gzip\n","├── [ 20K] target_users_subset_validation.gzip\n","├── [4.0M] test.gzip\n","├── [837K] train.gzip\n","├── [ 30M] train_valid.gzip\n","└── [ 61K] validation.gzip\n","\n"," 2.6G used in 0 directories, 17 files\n"]}]},{"cell_type":"code","source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p lightfm"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KNU-EKQa50fY","executionInfo":{"status":"ok","timestamp":1639132647495,"user_tz":-330,"elapsed":3718,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"2318c902-be64-4240-86ee-038ef3a9c928"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2021-12-10 10:37:34\n","\n","lightfm: 1.16\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.104+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","scipy : 1.4.1\n","implicit : 0.4.8\n","numpy : 1.19.5\n","IPython : 5.5.0\n","sklearn : 0.0\n","matplotlib: 3.2.2\n","sys : 3.7.12 (default, Sep 10 2021, 00:21:48) \n","[GCC 7.5.0]\n","pandas : 1.1.5\n","\n"]}]},{"cell_type":"code","source":["import sklearn\n","sklearn.__version__"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"Fcou97696w_S","executionInfo":{"status":"ok","timestamp":1639132663962,"user_tz":-330,"elapsed":628,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"6c9fc887-e152-4653-9ec2-d1eb816d4e0a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'1.0.1'"]},"metadata":{},"execution_count":59}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"4djbOBU750fZ"}},{"cell_type":"markdown","source":["**END**"],"metadata":{"id":"Idvv30U950fZ"}}]} \ No newline at end of file diff --git a/_notebooks/2022-01-12-seq-mab-mushroom.ipynb b/_notebooks/2022-01-12-seq-mab-mushroom.ipynb new file mode 100644 index 0000000..6612f38 --- /dev/null +++ b/_notebooks/2022-01-12-seq-mab-mushroom.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-12-seq-mab-mushroom.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P296669%20%7C%20Sequential%20Batch%20Learning%20in%20Stochastic%20MAB%20and%20Contextual%20MAB%20on%20Mushroom%20and%20Synthetic%20data.ipynb","timestamp":1644607243833}],"collapsed_sections":["aJdhotjJuyCg","F_7QyQIsvraK","6SzDe38oviFg","Y635PDE6vgdI","UiTTsyhFvdwc","-IL38-vIzIG1","4L_OJQQEvP4k","oAuUEthsu9TG"],"authorship_tag":"ABX9TyNovRrJ5FnlW7E7l5a6PFNk"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"2b72cbdd08374942859625d2047795f2":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_396e4164d325484e9956f597c2e133e0","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_46ce32b9cbc74045b96fc2fbc93f0575","IPY_MODEL_596798eaeac14aaea8c3cc476b7f329f","IPY_MODEL_c60ea6ae22a2457786ca6b8e3446e3f2"]}},"396e4164d325484e9956f597c2e133e0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"46ce32b9cbc74045b96fc2fbc93f0575":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_342e5fcfeeb04720834331a4274b5e28","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_9cbda8d73a0a41f8acb9c3bdacdc7a9b"}},"596798eaeac14aaea8c3cc476b7f329f":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_042addbfa66d4dd4a8543124cc312905","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":23,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":23,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3e9ddf2ca50f4c24bcf276eeeee63d01"}},"c60ea6ae22a2457786ca6b8e3446e3f2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ce6fa32eee4044df843b45c5693e065e","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 23/23 [00:09<00:00, 3.28it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_5ac6a207b502473fa8bd11d2adcc1201"}},"342e5fcfeeb04720834331a4274b5e28":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"9cbda8d73a0a41f8acb9c3bdacdc7a9b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"042addbfa66d4dd4a8543124cc312905":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"3e9ddf2ca50f4c24bcf276eeeee63d01":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ce6fa32eee4044df843b45c5693e065e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"5ac6a207b502473fa8bd11d2adcc1201":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3d6f8baea53a484faab78b4c50fbf620":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9689c9d0d5354f7782671e9b301c5840","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_4550c6895fa244efb6dd5cbc9dd98324","IPY_MODEL_b70c31ffb8784655ad0d4e59e12343d6","IPY_MODEL_c14978517f8047718a1c084947d702b0"]}},"9689c9d0d5354f7782671e9b301c5840":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4550c6895fa244efb6dd5cbc9dd98324":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_60d54986769b4e859de45bbce659b7f1","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 6%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0181512ceea0401db61675636ae223b5"}},"b70c31ffb8784655ad0d4e59e12343d6":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4b34f4a343b242abaef58ebe57e07a4d","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":16,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":16,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_476d47eec56d4ff0bec2900ed3505ba2"}},"c14978517f8047718a1c084947d702b0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_27609bb3e0594b5084e01fae4b04cc9c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1/16 [00:08<02:13, 8.89s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_606916b9b63b43969a01e84ec142f341"}},"60d54986769b4e859de45bbce659b7f1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"0181512ceea0401db61675636ae223b5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4b34f4a343b242abaef58ebe57e07a4d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"476d47eec56d4ff0bec2900ed3505ba2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"27609bb3e0594b5084e01fae4b04cc9c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"606916b9b63b43969a01e84ec142f341":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b8cc488f50764d7a80c565e120acf3bd":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_73139f73ef7d4769807b6dc081a48fb1","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_8c0a7a56a8d946bf953390fd61675c4d","IPY_MODEL_63ead4d72b27495cac3bb6a385895123","IPY_MODEL_a1d8d9e611534997828fa51cdab2bd0a"]}},"73139f73ef7d4769807b6dc081a48fb1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"8c0a7a56a8d946bf953390fd61675c4d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_afe8ad0a833b4f399a1f6d8367786fd3","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 7%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c55e9f4461c24c9f88ba2098ff957634"}},"63ead4d72b27495cac3bb6a385895123":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4472bedf87ec41aeb3306d6364386540","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":15,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":15,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_edc2a60460ef48bdb78972f403c14175"}},"a1d8d9e611534997828fa51cdab2bd0a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d971324520f34ce5b3b6994ddd0d0102","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1/15 [00:08<02:03, 8.81s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6d4282c72b80490cbdb027d3592849ea"}},"afe8ad0a833b4f399a1f6d8367786fd3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"c55e9f4461c24c9f88ba2098ff957634":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4472bedf87ec41aeb3306d6364386540":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"edc2a60460ef48bdb78972f403c14175":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d971324520f34ce5b3b6994ddd0d0102":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"6d4282c72b80490cbdb027d3592849ea":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"77454d4570db40ad9d90ed0f9d80f019":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_efe993ec7ad34cfaac8ca10da8b06b25","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a845d33bdbc048af807e19eb89699a1b","IPY_MODEL_e16dcd3650f64c61b1c53445f96d13d5","IPY_MODEL_124e06ade51442c3af9d364455ec278f"]}},"efe993ec7ad34cfaac8ca10da8b06b25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a845d33bdbc048af807e19eb89699a1b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_a87b20c55b6547d7bad171519b9c2029","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 29%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ce8d826cba9f41f8bb33997ceceab3c0"}},"e16dcd3650f64c61b1c53445f96d13d5":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_3ad0f66ba72c47ce8537870c6af56c0c","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":14,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":14,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3f68a6b94c614e87aea69a1fecf11d2e"}},"124e06ade51442c3af9d364455ec278f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_3b3af273314945ef951076ac38deb74c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 4/14 [00:08<00:16, 1.64s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ffd1831574034bc59136368ad3e60108"}},"a87b20c55b6547d7bad171519b9c2029":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ce8d826cba9f41f8bb33997ceceab3c0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3ad0f66ba72c47ce8537870c6af56c0c":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"3f68a6b94c614e87aea69a1fecf11d2e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3b3af273314945ef951076ac38deb74c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ffd1831574034bc59136368ad3e60108":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d8e841629bac43fbbea7f0a924aa9b94":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_dcde233fe4804bd8be4a35f7a781a985","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a31bcdbac3f54376814d4b9cd40d545b","IPY_MODEL_806b8ca3d7fe4e05b22b038a82eb4545","IPY_MODEL_a26e8dfe75d540208dae3d35a0444551"]}},"dcde233fe4804bd8be4a35f7a781a985":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a31bcdbac3f54376814d4b9cd40d545b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9a8c2b7f24ad47dea0cc84b99639f151","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 23%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_721da9071ffc493d97a523d2f2ac4f59"}},"806b8ca3d7fe4e05b22b038a82eb4545":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_0f5ac0bc528b4fa1b8d5a7aafeb36c79","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":13,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":13,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a37b2e9543074546bf43c7741f3a5650"}},"a26e8dfe75d540208dae3d35a0444551":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f95e707810f3426eb20134bad69e6a7f","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 3/13 [00:08<00:21, 2.19s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c065af81128d4149b26e71b7a6cb4c14"}},"9a8c2b7f24ad47dea0cc84b99639f151":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"721da9071ffc493d97a523d2f2ac4f59":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0f5ac0bc528b4fa1b8d5a7aafeb36c79":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"a37b2e9543074546bf43c7741f3a5650":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f95e707810f3426eb20134bad69e6a7f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"c065af81128d4149b26e71b7a6cb4c14":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"153549b4beb84e54b5fe4bec2eecf075":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_d7e2ff7ce49d44588f16e7abfbb9885c","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_6a821f0913784d64b2e98282b6ae15a1","IPY_MODEL_f736e0b03dea46dea371a5abcbcde9d7","IPY_MODEL_f0f773f30026425fa0dfdcefd1f7d2dc"]}},"d7e2ff7ce49d44588f16e7abfbb9885c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6a821f0913784d64b2e98282b6ae15a1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_914c417613a446c28921a6cb430218dc","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 8%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_76e6aa6b457347af8ef3146d3d1e7437"}},"f736e0b03dea46dea371a5abcbcde9d7":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_3d87aae69d6149fc98c0b0a642fd9194","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":12,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":12,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4a588c2572094039983c106ffd558d86"}},"f0f773f30026425fa0dfdcefd1f7d2dc":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_346efff4305041a5bd4913a00ebae73a","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1/12 [00:07<01:26, 7.87s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3a36af3ccfd54942b5e4643c373f60a2"}},"914c417613a446c28921a6cb430218dc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"76e6aa6b457347af8ef3146d3d1e7437":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3d87aae69d6149fc98c0b0a642fd9194":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"4a588c2572094039983c106ffd558d86":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"346efff4305041a5bd4913a00ebae73a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"3a36af3ccfd54942b5e4643c373f60a2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d1538f6f3b7641ccb48e2bbdef414389":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_3810f04feb094355bdc304447a3356b5","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_2528ae76b0a7423a8910adf401a5514a","IPY_MODEL_d8d6807eee1e44f78e10dcd5251c3e58","IPY_MODEL_21eac8d36aaf4fa2a0565329dedd965b"]}},"3810f04feb094355bdc304447a3356b5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2528ae76b0a7423a8910adf401a5514a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_411e0eeb49834c04bbca486de6d10b5d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 9%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d3282d958c424f84a1b0e2f65dc6602b"}},"d8d6807eee1e44f78e10dcd5251c3e58":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_28e68ecc22cf450d8ac9b8067ea0f801","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":11,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":11,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_310d810b45114471bbc5a6658de789df"}},"21eac8d36aaf4fa2a0565329dedd965b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9c7bed0810f34fe5948110f9c4ad1894","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1/11 [00:07<01:14, 7.48s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c8dc39d329de455ea39ec34406e6bd59"}},"411e0eeb49834c04bbca486de6d10b5d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"d3282d958c424f84a1b0e2f65dc6602b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"28e68ecc22cf450d8ac9b8067ea0f801":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"310d810b45114471bbc5a6658de789df":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9c7bed0810f34fe5948110f9c4ad1894":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"c8dc39d329de455ea39ec34406e6bd59":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"87c240ca0fe8431096f5092d00b6d63f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_6933623da13c468ea1caba8cb1e126f5","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3beaa54b58654d2199153c4229b50f84","IPY_MODEL_317c46945fd64488ad7b2cc0bf30b7d2","IPY_MODEL_fa2e58ea24f84364b95fcc3f1587e9ad"]}},"6933623da13c468ea1caba8cb1e126f5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3beaa54b58654d2199153c4229b50f84":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_afa78843d54d404ab99cd87d55a36bb9","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0c9761efc1a04534871cb6c05f232f4a"}},"317c46945fd64488ad7b2cc0bf30b7d2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_21b40574fdb44788af67ece3c7b770c6","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0a8e3b93b06740ad8cb9075dbf5d041a"}},"fa2e58ea24f84364b95fcc3f1587e9ad":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_bcaa037ccf1b47bf8d40956a370f93dc","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [00:07<00:00, 1.31it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2258f153219f48cc882eda53f0db33a2"}},"afa78843d54d404ab99cd87d55a36bb9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"0c9761efc1a04534871cb6c05f232f4a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"21b40574fdb44788af67ece3c7b770c6":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"0a8e3b93b06740ad8cb9075dbf5d041a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"bcaa037ccf1b47bf8d40956a370f93dc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2258f153219f48cc882eda53f0db33a2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6574536e49dd498bae5d87ab60144c59":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_a787648e15174568a65f7b5e17ee43d2","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_9fccdb73039f4b649f5389aa2b4099ad","IPY_MODEL_701a04ff58fc492f88774aa7139dcbc2","IPY_MODEL_37296861808045328c2c4eb74625987a"]}},"a787648e15174568a65f7b5e17ee43d2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9fccdb73039f4b649f5389aa2b4099ad":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_8a9c221fcb4a46f9af525b5b0373a598","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f155f2ec83cb416d8d375ae77330165b"}},"701a04ff58fc492f88774aa7139dcbc2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_af831b75ff7146fc99abb8677c84bdf3","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":1000,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1000,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_8fe71971bf8049bca4a5c6fde2bf7471"}},"37296861808045328c2c4eb74625987a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9a87fac6283543008f922a57f98b8514","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1000/1000 [10:38<00:00, 1.62it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ea8e8fab825a499c8fe628a021b731ca"}},"8a9c221fcb4a46f9af525b5b0373a598":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f155f2ec83cb416d8d375ae77330165b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"af831b75ff7146fc99abb8677c84bdf3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"8fe71971bf8049bca4a5c6fde2bf7471":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9a87fac6283543008f922a57f98b8514":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ea8e8fab825a499c8fe628a021b731ca":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d2a7f4cc89a44501b6423cff8f76f77d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9231d7a857624fbfa4c6dc1ba3307814","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_7dbee5d5ef884477be34e5a9889d13b1","IPY_MODEL_ed0108c515b94090bc7ee284284b535b","IPY_MODEL_ef228c5730d943149399ec81f4de4d46"]}},"9231d7a857624fbfa4c6dc1ba3307814":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7dbee5d5ef884477be34e5a9889d13b1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0dd4ce26eb994bf89f9d429982672f1d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3c78bcaf65d745928c7cec9b3618f2f4"}},"ed0108c515b94090bc7ee284284b535b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_3f9147ea50504b5e941cf600259b64c6","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":1000,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1000,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3cb22a9a850b42ce8d72132b34cede77"}},"ef228c5730d943149399ec81f4de4d46":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_3929bb10e2044d26869495a947ecd340","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1000/1000 [13:01<00:00, 1.28it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_cf4c98d2628e418b84ef3c798588675a"}},"0dd4ce26eb994bf89f9d429982672f1d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"3c78bcaf65d745928c7cec9b3618f2f4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3f9147ea50504b5e941cf600259b64c6":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"3cb22a9a850b42ce8d72132b34cede77":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3929bb10e2044d26869495a947ecd340":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"cf4c98d2628e418b84ef3c798588675a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"959b3e666a064b6d8917c8625ecd8fee":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_ba3a1e89d28342f683d0013065137cb2","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_53079098231c4906af72b59e6773052f","IPY_MODEL_3415712fbd16477b8eb7bf6844c56d51","IPY_MODEL_7269023608aa4b9c833466b4634a02cd"]}},"ba3a1e89d28342f683d0013065137cb2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"53079098231c4906af72b59e6773052f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_23ca025a590c4caba147cfee82b8fb82","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6892149d580c4521bfbc8ce341445d61"}},"3415712fbd16477b8eb7bf6844c56d51":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_0767ce00fdc24b709c3ca2169e13e545","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c11a3dcd03f24240826793c233950bea"}},"7269023608aa4b9c833466b4634a02cd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ff5a4612a01b47a8a805ca8a643956c3","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [00:27<00:00, 2.75s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f8e2be434a1846c797556912bad038f1"}},"23ca025a590c4caba147cfee82b8fb82":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"6892149d580c4521bfbc8ce341445d61":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0767ce00fdc24b709c3ca2169e13e545":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c11a3dcd03f24240826793c233950bea":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ff5a4612a01b47a8a805ca8a643956c3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f8e2be434a1846c797556912bad038f1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"85a8b0c33c104b8eb22fe9e21dc3d543":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f8a6e8d21951434b84789ff596bf188b","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3825eaf811174f30b38725c44d1536a6","IPY_MODEL_f0fad1d578be4cb790bcbc91e2088d85","IPY_MODEL_367d61cced8a4b7a9eb3763f11856cce"]}},"f8a6e8d21951434b84789ff596bf188b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3825eaf811174f30b38725c44d1536a6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0655f854114a4705ba29eabd0cc99e31","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f06c08cde5e14462a35c9b411e3cb167"}},"f0fad1d578be4cb790bcbc91e2088d85":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d769f331f73d42a1ae05b69c07023890","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_52ec6649d8e94615bf9397cbd9d85b48"}},"367d61cced8a4b7a9eb3763f11856cce":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_fa0bd788f3a8421d949951e60f790c07","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [00:26<00:00, 2.71s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2464f2977ba04e8a855461a081005305"}},"0655f854114a4705ba29eabd0cc99e31":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f06c08cde5e14462a35c9b411e3cb167":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d769f331f73d42a1ae05b69c07023890":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"52ec6649d8e94615bf9397cbd9d85b48":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"fa0bd788f3a8421d949951e60f790c07":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2464f2977ba04e8a855461a081005305":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"07d74cc8c1034e8e88e0ba68681e7d83":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_20755197a75045769c76c25cf9997309","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3aaecc6ab5334dbaaff91f3fd3d2dc92","IPY_MODEL_ccdfd084e55f4d48aa8d62ff900c54dc","IPY_MODEL_602801fccd6d4538bd22a5494c8a538e"]}},"20755197a75045769c76c25cf9997309":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3aaecc6ab5334dbaaff91f3fd3d2dc92":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_81b219b6e4414ff888646a18f038c6d2","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4622f6b8c7914be18ec086e119b14510"}},"ccdfd084e55f4d48aa8d62ff900c54dc":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_c603ecbfec974667805abf5f8366dde5","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":20,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":20,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_508403ada600443f97467dde6fb0dd3d"}},"602801fccd6d4538bd22a5494c8a538e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9840772f43134cee8eabacd2509f9fd7","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 20/20 [00:15<00:00, 1.29it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4e96a7f4873f4360b2118e7a3ddbfc48"}},"81b219b6e4414ff888646a18f038c6d2":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"4622f6b8c7914be18ec086e119b14510":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c603ecbfec974667805abf5f8366dde5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"508403ada600443f97467dde6fb0dd3d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9840772f43134cee8eabacd2509f9fd7":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"4e96a7f4873f4360b2118e7a3ddbfc48":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c19256a397ea4e79b3a0cbf5a424a3ea":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_4d1f5676bbaa4b3d8b81cc7be3cf6da3","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_eae2220f0119401a8759637bcf0e9f4c","IPY_MODEL_4aa44dd37d5243dfa24fea8f3dc6839b","IPY_MODEL_2deab0a58cd04226b834a99ee4ff4c90"]}},"4d1f5676bbaa4b3d8b81cc7be3cf6da3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"eae2220f0119401a8759637bcf0e9f4c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_16acd859901c4b8aa5f6b4473b74e249","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e4edb80a747a4e9684a556ff96563e7f"}},"4aa44dd37d5243dfa24fea8f3dc6839b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_3d74bbdd72dc4e0aadb4bd517358bbe7","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":20,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":20,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_809de5d470244a2b90daffd0477c28a4"}},"2deab0a58cd04226b834a99ee4ff4c90":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_4e13cae2ba7a44daa51794431d7af0c0","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 20/20 [00:15<00:00, 1.31it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2780d1d40a0644c498e860cf4839ac25"}},"16acd859901c4b8aa5f6b4473b74e249":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"e4edb80a747a4e9684a556ff96563e7f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3d74bbdd72dc4e0aadb4bd517358bbe7":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"809de5d470244a2b90daffd0477c28a4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4e13cae2ba7a44daa51794431d7af0c0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2780d1d40a0644c498e860cf4839ac25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a2c68a56898548f8a7e190dcf2d034f4":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_e009631910e042a2a07afe8cf0e6d118","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a850623290b14d07838e4203a29f9ba2","IPY_MODEL_9f140b090cb44361b39a6e09205c087c","IPY_MODEL_36fc4560cb99439c945c620138005a1b"]}},"e009631910e042a2a07afe8cf0e6d118":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a850623290b14d07838e4203a29f9ba2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_638ee1bca5d64cc7966556e3fb64c06e","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a454e1e1460046e3a98a4b28c48ecda0"}},"9f140b090cb44361b39a6e09205c087c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d0a15e2054af430bad52b6fc28fd0994","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f9295dd431fe4587a816d6c59f464399"}},"36fc4560cb99439c945c620138005a1b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_41ca699547f14bda8a83777d78e6bc86","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [15:36<00:00, 95.65s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c29bc574f0cd46309d8eb6ec40378025"}},"638ee1bca5d64cc7966556e3fb64c06e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a454e1e1460046e3a98a4b28c48ecda0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d0a15e2054af430bad52b6fc28fd0994":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"f9295dd431fe4587a816d6c59f464399":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"41ca699547f14bda8a83777d78e6bc86":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"c29bc574f0cd46309d8eb6ec40378025":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"858e460d943744248a4109a5c328d77c":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_700c0953693b48bea968c6c67964684d","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_c70158af000c40e59b588e745be4a2dd","IPY_MODEL_21b1b038a25f469295ea8e2b75c470ae","IPY_MODEL_c20bc14a583e452ba263c4e4f4f3147e"]}},"700c0953693b48bea968c6c67964684d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c70158af000c40e59b588e745be4a2dd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_e1199401ceee4d0ca1aaa72020618241","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_57b794458691470ba34efd35f0cd8a9c"}},"21b1b038a25f469295ea8e2b75c470ae":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4bd9a0cc11664bf4870c7fc46d151bda","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_301a50e882864065b9108e0a0cf0edde"}},"c20bc14a583e452ba263c4e4f4f3147e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_93f8c74e75504aadbf1cdec1c5b6a43f","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [00:14<00:00, 1.43s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a4d3c862e74242e4beeaaa880f4e3fc8"}},"e1199401ceee4d0ca1aaa72020618241":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"57b794458691470ba34efd35f0cd8a9c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4bd9a0cc11664bf4870c7fc46d151bda":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"301a50e882864065b9108e0a0cf0edde":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"93f8c74e75504aadbf1cdec1c5b6a43f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a4d3c862e74242e4beeaaa880f4e3fc8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"79e9b193d9284e4b898cb4344db92ef3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_8ea51857d7cf48c1beddb0a9bd3b764b","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_7b0e4960186b4307ae594b269043c16d","IPY_MODEL_aa39361015a84c158e922f8e0b8dca9a","IPY_MODEL_63acbf6a68e6455c91cbea40169d1ee9"]}},"8ea51857d7cf48c1beddb0a9bd3b764b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7b0e4960186b4307ae594b269043c16d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_e2c2139f96b345768e075ef3593fac06","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3dcbffa6176e4824b9a48cccfaf024cc"}},"aa39361015a84c158e922f8e0b8dca9a":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_740c9d9905b542d4a0edef78f1d8ca30","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_cd2dc9d9c7504cd4b118a33b7c24f683"}},"63acbf6a68e6455c91cbea40169d1ee9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_33707c49e4fd42ad932dd74e59f4a060","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [02:13<00:00, 13.01s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_60745768aedc4942bad8883d708b0e25"}},"e2c2139f96b345768e075ef3593fac06":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"3dcbffa6176e4824b9a48cccfaf024cc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"740c9d9905b542d4a0edef78f1d8ca30":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"cd2dc9d9c7504cd4b118a33b7c24f683":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"33707c49e4fd42ad932dd74e59f4a060":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"60745768aedc4942bad8883d708b0e25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"48077097754d4b96b1345fb1c06560b8":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_760943cea30f4236a0715b61a047fd99","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3bf9f4be47424b13b1db937f67894360","IPY_MODEL_28c1b766f56c41179840f2140da35b44","IPY_MODEL_487beca1cfc847c0a41a414968097b6b"]}},"760943cea30f4236a0715b61a047fd99":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3bf9f4be47424b13b1db937f67894360":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c28b5aecd2eb4353b99534e8eef2380c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f167c63d8ac54b07b31b261c77a98391"}},"28c1b766f56c41179840f2140da35b44":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4a8e550188ac481fa2838df0996d3b0b","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":10,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":10,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_50dfc1413ebd4298af9fc979de4ebb63"}},"487beca1cfc847c0a41a414968097b6b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f36eb7ce107741159cfc86169724580f","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 10/10 [00:49<00:00, 5.04s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_5b4edf526246433d8cca04eba8be3229"}},"c28b5aecd2eb4353b99534e8eef2380c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f167c63d8ac54b07b31b261c77a98391":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4a8e550188ac481fa2838df0996d3b0b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"50dfc1413ebd4298af9fc979de4ebb63":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f36eb7ce107741159cfc86169724580f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"5b4edf526246433d8cca04eba8be3229":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","source":["# Sequential Batch Learning in Stochastic MAB and Contextual MAB on Mushroom and Synthetic data"],"metadata":{"id":"i_1jvK-oGLjU"}},{"cell_type":"markdown","source":["## Executive summary\n","\n","| | |\n","| --- | --- |\n","| Problem | Learning user preferences online might have an impact of delay and training recommender system sequentially for every example is computationally heavy. |\n","| Hypothesis | A learning agent observes responses batched in groups over a certain time period. The impact of batch learning can be measured in terms of online behavior. |\n","| Prblm Stmt. | Given a finite set of arms ⁍, an environment ⁍ (⁍ is the distribution of rewards for action ⁍), and a time horizon ⁍, at each time step ⁍, the agent chooses an action ⁍ and receives a reward ⁍. The goal of the agent is to maximize the total reward ⁍. |\n","| Solution | Sequential batch learning is a more generalized way of learning which covers both offline and online settings as special cases bringing together their advantages. Unlike offline learning, sequential batch learning retains the sequential nature of the problem. Unlike online learning, it is often appealing to implement batch learning in large scale bandit problems. In this setting, responses are grouped in batches and observed by the agent only at the end of each batch. |\n","| Dataset | Mushroom, Synthetic |\n","| Preprocessing | Train/test split, label encoding |\n","| Metrics | Conversion rate, regret |\n","| Credits | [Danil Provodin](https://github.com/danilprov) |"],"metadata":{"id":"bLatuDpMGLge"}},{"cell_type":"markdown","source":["### Environments\n","\n","| Name | Type | Rewards |\n","| --- | --- | --- |\n","| env1 | 2-arm environment | [0.7, 0.5] |\n","| env2 | 2-arm environment | [0.7, 0.4] |\n","| env3 | 2-arm environment | [0.7, 0.1] |\n","| env4 | 4-arm environment | [0.35, 0.18, 0.47, 0.61] |\n","| env5 | 4-arm environment | [0.40, 0.75, 0.57, 0.49] |\n","| env6 | 4-arm environment | [0.70, 0.50, 0.30, 0.10] |"],"metadata":{"id":"y-pKQhr9GaAv"}},{"cell_type":"markdown","source":["### Simulation\n","\n","| Application | Policy |\n","| --- | --- |\n","| Multi-armed bandit (MAB) | Thompson Sampling (TS) |\n","| Multi-armed bandit (MAB) | Upper Confidence Bound (UCB) |\n","| Contextual MAB (CMAB) | Linear Thompson Sampling (LinTS) |\n","| Contextual MAB (CMAB) | Linear UCB (LinUCB) |"],"metadata":{"id":"QPyNX42KGiaR"}},{"cell_type":"markdown","source":["## Process flow"],"metadata":{"id":"4veQdUjFGCwk"}},{"cell_type":"markdown","source":["![](https://github.com/RecoHut-Stanzas/S873634/raw/main/images/process_flow.svg)"],"metadata":{"id":"Pm8qSJsaGEUN"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"VAwEfoPurXGo"}},{"cell_type":"markdown","source":["### Imports"],"metadata":{"id":"ngwffbD4rXE3"}},{"cell_type":"code","source":["import pandas as pd\n","import numpy as np\n","from numpy.linalg import inv\n","from sklearn.metrics import roc_auc_score\n","from sklearn.model_selection import GridSearchCV, train_test_split, cross_validate\n","from sklearn.tree import DecisionTreeClassifier\n","from scipy.optimize import minimize\n","from lightgbm import LGBMClassifier\n","from scipy.stats import beta\n","import pickle\n","import os\n","import shutil\n","\n","import tqdm\n","from tqdm.notebook import tqdm\n","from multiprocessing.dummy import Pool\n","from IPython.display import clear_output\n","import matplotlib.pyplot as plt\n","\n","from torch.utils.data import Dataset, DataLoader\n","\n","from __future__ import print_function\n","from abc import ABCMeta, abstractmethod"],"metadata":{"id":"vSHj786VrXC8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Data"],"metadata":{"id":"XXMnN2vioutk"}},{"cell_type":"markdown","source":["### Download"],"metadata":{"id":"W_L8uOd0rfK6"}},{"cell_type":"markdown","source":["This data set includes descriptions of hypothetical samples corresponding to 23 species of gilled mushrooms in the Agaricus and Lepiota Family (pp. 500-525). Each species is identified as definitely edible, definitely poisonous, or of unknown edibility and not recommended. This latter class was combined with the poisonous one. The Guide clearly states that there is no simple rule for determining the edibility of a mushroom; no rule like ``leaflets three, let it be'' for Poisonous Oak and Ivy. More details [here](https://archive.ics.uci.edu/ml/datasets/mushroom)."],"metadata":{"id":"-f9daEHdrG5E"}},{"cell_type":"code","source":["!mkdir -p data\n","!cd data && wget -q --show-progress https://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data\n","!cd data && wget -q --show-progress https://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.names"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"adGWyfxFrHij","executionInfo":{"status":"ok","timestamp":1639145418029,"user_tz":-330,"elapsed":1715,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"eb9c650f-63bc-4fcf-ef24-b1f698d71d4a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["agaricus-lepiota.da 100%[===================>] 364.95K 1.31MB/s in 0.3s \n","agaricus-lepiota.na 100%[===================>] 6.66K --.-KB/s in 0s \n"]}]},{"cell_type":"markdown","source":["### Preprocessing"],"metadata":{"id":"Aal7wGRfrhfe"}},{"cell_type":"code","source":["mushroom_data = pd.read_csv(\"data/agaricus-lepiota.data\", header=None)\n","\n","column_names = [\"classes\", \"cap-shape\", \"cap-surface\", \"cap-color\", \"bruises?\", \"odor\", \"gill-attachment\",\n"," \"gill-spacing\", \"gill-size\", \"gill-color\", \"stalk-shape\", \"stalk-root\", \"stalk-surface-above-ring\",\n"," \"stalk-surface-below-ring\", \"stalk-color-above-ring\", \"stalk-color-below-ring\", \"veil-type\", \n"," \"veil-color\", \"ring-number\", \"ring-type\", \"spore-print-color\", \"population\", \"habitat\"]\n"," \n","mushroom_data.columns = column_names\n","mushroom_data.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":278},"id":"TshNYFbQrhdi","executionInfo":{"status":"ok","timestamp":1639145489020,"user_tz":-330,"elapsed":516,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"88c714b1-d1c5-4666-8f70-9db8b6fe6584"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
classescap-shapecap-surfacecap-colorbruises?odorgill-attachmentgill-spacinggill-sizegill-colorstalk-shapestalk-rootstalk-surface-above-ringstalk-surface-below-ringstalk-color-above-ringstalk-color-below-ringveil-typeveil-colorring-numberring-typespore-print-colorpopulationhabitat
0pxsntpfcnkeesswwpwopksu
1exsytafcbkecsswwpwopnng
2ebswtlfcbnecsswwpwopnnm
3pxywtpfcnneesswwpwopksu
4exsgfnfwbktesswwpwoenag
\n","
"],"text/plain":[" classes cap-shape cap-surface ... spore-print-color population habitat\n","0 p x s ... k s u\n","1 e x s ... n n g\n","2 e b s ... n n m\n","3 p x y ... k s u\n","4 e x s ... n a g\n","\n","[5 rows x 23 columns]"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["mushroom_data.dtypes"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"sIbAiLjtrhbZ","executionInfo":{"status":"ok","timestamp":1639145504479,"user_tz":-330,"elapsed":518,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"f0602325-8225-4047-b8a9-503031bba8c4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["classes object\n","cap-shape object\n","cap-surface object\n","cap-color object\n","bruises? object\n","odor object\n","gill-attachment object\n","gill-spacing object\n","gill-size object\n","gill-color object\n","stalk-shape object\n","stalk-root object\n","stalk-surface-above-ring object\n","stalk-surface-below-ring object\n","stalk-color-above-ring object\n","stalk-color-below-ring object\n","veil-type object\n","veil-color object\n","ring-number object\n","ring-type object\n","spore-print-color object\n","population object\n","habitat object\n","dtype: object"]},"metadata":{},"execution_count":4}]},{"cell_type":"code","source":["# label encoding\n","for column in column_names:\n"," mushroom_data[column] = mushroom_data[column].astype('category')\n"," mushroom_data[column] = mushroom_data[column].cat.codes\n","\n","# split\n","idx_trn, idx_tst = train_test_split(mushroom_data.index, test_size=0.2, random_state=42, \n"," stratify=mushroom_data[['classes']])"],"metadata":{"id":"I420evWNrhZQ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["gini by factors"],"metadata":{"id":"c6AARTV0sSEw"}},{"cell_type":"code","source":["def gini(var):\n"," df = mushroom_data.copy()\n"," x_trn = df.loc[idx_trn, var]\n"," y_trn = df.loc[idx_trn, 'classes']\n"," x_tst = df.loc[idx_tst, var]\n"," y_tst = df.loc[idx_tst, 'classes']\n"," \n"," if x_trn.dtype in ['O','object']:\n"," cats = pd.DataFrame({'x': x_trn, 'y': y_trn}).fillna('#NAN#').groupby('x').agg('mean').sort_values('y').index.values\n"," X_trn = pd.Categorical(x_trn.fillna('#NAN#'), categories=cats, ordered=True).codes.reshape(-1, 1)\n"," X_tst = pd.Categorical(x_tst.fillna('#NAN#'), categories=cats, ordered=True).codes.reshape(-1, 1)\n"," else:\n"," repl = min(x_trn.min(), x_tst.min())-1 if np.isfinite(min(x_trn.min(), x_tst.min())-1) else -999999\n"," #repl = x_trn.min()-1 if np.isfinite(x_trn.min())-1 else -999999\n"," X_trn = x_trn.fillna(repl).replace(np.inf, repl).replace(-np.inf, repl).values.reshape(-1, 1)\n"," X_tst = x_tst.fillna(repl).replace(np.inf, repl).replace(-np.inf, repl).values.reshape(-1, 1)\n"," \n"," obvious_gini_trn = 2*roc_auc_score(y_trn, X_trn)-1\n"," obvious_gini_tst = 2*roc_auc_score(y_tst, X_tst)-1\n","\n"," if obvious_gini_trn < 0:\n"," obvious_gini_trn = -obvious_gini_trn\n"," obvious_gini_tst = -obvious_gini_tst\n","\n"," parameters = {'min_samples_leaf':[0.01, 0.025, 0.05, 0.1]}\n"," dt = DecisionTreeClassifier(random_state=1)\n"," clf = GridSearchCV(dt, parameters, cv=4, scoring='roc_auc', n_jobs=10)\n"," clf.fit(X_trn, y_trn)\n","\n"," true_gini_trn = 2*clf.best_score_-1\n"," true_gini_tst = 2*roc_auc_score(y_tst, clf.predict_proba(X_tst)[:, 1])-1\n","\n"," if true_gini_trn < 0:\n"," true_gini_trn = -true_gini_trn\n"," true_gini_tst = -true_gini_tst\n","\n"," if obvious_gini_trn > true_gini_trn:\n"," return [var, obvious_gini_trn, obvious_gini_tst]\n"," else:\n"," return [var, true_gini_trn, true_gini_tst]"],"metadata":{"id":"hjgbgx7NrhWv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["with Pool(20) as p:\n"," vars_gini = list(tqdm(p.imap(gini, column_names), total=len(column_names)))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["2b72cbdd08374942859625d2047795f2","396e4164d325484e9956f597c2e133e0","46ce32b9cbc74045b96fc2fbc93f0575","596798eaeac14aaea8c3cc476b7f329f","c60ea6ae22a2457786ca6b8e3446e3f2","342e5fcfeeb04720834331a4274b5e28","9cbda8d73a0a41f8acb9c3bdacdc7a9b","042addbfa66d4dd4a8543124cc312905","3e9ddf2ca50f4c24bcf276eeeee63d01","ce6fa32eee4044df843b45c5693e065e","5ac6a207b502473fa8bd11d2adcc1201"]},"id":"pwtreallsHVN","executionInfo":{"status":"ok","timestamp":1639145595544,"user_tz":-330,"elapsed":9639,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"d6e113bd-e273-4922-f04d-a4d6448d26b2"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2b72cbdd08374942859625d2047795f2","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/23 [00:00\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
classescap-shapecap-surfacecap-colorbruises?odorgill-attachmentgill-spacinggill-sizegill-colorstalk-shapestalk-rootstalk-surface-above-ringstalk-surface-below-ringstalk-color-above-ringstalk-color-below-ringveil-typeveil-colorring-numberring-typespore-print-colorpopulationhabitat
gini_train1.00.1852250.1809560.2077930.4958150.9674870.0401170.2562510.4905630.7583490.102960.4298980.5437430.5340910.5440610.5345530.00.0457910.1146170.6270000.7616040.5190780.444780
gini_test1.00.2127790.2011180.2372780.4900010.9695550.0446830.2578580.5352030.7600570.093950.4260110.5304220.5329510.5280230.5166070.00.0547050.1171930.6220520.7460500.5104360.476353
\n",""],"text/plain":["0 classes cap-shape ... population habitat\n","gini_train 1.0 0.185225 ... 0.519078 0.444780\n","gini_test 1.0 0.212779 ... 0.510436 0.476353\n","\n","[2 rows x 23 columns]"]},"metadata":{},"execution_count":8}]},{"cell_type":"markdown","source":["Correlation analysis"],"metadata":{"id":"2CWLsNxLsVYI"}},{"cell_type":"code","source":["vars_corrs = mushroom_data.loc[:, column_names].corr().abs().stack().reset_index().drop_duplicates()\n","vars_corrs = vars_corrs[vars_corrs.level_0!=vars_corrs.level_1]\n","vars_corrs.columns = ['var_1', 'var_2', 'correlation']\n","vars_corrs = vars_corrs.set_index(['var_1', 'var_2'], drop=True).sort_values(by='correlation', ascending=False)\n","\n","vars_drop = []\n","\n","for v in vars_corrs[vars_corrs.correlation > 0.7].index.values:\n"," if v[0] not in vars_drop and v[1] not in vars_drop:\n"," vars_drop.append(v[1] if vars_gini.loc[v[0], 'gini_train'] > vars_gini.loc[v[1], 'gini_train'] else v[0])\n"," \n","del v"],"metadata":{"id":"U94p4ETPscxi"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Feature selection"],"metadata":{"id":"q1MuoCp4sk8r"}},{"cell_type":"code","source":["# all variables\n","vars0 = column_names[1:]\n","\n","# drop values with gini less than 3%\n","vars1 = [v for v in vars0 if vars_gini.loc[v, 'gini_train'] >= 0.03]\n","\n","# drop correlated variables\n","vars2 = [v for v in vars1 if v not in vars_drop]\n","\n","i = 0\n","\n","for var_lst in [vars0, vars1, vars2]:\n"," i += 1\n"," lgb = LGBMClassifier(max_depth=1, n_estimators=250, random_state=42, n_jobs=30)\n"," \n"," cv = cross_validate(lgb, mushroom_data.loc[:, var_lst], mushroom_data.loc[:, 'classes'], \n"," cv=5, scoring='roc_auc', n_jobs=20, return_train_score=True)\n"," \n"," lgb.fit(mushroom_data[var_lst], mushroom_data['classes'])\n"," \n"," print({'Variables': len(var_lst), \n"," 'Train CV': round(cv['train_score'].mean()*2-1, 4), \n"," 'Test CV': round(cv['test_score'].mean()*2-1, 4)})\n"," \n","var_lst_imp = pd.Series(dict(zip(var_lst, lgb.feature_importances_)))\n","var_lst = [i for i in var_lst_imp.index if var_lst_imp.loc[i]>0]\n","print({'exclude': [i for i in var_lst_imp.index if var_lst_imp.loc[i]<=0]})\n","print(len(var_lst))\n","\n","forw_cols = []\n","current_ginis = pd.Series({'Train CV':0, 'Test CV':0})\n","\n","def forw(x):\n"," lgb = LGBMClassifier(max_depth=1, n_estimators=250, random_state=42, n_jobs=1)\n"," cv = cross_validate(lgb, mushroom_data.loc[:, forw_cols+[x]], mushroom_data.loc[:, 'classes'],\n"," cv=5, scoring='roc_auc', n_jobs=1, return_train_score=True)\n"," lgb.fit(mushroom_data.loc[:, forw_cols+[x]], mushroom_data.loc[:, 'classes'])\n"," return x, pd.Series({\n"," 'Train CV': cv['train_score'].mean()*2-1,\n"," 'Test CV': cv['test_score'].mean()*2-1\n"," })\n","\n","forwards_log = []\n","while len(forw_cols)<30:\n"," with Pool(20) as p:\n"," res = list(tqdm(p.imap(forw, [i for i in var_lst if i not in forw_cols]), total=len(var_lst)-len(forw_cols), leave=False))\n"," res = pd.DataFrame({i[0]:i[1] for i in res}).T\n"," delta = res - current_ginis\n"," if delta['Test CV'].max()<0:\n"," break\n"," best_var = delta['Test CV'].idxmax()\n"," forw_cols = forw_cols + [best_var]\n"," current_ginis = res.loc[best_var]\n"," forwards_log.append(current_ginis)\n"," clear_output()\n"," print(pd.DataFrame(forwards_log))\n","\n","clear_output()\n","forwards_log = pd.DataFrame(forwards_log)\n","forwards_log['Uplift Train CV'] = forwards_log['Train CV']-forwards_log['Train CV'].shift(1).fillna(0)\n","forwards_log['Uplift Test CV'] = forwards_log['Test CV']-forwards_log['Test CV'].shift(1).fillna(0)\n","print(forwards_log)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":153,"referenced_widgets":["3d6f8baea53a484faab78b4c50fbf620","9689c9d0d5354f7782671e9b301c5840","4550c6895fa244efb6dd5cbc9dd98324","b70c31ffb8784655ad0d4e59e12343d6","c14978517f8047718a1c084947d702b0","60d54986769b4e859de45bbce659b7f1","0181512ceea0401db61675636ae223b5","4b34f4a343b242abaef58ebe57e07a4d","476d47eec56d4ff0bec2900ed3505ba2","27609bb3e0594b5084e01fae4b04cc9c","606916b9b63b43969a01e84ec142f341","b8cc488f50764d7a80c565e120acf3bd","73139f73ef7d4769807b6dc081a48fb1","8c0a7a56a8d946bf953390fd61675c4d","63ead4d72b27495cac3bb6a385895123","a1d8d9e611534997828fa51cdab2bd0a","afe8ad0a833b4f399a1f6d8367786fd3","c55e9f4461c24c9f88ba2098ff957634","4472bedf87ec41aeb3306d6364386540","edc2a60460ef48bdb78972f403c14175","d971324520f34ce5b3b6994ddd0d0102","6d4282c72b80490cbdb027d3592849ea","77454d4570db40ad9d90ed0f9d80f019","efe993ec7ad34cfaac8ca10da8b06b25","a845d33bdbc048af807e19eb89699a1b","e16dcd3650f64c61b1c53445f96d13d5","124e06ade51442c3af9d364455ec278f","a87b20c55b6547d7bad171519b9c2029","ce8d826cba9f41f8bb33997ceceab3c0","3ad0f66ba72c47ce8537870c6af56c0c","3f68a6b94c614e87aea69a1fecf11d2e","3b3af273314945ef951076ac38deb74c","ffd1831574034bc59136368ad3e60108","d8e841629bac43fbbea7f0a924aa9b94","dcde233fe4804bd8be4a35f7a781a985","a31bcdbac3f54376814d4b9cd40d545b","806b8ca3d7fe4e05b22b038a82eb4545","a26e8dfe75d540208dae3d35a0444551","9a8c2b7f24ad47dea0cc84b99639f151","721da9071ffc493d97a523d2f2ac4f59","0f5ac0bc528b4fa1b8d5a7aafeb36c79","a37b2e9543074546bf43c7741f3a5650","f95e707810f3426eb20134bad69e6a7f","c065af81128d4149b26e71b7a6cb4c14","153549b4beb84e54b5fe4bec2eecf075","d7e2ff7ce49d44588f16e7abfbb9885c","6a821f0913784d64b2e98282b6ae15a1","f736e0b03dea46dea371a5abcbcde9d7","f0f773f30026425fa0dfdcefd1f7d2dc","914c417613a446c28921a6cb430218dc","76e6aa6b457347af8ef3146d3d1e7437","3d87aae69d6149fc98c0b0a642fd9194","4a588c2572094039983c106ffd558d86","346efff4305041a5bd4913a00ebae73a","3a36af3ccfd54942b5e4643c373f60a2","d1538f6f3b7641ccb48e2bbdef414389","3810f04feb094355bdc304447a3356b5","2528ae76b0a7423a8910adf401a5514a","d8d6807eee1e44f78e10dcd5251c3e58","21eac8d36aaf4fa2a0565329dedd965b","411e0eeb49834c04bbca486de6d10b5d","d3282d958c424f84a1b0e2f65dc6602b","28e68ecc22cf450d8ac9b8067ea0f801","310d810b45114471bbc5a6658de789df","9c7bed0810f34fe5948110f9c4ad1894","c8dc39d329de455ea39ec34406e6bd59"]},"id":"WzyBNZ9psmGi","executionInfo":{"status":"ok","timestamp":1639145844301,"user_tz":-330,"elapsed":68523,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"c12cae13-09ca-4583-fa7d-1cb2fe20519f"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" Train CV Test CV Uplift Train CV Uplift Test CV\n","odor 0.970307 0.843798 0.970307 0.843798\n","gill-size 0.979971 0.958396 0.009664 0.114598\n","gill-spacing 0.984611 0.964531 0.004640 0.006135\n","stalk-shape 0.993921 0.982878 0.009310 0.018347\n","habitat 0.995208 0.984953 0.001287 0.002075\n"]}]},{"cell_type":"code","source":["ids_vars = forwards_log[forwards_log['Uplift Test CV']>0.001].index.values.tolist()\n","vars_gini.loc[ids_vars,:]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":238},"id":"jjPfCIG-sw2k","executionInfo":{"status":"ok","timestamp":1639145844304,"user_tz":-330,"elapsed":47,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"9fa96409-314f-4dab-a148-c38601d0e10b"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
gini_traingini_test
0
odor0.9674870.969555
gill-size0.4905630.535203
gill-spacing0.2562510.257858
stalk-shape0.1029600.093950
habitat0.4447800.476353
\n","
"],"text/plain":[" gini_train gini_test\n","0 \n","odor 0.967487 0.969555\n","gill-size 0.490563 0.535203\n","gill-spacing 0.256251 0.257858\n","stalk-shape 0.102960 0.093950\n","habitat 0.444780 0.476353"]},"metadata":{},"execution_count":12}]},{"cell_type":"code","source":["mushroom_data_features = mushroom_data[ids_vars + [\"classes\"]]\n","mushroom_data = mushroom_data_features.loc[mushroom_data_features.index.repeat(4)].reset_index(drop=True)\n","\n","mushroom_data[\"a\"] = np.random.choice([0, 1], mushroom_data.shape[0])\n","mushroom_data[\"probs\"] = 1\n","mushroom_data[\"y\"] = 0\n","\n","eat_edible = (1-mushroom_data[\"classes\"]) * mushroom_data[\"a\"] * 1\n","eat_poisonous = mushroom_data[\"classes\"] * mushroom_data[\"a\"] * np.random.choice([1, -1], mushroom_data.shape[0])\n","mushroom_data[\"y\"] = eat_edible + eat_poisonous\n","new_names = ['X_' + str(i+1) for i in range(len(ids_vars))] \n","mushroom_data = mushroom_data.rename(columns=dict(zip(ids_vars, new_names)))\n","\n","mushroom_data_final = mushroom_data[new_names + ['a', 'y', 'probs']]\n","mushroom_data_final.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"vEXcrpMis7Pi","executionInfo":{"status":"ok","timestamp":1639145844306,"user_tz":-330,"elapsed":39,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"bcb3014b-6da3-4627-e4a8-9e224a253f2a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
X_1X_2X_3X_4X_5ayprobs
061005111
161005001
261005001
3610051-11
400001111
\n","
"],"text/plain":[" X_1 X_2 X_3 X_4 X_5 a y probs\n","0 6 1 0 0 5 1 1 1\n","1 6 1 0 0 5 0 0 1\n","2 6 1 0 0 5 0 0 1\n","3 6 1 0 0 5 1 -1 1\n","4 0 0 0 0 1 1 1 1"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["with open('data/mushroom_data_final.pickle', 'wb') as handle:\n"," pickle.dump(mushroom_data_final, handle, protocol=pickle.HIGHEST_PROTOCOL)"],"metadata":{"id":"ZTavtFcks9iR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Utilities"],"metadata":{"id":"p4B8_mndtBlr"}},{"cell_type":"markdown","source":["### Softmax"],"metadata":{"id":"dHg46NSstX9e"}},{"cell_type":"code","source":["def softmax(action_values, tau=1.0):\n"," \"\"\"\n"," Args:\n"," action_values (Numpy array): A 2D array of shape (batch_size, num_actions).\n"," The action-values computed by an action-value network.\n"," tau (float): The temperature parameter scalar.\n"," Returns:\n"," A 2D array of shape (batch_size, num_actions). Where each column is a probability distribution over\n"," the actions representing the policy.\n"," \"\"\"\n","\n"," # Compute the preferences by dividing the action-values by the temperature parameter tau\n"," preferences = action_values / tau\n"," # Compute the maximum preference across the actions\n"," max_preference = np.max(preferences, axis=1)\n","\n"," # your code here\n","\n"," # Reshape max_preference array which has shape [Batch,] to [Batch, 1]. This allows NumPy broadcasting\n"," # when subtracting the maximum preference from the preference of each action.\n"," reshaped_max_preference = max_preference.reshape((-1, 1))\n"," # print(reshaped_max_preference)\n","\n"," # Compute the numerator, i.e., the exponential of the preference - the max preference.\n"," exp_preferences = np.exp(preferences - reshaped_max_preference)\n"," # print(exp_preferences)\n"," # Compute the denominator, i.e., the sum over the numerator along the actions axis.\n"," sum_of_exp_preferences = np.sum(exp_preferences, axis=1)\n"," # print(sum_of_exp_preferences)\n","\n"," # your code here\n","\n"," # Reshape sum_of_exp_preferences array which has shape [Batch,] to [Batch, 1] to allow for NumPy broadcasting\n"," # when dividing the numerator by the denominator.\n"," reshaped_sum_of_exp_preferences = sum_of_exp_preferences.reshape((-1, 1))\n"," # print(reshaped_sum_of_exp_preferences)\n","\n"," # Compute the action probabilities according to the equation in the previous cell.\n"," action_probs = exp_preferences / reshaped_sum_of_exp_preferences\n"," # print(action_probs)\n","\n"," # your code here\n","\n"," # squeeze() removes any singleton dimensions. It is used here because this function is used in the\n"," # agent policy when selecting an action (for which the batch dimension is 1.) As np.random.choice is used in\n"," # the agent policy and it expects 1D arrays, we need to remove this singleton batch dimension.\n"," action_probs = action_probs.squeeze()\n"," return action_probs"],"metadata":{"id":"utuZpkQktdjN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","# rand_generator = np.random.RandomState(0)\n","# action_values = rand_generator.normal(0, 1, (2, 4))\n","# tau = 0.5\n","\n","# action_probs = softmax(action_values, tau)\n","# print(\"action_probs\", action_probs)\n","\n","# assert (np.allclose(action_probs, np.array([\n","# [0.25849645, 0.01689625, 0.05374514, 0.67086216],\n","# [0.84699852, 0.00286345, 0.13520063, 0.01493741]\n","# ])))\n","\n","# action_values = np.array([[0.0327, 0.0127, 0.0688]])\n","# tau = 1.\n","# action_probs = softmax(action_values, tau)\n","# print(\"action_probs\", action_probs)\n","\n","# assert np.allclose(action_probs, np.array([0.3315, 0.3249, 0.3436]), atol=1e-04)\n","\n","# print(\"Passed the asserts! (Note: These are however limited in scope, additional testing is encouraged.)\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Mbk-YyyHtfqc","executionInfo":{"status":"ok","timestamp":1639145939697,"user_tz":-330,"elapsed":477,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"c0665afb-a0c1-460c-e4d7-7d9e20103146"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["action_probs [[0.25849645 0.01689625 0.05374514 0.67086216]\n"," [0.84699852 0.00286345 0.13520063 0.01493741]]\n","action_probs [0.33145968 0.32489634 0.34364398]\n","Passed the asserts! (Note: These are however limited in scope, additional testing is encouraged.)\n"]}]},{"cell_type":"markdown","source":["### Replay buffer"],"metadata":{"id":"CsJuwC4nth4A"}},{"cell_type":"code","source":["class ReplayBuffer:\n"," def __init__(self, size, seed):\n"," \"\"\"\n"," Args:\n"," size (integer): The size of the replay buffer.\n"," minibatch_size (integer): The sample size.\n"," seed (integer): The seed for the random number generator.\n"," \"\"\"\n"," self.buffer = []\n"," self.rand_generator = np.random.RandomState(seed)\n"," self.max_size = size\n","\n"," def append(self, state, action, reward):\n"," \"\"\"\n"," Args:\n"," state (Numpy array): The state.\n"," action (integer): The action.\n"," reward (float): The reward.\n"," terminal (integer): 1 if the next state is a terminal state and 0 otherwise.\n"," next_state (Numpy array): The next state.\n"," \"\"\"\n"," if len(self.buffer) == self.max_size:\n"," del self.buffer[0]\n"," self.buffer.append([state, action, reward])\n","\n"," def sample(self, last_action):\n"," \"\"\"\n"," Returns:\n"," A list of transition tuples including state, action, reward, terinal, and next_state\n"," \"\"\"\n"," state, action, reward = map(list, zip(*self.buffer))\n"," idxs = [elem == last_action for elem in action]\n"," X = [b for a, b in zip(idxs, state) if a]\n"," y = [b for a, b in zip(idxs, reward) if a]\n","\n"," return X, y\n","\n"," def size(self):\n"," return len(self.buffer)"],"metadata":{"id":"mPZIB5GOtm7D"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == \"__main__\":\n","\n","# buffer = ReplayBuffer(size=100000, seed=1)\n","# buffer.append([1, 2, 3], 0, 1)\n","# buffer.append([4, 21, 3], 1, 1)\n","# buffer.append([0, 1, 1], 0, 0)\n","\n","# print(buffer.sample(0))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XCsLBOzDtozO","executionInfo":{"status":"ok","timestamp":1639145971783,"user_tz":-330,"elapsed":9,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"fbd79fdd-1c19-4e74-ce5d-77d8b66a834c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["([[1, 2, 3], [0, 1, 1]], [1, 0])\n"]}]},{"cell_type":"markdown","source":["### Data Generator"],"metadata":{"id":"MGlV70x3tpu7"}},{"cell_type":"code","source":["def generate_samples(num_samples, num_features, num_arms, return_dataframe=False):\n"," np.random.seed(1)\n"," # generate pseudo features X and \"true\" arms' weights\n"," X = np.random.randint(0, 4, size=(num_samples, num_features))\n"," actions_weights = np.random.normal(loc=-1., scale=1, size=(num_arms, num_features))\n","\n"," # apply data generating policy\n"," policy_weights = np.random.normal(size=(num_arms, num_features))\n"," action_scores = np.dot(X, policy_weights.T)\n"," action_probs = softmax(action_scores, tau=10)\n"," A = np.zeros((num_samples, 1))\n"," for i in range(num_samples):\n"," A[i, 0] = np.random.choice(range(num_arms), 1, p=action_probs[i, :])\n","\n"," # store probabilities of choosing a particular action\n"," _rows = np.zeros_like(A, dtype=np.intp)\n"," _columns = A.astype(int)\n"," probs = action_probs[_rows, _columns]\n","\n"," # calculate \"true\" outcomes Y\n"," ## broadcasting chosen actions to action weights\n"," matrix_multiplicator = actions_weights[_columns].squeeze() # (num_samples x num_features) matrix\n"," rewards = np.sum(X * matrix_multiplicator, axis=1).reshape(-1, 1)\n"," Y = (np.sign(rewards) + 1) / 2\n","\n"," if return_dataframe:\n"," column_names = ['X_' + str(i+1) for i in range(num_features)]\n"," X = pd.DataFrame(X, columns=column_names)\n"," A = pd.DataFrame(A, columns=['a'])\n"," Y = pd.DataFrame(Y, columns=['y'])\n"," probs = pd.DataFrame(probs, columns=['probs'])\n","\n"," return pd.concat([X, A, Y, probs], axis=1)\n"," else:\n"," return X, A, Y, probs"],"metadata":{"id":"2bxVP6RGtxDa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# dataset = generate_samples(100000, 4, 3, True)\n","# dataset.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"UH5mKo3htyWH","executionInfo":{"status":"ok","timestamp":1639146016409,"user_tz":-330,"elapsed":4932,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"fca14966-0540-4ed6-e390-3ad54fb80465"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
X_1X_2X_3X_4ayprobs
013000.00.00.266661
131311.00.00.236514
230011.00.00.236514
303101.00.00.236514
421200.00.00.266661
\n","
"],"text/plain":[" X_1 X_2 X_3 X_4 a y probs\n","0 1 3 0 0 0.0 0.0 0.266661\n","1 3 1 3 1 1.0 0.0 0.236514\n","2 3 0 0 1 1.0 0.0 0.236514\n","3 0 3 1 0 1.0 0.0 0.236514\n","4 2 1 2 0 0.0 0.0 0.266661"]},"metadata":{},"execution_count":20}]},{"cell_type":"markdown","source":["### Data loader"],"metadata":{"id":"PYr5NIxatzVv"}},{"cell_type":"code","source":["def data_randomizer(pickle_file, seed=None):\n"," if isinstance(pickle_file, str):\n"," with open(pickle_file, 'rb') as f:\n"," dataset = pickle.load(f)\n"," else:\n"," dataset = pickle_file\n","\n"," actions = sorted(dataset.iloc[:, -3].unique().tolist())\n"," tst_smpl = pd.DataFrame().reindex_like(dataset).dropna()\n"," ratio = 0.1\n","\n"," for action in actions:\n"," action_subsample = dataset[dataset.iloc[:, -3] == action]\n"," action_drop, action_use = train_test_split(action_subsample.index, test_size=ratio,\n"," random_state=seed,\n"," stratify=action_subsample.iloc[:, -2])\n"," tst_smpl = pd.concat([tst_smpl,\n"," action_subsample.loc[action_use]]).sample(frac=1, random_state=seed)\n","\n"," tst_smpl = tst_smpl.reset_index(drop=True)\n","\n"," del action_drop, action_use\n","\n"," X = tst_smpl.iloc[:, :-3].to_numpy()\n"," A = tst_smpl.iloc[:, -3].to_numpy()\n"," Y = tst_smpl.iloc[:, -2].to_numpy()\n"," probs = tst_smpl.iloc[:, -1].to_numpy()\n","\n"," return X, A, Y/probs"],"metadata":{"id":"3sW2YJahuCbl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class BanditDataset(Dataset):\n"," def __init__(self, pickle_file, seed=None):\n"," # load dataset\n"," X, A, Y = data_randomizer(pickle_file, seed)\n"," self.features = X\n"," self.actions = A\n"," self.rewards = Y\n","\n"," def __len__(self):\n"," return len(self.rewards)\n","\n"," def __getitem__(self, idx):\n"," feature_vec = self.features[idx]\n"," action = self.actions[idx]\n"," reward = self.rewards[idx]\n","\n"," return feature_vec, action, reward"],"metadata":{"id":"NcdcUSeruEUD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","# dir = 'data/mushroom_data_final.pickle'\n","# data = data_randomizer(dir)\n","\n","# dataset = BanditDataset(pickle_file=dir, seed=1)\n","# print(len(dataset))\n","# print(dataset.__len__())\n","# print(dataset[420])\n","# print(dataset[421])\n","# print(dataset[0])\n","# print(dataset[1])\n","\n","# dl = DataLoader(dataset, batch_size=2, shuffle=True)\n","\n","# print(next(iter(dl)))\n","\n","# dataset = generate_samples(100000, 4, 3, True)\n","# dataset = BanditDataset(pickle_file=dataset, seed=1)\n","# print(len(dataset))\n","# print(dataset.__len__())\n","# print(dataset[420])\n","# print(dataset[421])\n","# print(dataset[0])\n","# print(dataset[1])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8v7s0-QIuGPR","executionInfo":{"status":"ok","timestamp":1639146132864,"user_tz":-330,"elapsed":5534,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"dada2526-7ef3-476c-e6f7-141d7f62e7c0"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["3251\n","3251\n","(array([5., 0., 0., 1., 0.]), 1.0, 1.0)\n","(array([7., 1., 0., 1., 0.]), 0.0, 0.0)\n","(array([3., 0., 0., 0., 1.]), 0.0, 0.0)\n","(array([7., 1., 0., 1., 0.]), 1.0, 1.0)\n","[tensor([[5., 0., 0., 1., 0.],\n"," [5., 0., 1., 1., 1.]], dtype=torch.float64), tensor([0., 0.], dtype=torch.float64), tensor([0., 0.], dtype=torch.float64)]\n","10001\n","10001\n","(array([1., 1., 1., 1.]), 2.0, 0.0)\n","(array([1., 3., 3., 3.]), 2.0, 0.0)\n","(array([3., 3., 0., 3.]), 2.0, 0.0)\n","(array([2., 1., 0., 1.]), 1.0, 0.0)\n"]}]},{"cell_type":"markdown","source":["### Plot script"],"metadata":{"id":"IsZ9fTsiuPwc"}},{"cell_type":"code","source":["def get_leveled_data(arr):\n"," \"\"\"\n"," Args:\n"," arr: list of lists os different length\n"," Returns:\n"," average result over arr, axis=0\n"," \"\"\"\n"," b = np.zeros([len(arr), len(max(arr, key=lambda x: len(x)))])\n"," b[:, :] = np.nan\n"," for i, j in enumerate(arr):\n"," b[i][0:len(j)] = j\n","\n"," return b"],"metadata":{"id":"Whuf3fm3uUdS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def smooth(data, k):\n"," num_episodes = data.shape[1]\n"," num_runs = data.shape[0]\n","\n"," smoothed_data = np.zeros((num_runs, num_episodes))\n","\n"," for i in range(num_episodes):\n"," if i < k:\n"," smoothed_data[:, i] = np.mean(data[:, :i + 1], axis=1)\n"," else:\n"," smoothed_data[:, i] = np.mean(data[:, i - k:i + 1], axis=1)\n","\n"," return smoothed_data"],"metadata":{"id":"eDcvasCOubnj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_result(result_batch, result_online, batch_size):\n"," plt_agent_sweeps = []\n"," num_steps = np.inf\n","\n"," fig, ax = plt.subplots(figsize=(8, 6))\n","\n"," for data, label in zip([result_batch, result_online], ['batch', 'online']):\n"," sum_reward_data = get_leveled_data(data)\n","\n"," # smooth data\n"," smoothed_sum_reward = smooth(data=sum_reward_data, k=100)\n","\n"," mean_smoothed_sum_reward = np.mean(smoothed_sum_reward, axis=0)\n","\n"," if mean_smoothed_sum_reward.shape[0] < num_steps:\n"," num_steps = mean_smoothed_sum_reward.shape[0]\n","\n"," plot_x_range = np.arange(0, mean_smoothed_sum_reward.shape[0])\n"," graph_current_agent_sum_reward, = ax.plot(plot_x_range, mean_smoothed_sum_reward[:],\n"," label=label)\n"," plt_agent_sweeps.append(graph_current_agent_sum_reward)\n","\n","\n"," update_points = np.ceil(np.arange(num_steps) / batch_size).astype(int)\n"," ax.plot(plot_x_range, mean_smoothed_sum_reward[update_points], label='upper bound')\n","\n"," ax.legend(handles=plt_agent_sweeps, fontsize=13)\n"," ax.set_title(\"Learning Curve\", fontsize=15)\n"," ax.set_xlabel('Episodes', fontsize=14)\n"," ax.set_ylabel('reward', rotation=0, labelpad=40, fontsize=14)\n"," # ax.set_ylim([-300, 300])\n","\n"," plt.tight_layout()\n"," plt.show()"],"metadata":{"id":"6DI-dp6fuawY"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Agents"],"metadata":{"id":"ndaq3Ha6ucgA"}},{"cell_type":"markdown","source":["### Base Agent"],"metadata":{"id":"nK6ccVZfukKl"}},{"cell_type":"markdown","source":["An abstract class that specifies the Agent API for RL-Glue-py."],"metadata":{"id":"5vxRMmXluqMk"}},{"cell_type":"code","source":["class BaseAgent:\n"," \"\"\"Implements the agent for an RL-Glue environment.\n"," Note:\n"," agent_init, agent_start, agent_step, agent_end, agent_cleanup, and\n"," agent_message are required methods.\n"," \"\"\"\n","\n"," __metaclass__ = ABCMeta\n","\n"," def __init__(self):\n"," pass\n","\n"," @abstractmethod\n"," def agent_init(self, agent_info={}):\n"," \"\"\"Setup for the agent called when the experiment first starts.\"\"\"\n","\n"," @abstractmethod\n"," def agent_start(self, observation):\n"," \"\"\"The first method called when the experiment starts, called after\n"," the environment starts.\n"," Args:\n"," observation (Numpy array): the state observation from the environment's evn_start function.\n"," Returns:\n"," The first action the agent takes.\n"," \"\"\"\n","\n"," @abstractmethod\n"," def agent_step(self, reward, observation):\n"," \"\"\"A step taken by the agent.\n"," Args:\n"," reward (float): the reward received for taking the last action taken\n"," observation (Numpy array): the state observation from the\n"," environment's step based, where the agent ended up after the\n"," last step\n"," Returns:\n"," The action the agent is taking.\n"," \"\"\"\n","\n"," @abstractmethod\n"," def agent_end(self, reward):\n"," \"\"\"Run when the agent terminates.\n"," Args:\n"," reward (float): the reward the agent received for entering the terminal state.\n"," \"\"\"\n","\n"," @abstractmethod\n"," def agent_cleanup(self):\n"," \"\"\"Cleanup done after the agent ends.\"\"\"\n","\n"," @abstractmethod\n"," def agent_message(self, message):\n"," \"\"\"A function used to pass information from the agent to the experiment.\n"," Args:\n"," message: The message passed to the agent.\n"," Returns:\n"," The response (or answer) to the message.\n"," \"\"\""],"metadata":{"id":"tIP4FINouoP1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Random Agent"],"metadata":{"id":"aJdhotjJuyCg"}},{"cell_type":"code","source":["class RandomAgent(BaseAgent):\n"," def __init__(self):\n"," super().__init__()\n"," self.num_actions = None\n","\n"," def agent_init(self, agent_info=None):\n"," if agent_info is None:\n"," agent_info = {}\n"," self.num_actions = agent_info.get('num_actions', 2)\n","\n"," def agent_start(self, observation):\n"," pass\n","\n"," def agent_step(self, reward, observation):\n"," pass\n","\n"," def agent_end(self, reward):\n"," pass\n","\n"," def agent_cleanup(self):\n"," pass\n","\n"," def agent_message(self, message):\n"," pass\n","\n"," def agent_policy(self, observation):\n"," return np.random.choice(self.num_actions)"],"metadata":{"id":"_nofgAnQu3em"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","# ag = RandomAgent()\n","# print(ag.num_actions)\n","\n","# ag.agent_init()\n","# print(ag.num_actions)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HwKhrIoVu5M3","executionInfo":{"status":"ok","timestamp":1639146301441,"user_tz":-330,"elapsed":6,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"57147a1f-d407-49ad-8e62-4d6c9d90922e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["None\n","2\n"]}]},{"cell_type":"code","source":["class Agent(BaseAgent):\n"," \"\"\"agent does *no* learning, selects random action always\"\"\"\n","\n"," def __init__(self):\n"," super().__init__()\n"," self.arm_count = None\n"," self.last_action = None\n"," self.num_actions = None\n"," self.q_values = None\n"," self.step_size = None\n"," self.initial_value = 0.0\n"," self.batch_size = None\n"," self.q_values_oracle = None # used for batch updates\n","\n"," def agent_init(self, agent_info=None):\n"," \"\"\"Setup for the agent called when the experiment first starts.\"\"\"\n","\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," self.num_actions = agent_info.get(\"num_actions\", 2)\n"," self.initial_value = agent_info.get(\"initial_value\", 0.0)\n"," self.q_values = np.ones(agent_info.get(\"num_actions\", 2)) * self.initial_value\n"," self.step_size = agent_info.get(\"step_size\", 0.1)\n"," self.batch_size = agent_info.get('batch_size', 1)\n"," self.q_values_oracle = self.q_values.copy()\n"," self.arm_count = np.zeros(self.num_actions) # [0.0 for _ in range(self.num_actions)]\n"," # self.last_action = np.random.choice(self.num_actions) # set first action to random\n","\n"," def agent_start(self, observation):\n"," \"\"\"The first method called when the experiment starts, called after\n"," the environment starts.\n"," Args:\n"," observation (Numpy array): the state observation from the\n"," environment's evn_start function.\n"," Returns:\n"," The first action the agent takes.\n"," \"\"\"\n"," self.last_action = np.random.choice(self.num_actions)\n","\n"," return self.last_action\n","\n"," def agent_step(self, reward, observation):\n"," \"\"\"A step taken by the agent.\n"," Args:\n"," reward (float): the reward received for taking the last action taken\n"," observation (Numpy array): the state observation from the\n"," environment's step based, where the agent ended up after the\n"," last step\n"," Returns:\n"," The action the agent is taking.\n"," \"\"\"\n"," # local_action = 0 # choose the action here\n"," self.last_action = np.random.choice(self.num_actions)\n","\n"," return self.last_action\n","\n"," def agent_end(self, reward):\n"," pass\n","\n"," def agent_cleanup(self):\n"," pass\n","\n"," def agent_message(self, message):\n"," pass"],"metadata":{"id":"egiYidMHvP7D"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def argmax(q_values):\n"," \"\"\"\n"," Takes in a list of q_values and returns the index of the item\n"," with the highest value. Breaks ties randomly.\n"," returns: int - the index of the highest value in q_values\n"," \"\"\"\n"," top_value = float(\"-inf\")\n"," ties = []\n","\n"," for i in range(len(q_values)):\n"," if q_values[i] > top_value:\n"," ties = [i]\n"," top_value = q_values[i]\n"," elif q_values[i] == top_value:\n"," ties.append(i)\n","\n"," return np.random.choice(ties)"],"metadata":{"id":"5w620FFCvTu8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Greedy Agent"],"metadata":{"id":"F_7QyQIsvraK"}},{"cell_type":"code","source":["class GreedyAgent(Agent):\n"," def __init__(self):\n"," super().__init__()\n","\n"," def agent_init(self, agent_info=None):\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," super().agent_init(agent_info)\n","\n"," def agent_step(self, reward, observation):\n"," \"\"\"\n"," Takes one step for the agent. It takes in a reward and observation and\n"," returns the action the agent chooses at that time step.\n"," Arguments:\n"," reward -- float, the reward the agent received from the environment after taking the last action.\n"," observation -- float, the observed state the agent is in. Do not worry about this as you will not use it\n"," until future lessons\n"," Returns:\n"," current_action -- int, the action chosen by the agent at the current time step.\n"," \"\"\"\n","\n"," a = self.last_action\n"," self.arm_count[a] += 1\n"," self.q_values_oracle[a] = self.q_values_oracle[a] + 1 / self.arm_count[a] * (reward - self.q_values_oracle[a])\n","\n"," if sum(self.arm_count) % self.batch_size == 0:\n"," self.q_values = self.q_values_oracle.copy()\n","\n"," current_action = argmax(self.q_values)\n"," self.last_action = current_action\n","\n"," return current_action"],"metadata":{"id":"my_xM4bpvVDf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### ϵ-Greedy Agent"],"metadata":{"id":"6SzDe38oviFg"}},{"cell_type":"code","source":["class EpsilonGreedyAgent(Agent):\n"," def __init__(self):\n"," super().__init__()\n"," self.epsilon = None\n","\n"," def agent_init(self, agent_info=None):\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," super().agent_init(agent_info)\n"," self.epsilon = agent_info.get(\"epsilon\", 0.1)\n","\n"," def agent_step(self, reward, observation):\n"," \"\"\"\n"," Takes one step for the agent. It takes in a reward and observation and\n"," returns the action the agent chooses at that time step.\n"," Arguments:\n"," reward -- float, the reward the agent received from the environment after taking the last action.\n"," observation -- float, the observed state the agent is in. Do not worry about this as you will not use it\n"," until future lessons\n"," Returns:\n"," current_action -- int, the action chosen by the agent at the current time step.\n"," \"\"\"\n","\n"," a = self.last_action\n","\n"," self.arm_count[a] += 1\n"," self.q_values_oracle[a] = self.q_values_oracle[a] + 1 / self.arm_count[a] * (reward - self.q_values_oracle[a])\n","\n"," if np.sum(self.arm_count) % self.batch_size == 0:\n"," self.q_values = self.q_values_oracle.copy()\n","\n"," if np.random.random() < self.epsilon:\n"," current_action = np.random.choice(range(len(self.arm_count)))\n"," else:\n"," current_action = argmax(self.q_values)\n","\n"," self.last_action = current_action\n","\n"," return current_action"],"metadata":{"id":"bPTYzIzTvWYZ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### UCB Agent"],"metadata":{"id":"Y635PDE6vgdI"}},{"cell_type":"code","source":["class UCBAgent(Agent):\n"," def __init__(self):\n"," super().__init__()\n"," self.upper_bounds = None\n"," self.alpha = None # exploration parameter\n","\n"," def agent_init(self, agent_info=None):\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," super().agent_init(agent_info)\n"," self.alpha = agent_info.get(\"alpha\", 1.0)\n"," self.arm_count = np.ones(self.num_actions)\n"," self.upper_bounds = np.sqrt(np.log(np.sum(self.arm_count)) / self.arm_count)\n","\n"," def agent_step(self, reward, observation):\n"," a = self.last_action\n","\n"," self.arm_count[a] += 1\n"," self.q_values_oracle[a] = self.q_values_oracle[a] + 1 / self.arm_count[a] * (reward - self.q_values_oracle[a])\n","\n"," # since we start with arms_count = np.ones(num_actions),\n"," # we should subtract num_actions to get number of the current round\n"," if (np.sum(self.arm_count) - self.num_actions) % self.batch_size == 0:\n"," self.q_values = self.q_values_oracle.copy()\n"," self.upper_bounds = np.sqrt(np.log(np.sum(self.arm_count)) / self.arm_count)\n","\n"," # if min(self.q_values + self.alpha * self.upper_bounds) < max(self.q_values):\n"," # print(f'Distinguish suboptimal arm at step {sum(self.arm_count)}')\n"," current_action = argmax(self.q_values + self.alpha * self.upper_bounds)\n"," # current_action = np.argmax(self.q_values + self.alpha * self.upper_bounds)\n","\n"," self.last_action = current_action\n","\n"," return current_action"],"metadata":{"id":"PV_TaTk1vX0T"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### TS Agent"],"metadata":{"id":"UiTTsyhFvdwc"}},{"cell_type":"code","source":["class TSAgent(Agent):\n"," def agent_step(self, reward, observation):\n"," a = self.last_action\n"," self.arm_count[a] += 1\n"," self.q_values_oracle[a] = self.q_values_oracle[a] + 1 / self.arm_count[a] * (reward - self.q_values_oracle[a])\n","\n"," if (np.sum(self.arm_count) - self.num_actions) % self.batch_size == 0:\n"," self.q_values = self.q_values_oracle.copy()\n","\n"," # sample from posteriors\n"," theta = [beta.rvs(a + 1, b + 1, size=1) for a, b in\n"," zip(self.q_values * self.arm_count, self.arm_count - self.q_values * self.arm_count)]\n"," # choose the max realization\n"," current_action = argmax(theta)\n"," self.last_action = current_action\n","\n"," return current_action"],"metadata":{"id":"TT3oZeNavY4e"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### LinUCB Agent"],"metadata":{"id":"-IL38-vIzIG1"}},{"cell_type":"code","source":["class LinUCBAgent(BaseAgent):\n","\n"," def __init__(self):\n"," super().__init__()\n"," self.name = \"LinUCB\"\n","\n"," def agent_init(self, agent_info=None):\n","\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," self.num_actions = agent_info.get('num_actions', 3)\n"," self.alpha = agent_info.get('alpha', 1)\n"," self.batch_size = agent_info.get('batch_size', 1)\n"," # Set random seed for policy for each run\n"," self.policy_rand_generator = np.random.RandomState(agent_info.get(\"seed\", None))\n","\n"," self.last_action = None\n"," self.last_state = None\n"," self.num_round = None\n","\n"," def agent_policy(self, observation):\n"," p_t = np.zeros(self.num_actions)\n","\n"," for i in range(self.num_actions):\n"," # initialize theta hat\n"," self.theta = inv(self.A[i]).dot(self.b[i])\n"," # get context of each arm from flattened vector of length 100\n"," cntx = observation\n"," # get gain reward of each arm\n"," p_t[i] = self.theta.T.dot(cntx) + self.alpha * np.sqrt(np.maximum(cntx.dot(inv(self.A[i]).dot(cntx)), 0))\n"," # action = np.random.choice(np.where(p_t == max(p_t))[0])\n"," action = self.policy_rand_generator.choice(np.where(p_t == max(p_t))[0])\n","\n"," return action\n","\n"," def agent_start(self, observation):\n"," # Specify feature dimension\n"," self.ndims = len(observation)\n","\n"," self.A = np.zeros((self.num_actions, self.ndims, self.ndims))\n"," # Instantiate b as a 0 vector of length ndims.\n"," self.b = np.zeros((self.num_actions, self.ndims, 1))\n"," # set each A per arm as identity matrix of size ndims\n"," for arm in range(self.num_actions):\n"," self.A[arm] = np.eye(self.ndims)\n","\n"," self.A_oracle = self.A.copy()\n"," self.b_oracle = self.b.copy()\n","\n"," self.last_state = observation\n"," self.last_action = self.agent_policy(self.last_state)\n"," self.num_round = 0\n","\n"," return self.last_action\n","\n"," def agent_update(self, reward):\n"," self.A_oracle[self.last_action] = self.A_oracle[self.last_action] + np.outer(self.last_state, self.last_state)\n"," self.b_oracle[self.last_action] = np.add(self.b_oracle[self.last_action].T, self.last_state * reward).reshape(self.ndims, 1)\n","\n"," def agent_step(self, reward, observation):\n"," if reward is not None:\n"," self.agent_update(reward)\n"," # it is a good question whether I should increment num_round outside\n"," # condition or not (since theoretical result doesn't clarify this\n"," self.num_round += 1\n","\n"," if self.num_round % self.batch_size == 0:\n"," self.A = self.A_oracle.copy()\n"," self.b = self.b_oracle.copy()\n","\n"," self.last_state = observation\n"," self.last_action = self.agent_policy(self.last_state)\n","\n"," return self.last_action\n","\n"," def agent_end(self, reward):\n"," if reward is not None:\n"," self.agent_update(reward)\n"," self.num_round += 1\n","\n"," if self.num_round % self.batch_size == 0:\n"," self.A = self.A_oracle.copy()\n"," self.b = self.b_oracle.copy()\n","\n"," def agent_message(self, message):\n"," pass\n","\n"," def agent_cleanup(self):\n"," pass"],"metadata":{"id":"gyXQwM2-zKR_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","# agent_info = {'alpha': 2,\n","# 'num_actions': 4,\n","# 'seed': 1}\n","\n","# # check initialization\n","# linucb = LinUCBAgent()\n","# linucb.agent_init(agent_info)\n","# print(linucb.num_actions, linucb.alpha)\n","\n","# assert linucb.num_actions == 4\n","# assert linucb.alpha == 2\n","\n","# # check policy\n","# observation = np.array([1, 2, 5, 0])\n","# linucb.A = np.zeros((linucb.num_actions, len(observation), len(observation)))\n","# # Instantiate b as a 0 vector of length ndims.\n","# linucb.b = np.zeros((linucb.num_actions, len(observation), 1))\n","# # set each A per arm as identity matrix of size ndims\n","# for arm in range(linucb.num_actions):\n","# linucb.A[arm] = np.eye(len(observation))\n","\n","# action = linucb.agent_policy(observation)\n","# print(action)\n","\n","# assert action == 1\n","\n","# # check start\n","# observation = np.array([1, 2, 5, 0])\n","# linucb.agent_start(observation)\n","# print(linucb.ndims)\n","# print(linucb.last_state, linucb.last_action)\n","\n","# assert linucb.ndims == len(observation)\n","# assert np.allclose(linucb.last_state, observation)\n","# assert np.allclose(linucb.b, np.zeros((linucb.num_actions, len(observation), 1)))\n","# assert np.allclose(linucb.A, np.array([np.eye(len(observation)), np.eye(len(observation)),\n","# np.eye(len(observation)), np.eye(len(observation))]))\n","# assert linucb.last_action == 3\n","\n","# # check step\n","# observation = np.array([5, 3, 1, 2])\n","# reward = 1\n","\n","# action = linucb.agent_step(reward, observation)\n","# print(linucb.A)\n","# print(linucb.b)\n","# print(action)\n","\n","# true_A = np.array([[2., 2., 5., 0.],\n","# [2., 5., 10., 0.],\n","# [5., 10., 26., 0.],\n","# [0., 0., 0., 1.]])\n","\n","# true_b = np.array([[1.],\n","# [2.],\n","# [5.],\n","# [0.]])\n","\n","# for i in range(3):\n","# assert np.allclose(linucb.A[i], np.eye(4))\n","# assert np.allclose(linucb.b[i], np.zeros((linucb.num_actions, 4, 1)))\n","# assert np.allclose(linucb.A[3], true_A)\n","# assert np.allclose(linucb.b[3], true_b)\n","# assert linucb.last_action == 0\n","\n","# observation = np.array([3, 1, 3, 5])\n","# reward = None\n","\n","# action = linucb.agent_step(reward, observation)\n","# print(linucb.A)\n","# print(linucb.b)\n","# print(action)\n","\n","# assert np.allclose(linucb.A[3], true_A)\n","# assert np.allclose(linucb.b[3], true_b)\n","# assert action == 0\n","\n","# # check batch size\n","# agent_info = {'alpha': 2,\n","# 'num_actions': 4,\n","# 'seed': 1,\n","# 'batch_size': 2}\n","# linucb = LinUCBAgent()\n","# linucb.agent_init(agent_info)\n","# observation = np.array([1, 2, 5, 0])\n","# linucb.agent_start(observation)\n","# assert linucb.num_round == 0\n","# assert linucb.last_action == 1\n","\n","# observation = np.array([5, 3, 1, 2])\n","# reward = 1\n","\n","# action = linucb.agent_step(reward, observation)\n","# assert linucb.num_round == 1\n","# assert np.allclose(linucb.b, np.zeros((linucb.num_actions, len(observation), 1)))\n","# assert np.allclose(linucb.A, np.array([np.eye(len(observation)), np.eye(len(observation)),\n","# np.eye(len(observation)), np.eye(len(observation))]))\n","\n","# for i in [0, 2, 3]:\n","# assert np.allclose(linucb.A_oracle[i], np.eye(4))\n","# assert np.allclose(linucb.b_oracle[i], np.zeros((linucb.num_actions, 4, 1)))\n","# assert np.allclose(linucb.A_oracle[1], true_A)\n","# assert np.allclose(linucb.b_oracle[1], true_b)\n","\n","# observation = np.array([3, 1, 3, 5])\n","# reward = None\n","# action = linucb.agent_step(reward, observation)\n","# # sinse reward is None, nothing should happen\n","# assert linucb.num_round == 1\n","# assert np.allclose(linucb.b, np.zeros((linucb.num_actions, len(observation), 1)))\n","# assert np.allclose(linucb.A, np.array([np.eye(len(observation)), np.eye(len(observation)),\n","# np.eye(len(observation)), np.eye(len(observation))]))\n","\n","# for i in [0, 2, 3]:\n","# assert np.allclose(linucb.A_oracle[i], np.eye(4))\n","# assert np.allclose(linucb.b_oracle[i], np.zeros((linucb.num_actions, 4, 1)))\n","# assert np.allclose(linucb.A_oracle[1], true_A)\n","# assert np.allclose(linucb.b_oracle[1], true_b)\n","\n","# observation = np.array([3, 0, 2, 5])\n","# reward = 0\n","# action = linucb.agent_step(reward, observation)\n","\n","# assert linucb.num_round == 2\n","# assert np.allclose(linucb.b, linucb.b_oracle)\n","# assert np.allclose(linucb.A, linucb.A_oracle)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Nkdq0ya9zNlE","executionInfo":{"status":"ok","timestamp":1639148410291,"user_tz":-330,"elapsed":5,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b7a9b9c5-e3d1-4fc8-bdc5-321172c6ebdb"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["4 2\n","1\n","4\n","[1 2 5 0] 3\n","[[[ 1. 0. 0. 0.]\n"," [ 0. 1. 0. 0.]\n"," [ 0. 0. 1. 0.]\n"," [ 0. 0. 0. 1.]]\n","\n"," [[ 1. 0. 0. 0.]\n"," [ 0. 1. 0. 0.]\n"," [ 0. 0. 1. 0.]\n"," [ 0. 0. 0. 1.]]\n","\n"," [[ 1. 0. 0. 0.]\n"," [ 0. 1. 0. 0.]\n"," [ 0. 0. 1. 0.]\n"," [ 0. 0. 0. 1.]]\n","\n"," [[ 2. 2. 5. 0.]\n"," [ 2. 5. 10. 0.]\n"," [ 5. 10. 26. 0.]\n"," [ 0. 0. 0. 1.]]]\n","[[[0.]\n"," [0.]\n"," [0.]\n"," [0.]]\n","\n"," [[0.]\n"," [0.]\n"," [0.]\n"," [0.]]\n","\n"," [[0.]\n"," [0.]\n"," [0.]\n"," [0.]]\n","\n"," [[1.]\n"," [2.]\n"," [5.]\n"," [0.]]]\n","0\n","[[[ 1. 0. 0. 0.]\n"," [ 0. 1. 0. 0.]\n"," [ 0. 0. 1. 0.]\n"," [ 0. 0. 0. 1.]]\n","\n"," [[ 1. 0. 0. 0.]\n"," [ 0. 1. 0. 0.]\n"," [ 0. 0. 1. 0.]\n"," [ 0. 0. 0. 1.]]\n","\n"," [[ 1. 0. 0. 0.]\n"," [ 0. 1. 0. 0.]\n"," [ 0. 0. 1. 0.]\n"," [ 0. 0. 0. 1.]]\n","\n"," [[ 2. 2. 5. 0.]\n"," [ 2. 5. 10. 0.]\n"," [ 5. 10. 26. 0.]\n"," [ 0. 0. 0. 1.]]]\n","[[[0.]\n"," [0.]\n"," [0.]\n"," [0.]]\n","\n"," [[0.]\n"," [0.]\n"," [0.]\n"," [0.]]\n","\n"," [[0.]\n"," [0.]\n"," [0.]\n"," [0.]]\n","\n"," [[1.]\n"," [2.]\n"," [5.]\n"," [0.]]]\n","0\n"]}]},{"cell_type":"markdown","source":["### LinTS Agent"],"metadata":{"id":"4L_OJQQEvP4k"}},{"cell_type":"code","source":["class LinTSAgent(BaseAgent):\n","\n"," def __init__(self):\n"," super().__init__()\n"," self.name = \"LinTS\"\n","\n"," def agent_init(self, agent_info=None):\n","\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," self.num_actions = agent_info.get('num_actions', 3)\n"," self.alpha = agent_info.get('alpha', 1)\n"," self.lambda_ = agent_info.get('lambda', 1)\n"," self.batch_size = agent_info.get('batch_size', 1)\n"," # Set random seed for policy for each run\n"," self.policy_rand_generator = np.random.RandomState(agent_info.get(\"seed\", None))\n","\n"," self.replay_buffer = ReplayBuffer(agent_info['replay_buffer_size'],\n"," agent_info.get(\"seed\"))\n","\n","\n"," self.last_action = None\n"," self.last_state = None\n"," self.num_round = None\n","\n"," def agent_policy(self, observation, mode='sample'):\n"," p_t = np.zeros(self.num_actions)\n"," cntx = observation\n","\n"," for i in range(self.num_actions):\n"," # sampling weights after update\n"," self.w = self.get_weights(i)\n","\n"," # using weight depending on mode\n"," if mode == 'sample':\n"," w = self.w # weights are samples of posteriors\n"," elif mode == 'expected':\n"," w = self.m[i] # weights are expected values of posteriors\n"," else:\n"," raise Exception('mode not recognized!')\n","\n"," # calculating probabilities\n"," p_t[i] = 1 / (1 + np.exp(-1 * cntx.dot(w)))\n"," action = self.policy_rand_generator.choice(np.where(p_t == max(p_t))[0])\n"," # probs = softmax(p_t.reshape(1, -1))\n"," # action = self.policy_rand_generator.choice(a=range(self.num_actions), p=probs)\n","\n"," return action\n","\n"," def get_weights(self, arm):\n"," return np.random.normal(self.m[arm], self.alpha * self.q[arm] ** (-1.0), size=len(self.w))\n","\n"," # the loss function\n"," def loss(self, w, *args):\n"," X, y = args\n"," return 0.5 * (self.q[self.last_action] * (w - self.m[self.last_action])).dot(w - self.m[self.last_action]) + np.sum(\n"," [np.log(1 + np.exp(-y[j] * w.dot(X[j]))) for j in range(y.shape[0])])\n","\n"," # the gradient\n"," def grad(self, w, *args):\n"," X, y = args\n"," return self.q[self.last_action] * (w - self.m[self.last_action]) + (-1) * np.array(\n"," [y[j] * X[j] / (1. + np.exp(y[j] * w.dot(X[j]))) for j in range(y.shape[0])]).sum(axis=0)\n","\n"," # fitting method\n"," def agent_update(self, X, y):\n"," # step 1, find w\n"," self.w = minimize(self.loss, self.w, args=(X, y), jac=self.grad, method=\"L-BFGS-B\",\n"," options={'maxiter': 20, 'disp': False}).x\n"," # self.m_oracle[self.last_action] = self.w\n"," self.m[self.last_action] = self.w\n","\n"," # step 2, update q\n"," P = (1 + np.exp(1 - X.dot(self.m[self.last_action]))) ** (-1)\n"," #self.q_oracle[self.last_action] = self.q[self.last_action] + (P * (1 - P)).dot(X ** 2)\n"," self.q[self.last_action] = self.q[self.last_action] + (P * (1 - P)).dot(X ** 2)\n","\n"," def agent_start(self, observation):\n"," # Specify feature dimension\n"," self.ndims = len(observation)\n","\n"," # initializing parameters of the model\n"," self.m = np.zeros((self.num_actions, self.ndims))\n"," self.q = np.ones((self.num_actions, self.ndims)) * self.lambda_\n"," # initializing weights using any arm (e.g. 0) because they all equal\n"," self.w = np.array([0.]*self.ndims, dtype=np.float64)\n","\n"," # self.m_oracle = self.m.copy()\n"," # self.q_oracle = self.q.copy()\n","\n"," self.last_state = observation\n"," self.last_action = self.agent_policy(self.last_state)\n"," self.num_round = 0\n","\n"," return self.last_action\n","\n","\n"," def agent_step(self, reward, observation):\n"," # Append new experience to replay buffer\n"," if reward is not None:\n"," self.replay_buffer.append(self.last_state, self.last_action, reward)\n"," # it is a good question whether I should increment num_round outside\n"," # condition or not (since theoretical result doesn't clarify this\n"," self.num_round += 1\n","\n"," if self.num_round % self.batch_size == 0:\n"," X, y = self.replay_buffer.sample(self.last_action)\n"," X = np.array(X)\n"," y = np.array(y)\n"," self.agent_update(X, y)\n"," # self.m = self.m_oracle.copy()\n"," # self.q = self.q_oracle.copy()\n","\n"," self.last_state = observation\n"," self.last_action = self.agent_policy(self.last_state)\n","\n"," return self.last_action\n","\n"," def agent_end(self, reward):\n"," # Append new experience to replay buffer\n"," if reward is not None:\n"," self.replay_buffer.append(self.last_state, self.last_action, reward)\n"," # it is a good question whether I should increment num_round outside\n"," # condition or not (since theoretical result doesn't clarify this\n"," self.num_round += 1\n","\n"," if self.num_round % self.batch_size == 0:\n"," X, y = self.replay_buffer.sample(self.last_action)\n"," X = np.array(X)\n"," y = np.array(y)\n"," self.agent_update(X, y)\n"," # self.m = self.m_oracle.copy()\n"," # self.q = self.q_oracle.copy()\n","\n"," def agent_message(self, message):\n"," pass\n","\n"," def agent_cleanup(self):\n"," pass"],"metadata":{"id":"X7mSGLS4vP20"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","# agent_info = {'alpha': 2,\n","# 'num_actions': 3,\n","# 'seed': 1,\n","# 'lambda': 2,\n","# 'replay_buffer_size': 100000}\n","\n","# np.random.seed(1)\n","# # check initialization\n","# lints = LinTSAgent()\n","# lints.agent_init(agent_info)\n","# print(lints.num_actions, lints.alpha, lints.lambda_)\n","\n","# assert lints.num_actions == 3\n","# assert lints.alpha == 2\n","# assert lints.lambda_ == 2\n","\n","# # check agent policy\n","# observation = np.array([1, 2, 5, 0])\n","# lints.m = np.zeros((lints.num_actions, len(observation)))\n","# lints.q = np.ones((lints.num_actions, len(observation))) * lints.lambda_\n","# lints.w = np.random.normal(lints.m[0], lints.alpha * lints.q[0] ** (-1.0), size=len(observation))\n","# print(lints.w)\n","# action = lints.agent_policy(observation)\n","# print(action)\n","\n","# # check agent start\n","# observation = np.array([1, 2, 5, 0])\n","# lints.agent_start(observation)\n","# # manually reassign w to np.random.normal, because I np.seed doesn't work inside the class\n","# np.random.seed(1)\n","# lints.w = np.random.normal(lints.m[0], lints.alpha * lints.q[0] ** (-1.0), size=len(observation))\n","# print(lints.ndims)\n","# print(lints.last_state, lints.last_action)\n","# print(lints.last_action)\n","# assert lints.ndims == len(observation)\n","# assert np.allclose(lints.last_state, observation)\n","# assert np.allclose(lints.m, np.zeros((lints.num_actions, lints.ndims)))\n","# assert np.allclose(lints.q, np.ones((lints.num_actions, lints.ndims)) * lints.lambda_)\n","# assert np.allclose(lints.w, np.array([ 1.62434536, -0.61175641, -0.52817175, -1.07296862]))\n","# # assert lints.last_action == 1\n","\n","# # check step\n","# observation = np.array([5, 3, 1, 2])\n","# reward = 1\n","# action = lints.agent_step(reward, observation)\n","# print(action)\n","\n","# observation = np.array([1, 3, 2, 1])\n","# reward = 0\n","# action = lints.agent_step(reward, observation)\n","# print(action)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"yr9HfGeJy-vR","executionInfo":{"status":"ok","timestamp":1639148425016,"user_tz":-330,"elapsed":3,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"4037b3ed-73b8-4057-8ea7-c1cb4d7fd177"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["3 2 2\n","[ 1.62434536 -0.61175641 -0.52817175 -1.07296862]\n","1\n","4\n","[1 2 5 0] 1\n","1\n","1\n","1\n"]}]},{"cell_type":"markdown","source":["## Environments"],"metadata":{"id":"IQr77JFsu6Jq"}},{"cell_type":"markdown","source":["### Base Environment"],"metadata":{"id":"oAuUEthsu9TG"}},{"cell_type":"markdown","source":["Abstract environment base class for RL-Glue-py."],"metadata":{"id":"pEz4-17VvAZQ"}},{"cell_type":"code","source":["class BaseEnvironment:\n"," \"\"\"Implements the environment for an RLGlue environment\n"," Note:\n"," env_init, env_start, env_step, env_cleanup, and env_message are required\n"," methods.\n"," \"\"\"\n","\n"," __metaclass__ = ABCMeta\n","\n"," def __init__(self):\n"," reward = None\n"," observation = None\n"," termination = None\n"," self.reward_state_term = (reward, observation, termination)\n","\n"," @abstractmethod\n"," def env_init(self, env_info={}):\n"," \"\"\"Setup for the environment called when the experiment first starts.\n"," Note:\n"," Initialize a tuple with the reward, first state observation, boolean\n"," indicating if it's terminal.\n"," \"\"\"\n","\n"," @abstractmethod\n"," def env_start(self):\n"," \"\"\"The first method called when the experiment starts, called before the\n"," agent starts.\n"," Returns:\n"," The first state observation from the environment.\n"," \"\"\"\n","\n"," @abstractmethod\n"," def env_step(self, action):\n"," \"\"\"A step taken by the environment.\n"," Args:\n"," action: The action taken by the agent\n"," Returns:\n"," (float, state, Boolean): a tuple of the reward, state observation,\n"," and boolean indicating if it's terminal.\n"," \"\"\"\n","\n"," @abstractmethod\n"," def env_cleanup(self):\n"," \"\"\"Cleanup done after the environment ends\"\"\"\n","\n"," @abstractmethod\n"," def env_message(self, message):\n"," \"\"\"A message asking the environment for information\n"," Args:\n"," message: the message passed to the environment\n"," Returns:\n"," the response (or answer) to the message\n"," \"\"\""],"metadata":{"id":"qqZDb6A9u-85"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### k-arm Environment"],"metadata":{"id":"_Mm2zTKyv3Fb"}},{"cell_type":"code","source":["class Environment(BaseEnvironment):\n"," \"\"\"Implements the environment for an RLGlue environment\n"," Note:\n"," env_init, env_start, env_step, env_cleanup, and env_message are required\n"," methods.\n"," \"\"\"\n","\n"," actions = [0]\n","\n"," def __init__(self):\n"," super().__init__()\n"," reward = None\n"," observation = None\n"," termination = None\n"," self.seed = None\n"," self.k = None\n"," self.reward_type = None\n"," self.custom_arms = None\n"," self.reward_state_term = (reward, observation, termination)\n"," self.count = 0\n"," self.arms = []\n"," self.subopt_gaps = None\n","\n"," def env_init(self, env_info=None):\n"," \"\"\"Setup for the environment called when the experiment first starts.\n"," Note:\n"," Initialize a tuple with the reward, first state observation, boolean\n"," indicating if it's terminal.\n"," \"\"\"\n","\n"," if env_info is None:\n"," env_info = {}\n"," self.k = env_info.get(\"num_actions\", 2)\n"," self.reward_type = env_info.get(\"reward_type\", \"subgaussian\")\n"," self.custom_arms = env_info.get(\"arms_values\", None)\n","\n"," if self.reward_type not in ['Bernoulli', 'subgaussian']:\n"," raise ValueError('Unknown reward_type: ' + str(self.reward_type))\n","\n"," if self.custom_arms is None:\n"," if self.reward_type == 'Bernoulli':\n"," self.arms = np.random.uniform(0, 1, self.k)\n"," else:\n"," self.arms = np.random.randn(self.k)\n"," else:\n"," self.arms = self.custom_arms\n"," self.subopt_gaps = np.max(self.arms) - self.arms\n","\n"," local_observation = 0 # An empty NumPy array\n","\n"," self.reward_state_term = (0.0, local_observation, False)\n","\n"," def env_start(self):\n"," \"\"\"The first method called when the experiment starts, called before the\n"," agent starts.\n"," Returns:\n"," The first state observation from the environment.\n"," \"\"\"\n","\n"," return self.reward_state_term[1]\n","\n"," def env_step(self, action):\n"," \"\"\"A step taken by the environment.\n"," Args:\n"," action: The action taken by the agent\n"," Returns:\n"," (float, state, Boolean): a tuple of the reward, state observation,\n"," and boolean indicating if it's terminal.\n"," \"\"\"\n"," if self.reward_type == 'Bernoulli':\n"," reward = np.random.binomial(1, self.arms[action], 1)\n"," else:\n"," reward = self.arms[action] + np.random.randn()\n"," obs = self.reward_state_term[1]\n","\n"," self.reward_state_term = (reward, obs, False)\n","\n"," return self.reward_state_term\n","\n"," def env_cleanup(self):\n"," \"\"\"Cleanup done after the environment ends\"\"\"\n"," pass\n","\n"," def env_message(self, message):\n"," \"\"\"A message asking the environment for information\n"," Args:\n"," message (string): the message passed to the environment\n"," Returns:\n"," string: the response (or answer) to the message\n"," \"\"\"\n"," if message == \"what is the current reward?\":\n"," return \"{}\".format(self.reward_state_term[0])\n","\n"," # else\n"," return \"I don't know how to respond to your message\""],"metadata":{"id":"YLTkxvLGvC4g"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Offline Evaluator"],"metadata":{"id":"naYFRhirzYfu"}},{"cell_type":"code","source":["class OfflineEvaluator:\n"," def __init__(self, eval_info=None):\n","\n"," if eval_info is None:\n"," eval_info = {}\n","\n"," self.dataset = eval_info['dataset']\n"," self.agent = eval_info['agent']\n","\n"," if not isinstance(self.dataset, Dataset):\n"," raise TypeError('dataset ' + \"must be a \" + str(Dataset))\n"," if not isinstance(self.agent, BaseAgent):\n"," raise TypeError('agent ' + \"must be a \" + str(BaseAgent))\n","\n"," self.total_reward = None\n"," self.average_reward = None\n"," self.num_matches = None\n"," self.idxs = range(self.dataset.__len__())\n"," self.counter = None\n","\n"," def eval_start(self):\n"," self.total_reward = 0\n"," self.average_reward = [0]\n"," self.num_matches = 0\n"," self.idxs = range(self.dataset.__len__())\n"," self.counter = 0\n","\n"," def _get_observation(self):\n"," idx = self.idxs[self.counter]\n"," self.counter += 1\n","\n"," return self.dataset.__getitem__(idx)\n","\n"," def eval_step(self):\n"," observation = self._get_observation()\n","\n"," state = observation[0]\n"," true_action = observation[1]\n"," reward = observation[2]\n","\n"," pred_action = self.agent.agent_policy(state)\n","\n"," if true_action != pred_action:\n"," return\n","\n"," self.num_matches += 1\n"," aw_reward = self.average_reward[-1] + (reward - self.average_reward[-1]) / self.num_matches\n"," self.average_reward.append(aw_reward)\n"," self.total_reward += reward\n","\n"," def eval_run(self):\n"," self.eval_start()\n","\n"," while self.counter < self.dataset.__len__():\n"," self.eval_step()\n","\n"," return self.average_reward"],"metadata":{"id":"iJLVLYunzbPe"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","\n","# dir1 = 'data/mushroom_data_final.pickle'\n","\n","# ra = RandomAgent()\n","# agent_info = {'num_actions': 2}\n","# ra.agent_init(agent_info)\n","\n","# result = []\n","# result1 = []\n","\n","# for seed_ in [1, 5, 10]: # , 2, 3, 32, 123, 76, 987, 2134]:\n","# dataset = BanditDataset(pickle_file=dir1, seed=seed_)\n","\n","# eval_info = {'dataset': dataset, 'agent': ra}\n","# evaluator = OfflineEvaluator(eval_info)\n","\n","# reward = evaluator.eval_run()\n","\n","# result.append(reward)\n","# result1.append(evaluator.total_reward)\n","\n","# for elem in result:\n","# plt.plot(elem)\n","# plt.legend()\n","# plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":282},"id":"BY8sASekzc6S","executionInfo":{"status":"ok","timestamp":1639148441141,"user_tz":-330,"elapsed":14,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"1ddaab3a-5b2f-41a8-842f-407aaa3b1f62"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["No handles with labels found to put in legend.\n"]},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcdb3/8ddn9ux703RNV+nCHkorWLayKhR3CkpRtFe9uFxFL4p6FfxdFy5XrlcUyyKLyCIIVG+VTWQvELaWUrq3NG3SpEmafTLb5/fHmSSTNEmbZtqkM5/n49HmnO/5zvl+52Tynu9858wZUVWMMcakPtdId8AYY8zhYYFvjDFpwgLfGGPShAW+McakCQt8Y4xJE56R7sBAiouLtby8fKS7YYwxR5TXX399j6qW9Ldt1AZ+eXk5lZWVI90NY4w5oojI9oG22ZSOMcakCQt8Y4xJExb4xhiTJkbtHL4xxqS7cDhMVVUVwWBwn22BQIAJEybg9XoPeH8W+MYYM0pVVVWRk5NDeXk5ItJdrqrU19dTVVXFlClTDnh/SZnSEZE7RKRWRN4ZYLuIyK9EZJOIrBaRE5LRrjHGpLJgMEhRUVGvsAcQEYqKivod+Q8mWXP4dwLnDbL9fGBG/N8y4LdJatcYY1Ja37DfX/lgkhL4qvoc0DBIlcXA3epYBeSLSFky2u7P7ZWP89L29w7V7o0x5oh0uObwxwM7Etar4mXViZVEZBnOKwAmTZp00I3dtPZqANYsXXPQ+zDGmFQzqk7LVNXlqlqhqhUlJf1+MtgYY9LKQF9SdTBfXnW4An8nMDFhfUK8zBhjzAACgQD19fX7hHvXWTqBQGBI+ztcUzorgKtE5H7gZKBJVav3cxtjjElrEyZMoKqqirq6un22dZ2HPxRJCXwRuQ84HSgWkSrgPwAvgKreAqwELgA2Ae3A55LRrjHGpDKv1zuk8+z3JymBr6pL9rNdgX9NRlvGGGMOzqh609YYY8yhY4FvjDFpwgLfGGPShAW+McakCQt8Y4xJExb4xhiTJizwjTEmTVjgG2NMmrDAN8aYNGGBb4wxacIC3xhj0oQFvjHGpAkLfGOMSRMW+MYYkyYs8I0xJk1Y4BtjTJqwwDfGmDSRlMAXkfNEZL2IbBKRa/rZPklEnhGRN0VktYhckIx2jTHGHLhhB76IuIGbgfOB2cASEZndp9r3gQdV9XjgEuA3w23XGGPM0CRjhD8P2KSqW1Q1BNwPLO5TR4Hc+HIesCsJ7RpjjBmCZHyJ+XhgR8J6FXBynzo/Ap4Qka8CWcCiJLRrjDFmCA7Xm7ZLgDtVdQJwAXCPiOzTtogsE5FKEamsq6s7TF0zxpj0kIzA3wlMTFifEC9LdCXwIICqvgwEgOK+O1LV5apaoaoVJSUlSeiaMcaYLskI/NeAGSIyRUR8OG/KruhT533gLAARmYUT+DaEN8aYw2jYga+qEeAq4HFgHc7ZOGtF5DoRuShe7VvAF0XkbeA+4ApV1eG2bYwx5sAl401bVHUlsLJP2Q8Tlt8FTklGW8YYYw5OSn/S9om1NSPdBWOMGTVSOvCX3fP6SHfBGGNGjZQOfIBXtzaMdBeMMWZUSPnA/9TvXua5DXZCkDHGpHzgA9S3dY50F4wxZsSlReBneJNyMpIxxhzR0iLwfR4Z6S4YY8yIS4vAD0XsM17GGJMegR+NjXQXjDFmxKVH4Ecs8I0xxgLfGGPSRJoEfnSku2CMMSMuPQLf5vCNMSZNAt+mdIwxxgLfGGPSRXoEftTOwzfGmPQIfBvhG2NMmgR+1M7SMcaYpAS+iJwnIutFZJOIXDNAnU+JyLsislZE/piMdg+UjfCNMSYJ32krIm7gZuBsoAp4TURWxL/HtqvODOC7wCmq2igiY4bb7lBY4BtjTHJG+POATaq6RVVDwP3A4j51vgjcrKqNAKpam4R2D5idh2+MMckJ/PHAjoT1qnhZopnATBF5UURWich5/e1IRJaJSKWIVNbVJedbqiYVZtoI3xhjOHxv2nqAGcDpwBLgVhHJ71tJVZeraoWqVpSUlAy70esWz6Ewy0enBb4xxiQl8HcCExPWJ8TLElUBK1Q1rKpbgQ04TwCHlIjg87hshG+MMSQn8F8DZojIFBHxAZcAK/rUeRRndI+IFONM8WxJQtuDcovg97hsDt8YY0hC4KtqBLgKeBxYBzyoqmtF5DoRuShe7XGgXkTeBZ4Bvq2q9cNte39cAj63jfCNMQaScFomgKquBFb2KfthwrIC34z/O2xcNqVjjDHdUvqTti5XPPBtSscYY1I78AWb0jHGmC4pHfguFzalY4wxcakd+DaHb4wx3VI+8L1uF+GYBb4xxqR84HtcQsS+AMUYY1I78N0unMCPKc6ZocYYk75SOvAXTC3G43buYjRmgW+MSW8pHfh5mV48bgEgYoFvjElzKR344EzpgAW+McakQeA7dzFin7Y1xqS5lA98b3xKJxi2wDfGpLeUD3x3fIQ//6dPj3BPjDFmZKV84CeyT9waY9JZygd+ScMb+AgD8N0/rxnh3hhjzMhJucCPJV5GYc8mzn5lKf/huRuAh9+oGqFeGWPMyEu5wO8l1ALAsa7NI9wRY4wZeUkJfBE5T0TWi8gmEblmkHofFxEVkYpktLtfax8FYFKuczeLs/2HpVljjBmNhh34IuIGbgbOB2YDS0Rkdj/1coCvA68Mt83BxBKvmfPiTQDkemOcO6eUgkzvoWzaGGNGtWSM8OcBm1R1i6qGgPuBxf3Uux74ORBMQptD07iNMTkBals6D3vTxhgzWiQj8McDOxLWq+Jl3UTkBGCiqv7fYDsSkWUiUikilXV1dUnoWo+5bKapI0wwHE3qfo0x5khxyN+0FREX8N/At/ZXV1WXq2qFqlaUlJQcVHuxAS6DPM7TBMDu5sP/AsMYY0aDZAT+TmBiwvqEeFmXHGAu8E8R2QbMB1Yctjdu44r8zumap93wz8PZrDHGjBrJCPzXgBkiMkVEfMAlwIqujarapKrFqlququXAKuAiVa1MQtsHrMgXOZzNGWPMqDPswFfVCHAV8DiwDnhQVdeKyHUictFw9z9UMe3/8gkF0tq9HLYrZxpj0pAnGTtR1ZXAyj5lPxyg7unJaHOofJGewN/R0M7UkuyR6IYxxoyYlPukbQxNWO4iEA3z0JcWALC9vv2w98sYY0ZaygV+ou4TMH1ZEA1RXpwFwNY9bSPWJ2OMGSkpHfgxiS+4fRANUZTlQwSu++u7dEbsfHxjTHpJucBPfM82Rjzx44EvInSdpr/irV2Hv3PGGDOCUi7wE3Vnv9sH0XCvbS6RfeobY0wqS+nAj3ZluscHu9cCcPmCyQD4vSl9140xZh8pl3qxhHNzuqd06jdBzWp49zGuPHUKAJ0H8qXm4SA02ZemGGNSQ8oFfqIowL9v6ynY9iKFWT4Adu7t2PcGu96E1Q/2rD/wGfjlHIhFIWYf1jLGHNlSOvBjAmQU9BS015MT8DJ9TDZNm1ZBx97eN7h7Mfz5i9BS46xvetL5eV0hXF8MA1yYzRhjjgQpF/iJV8uMdk3pBPKcn8G98NdvcnXgMX5QfRV7fn1279MzIyHn55t/2DfcNQrbX3KeJP50Bfz2VHjxf+C5Gw7dnTGDCzZBuAM6W5zf2ZP/0f1ejTFmX0m5tMJoFTv135yFT94J93wUfNlQeTvnxbcXt23gS/e9xS2fPdEJ+0h8mufdR+G4y5zlaWfC5n84y3de0LuBJ9c4Pxd++4D7FAxHcbsEr7uf59pdb8LGp+CZn8DHboVjPnXA++0lGgGXG1LtTKRYDFwuaNgKr90Gr94K0T5favPiTVBQDi6Pc3ZWZhEUTYesYphxrnNMYlFoqwO3Fxq3w7bnnH3njYdIMH5bPxRNg9zxkFkI+ZOcfTVsgbr3YOJ8yC0bvL+qh/Z30FLjfKjQn3Po2uhuazc0V0HeRBA3hOMfXswZB+5RGCOqEOl0fsfiGvz3oAqdzeDJcE7w6GwFjz9+jrdA/UZnX8UzwX9kX5JlFP6mhkcTR/iu+N2bdiaMmQOxfa+Y+b2Nn4Y1P4M9G3sKm3dBTUKYf+xWuPEoiIX3uT3gPEAO4IHQHAxzzI+eIMPrpvL7i8jyJxz+jU/BvR/vWf/zFyHUCv/4f07YfPQWKPmAsy0Wg1eXQ0cDnPJ154++vQEe+RfY+ETPPsafCOf/AiZUwDsPwws3wbFLnAfznI86QXY4dQVtzTvOemaBEyBdv5fccc4Tb/XbUDobWmudAI9FnZCteq33/qacBmOPdkb5hVMhkAubnoLNzzh/rKH49ZO2Pe/8HO6rMZe392OgYIozZejywKSTobXO6U/TDlj/N+hohHHHQdmxTmBMPBnGzHZ+jy53P8cn3uf2Pc796RJsdk4rbtjsHJvW3VC7DtavdMIss8h5olKFqafBzPOc2xdNd0462LPBeTxnFUP1aqft1t0w9+POiQk7K53bZhTA7nec5axiZz2QB++vgl1v9P6QS6KcMicMi6Y5j8OWamdwFYs498eX7fx0eZzHsricxwEC3gwI5DuPhfYG50m6pdrZ5vHH99HmHMusEqc/OWOdv9HcceDNdOq7POAJQNWrzoAg3NEzgANnf5mFUDrX2Wcg3/mbbd7lvCrsaNj/79+XDdljnHa8GT3r/lznvmeXOutNVc6rzuKZTr1YBIpmONsSnniC4SihaIwcvwfpKo9F+39sJInoKJ2Xrqio0MrKoV9BeW9HGx96cD4A/zflMiYtjH+n+vIznIO//cX+b3jKN5xwOfEKeP3O7uILs//IA187l0xC8J8JI7pAnjOlAPDJu2DWRcQUXPGRe3VTB163q9cXp9+zajv/+WglRdLMpxaewNfOP9YZMa78thNUXc6+Dp7s59pzF98CG/4G7z7Wuzx/MuzdPvBByZsETe/vW37hr5wAGjNr6COXvTucICg/xfnDq13nPPF0jfg6GqHyDnjjHie4y46B918efJ9lxzqj1tbd/W/35zqBVlAOcz/hhOlgIp1OSAf3QuNW2Pq888QxZpYzcvf4ncCY/EGnrsfv/E4zCpwnl0C+EyDRsBMMezY6f9hHfRjWPurcn1CrcxZYX3kTnRBqre0ZDScqmAIZ+U7/8ifCxiedUWaiwqngzYLda/a9vdsPx17i9FdjTkiE2mDrc074D8SfixOmvnjoAgi73S7W+wO05ZTygt/NZkK4IyG8sShhfxYE8sgJFBDsbOEoXyFzM0pp6mxionhZEPXiqd/Axr1bCXu8TAqUomFoDSlNMT8FkTrCvgIypBNvuIXc4C5aXLnUaS4+t5AnHWRHGnBplA4CtJKJutyEXQFiCm0aoNObQz6tZIfryYs1EfLmkhHZi2iMiC+PqHjwRloIlR5Pi7eYBvKojubS1hHErWGyCZJJkMnBdYQ8OWRLkCxtB4+PvUXHoznjcPkyCDfX4ssdQ5ZP8GoYIp0ES45h5+46XNueRWMRYjEl1tlKJkGywg1khhvwx/o5CaSPZrJpcBXQ7sqlMeZnW6iANgJMcjcwxtNOcWwPL2WfTfNJX2PZwmn73d9AROR1Ve33+0ZSOvDzXH5uXHQzJ5edDHec7/wRDzRKH1/h/HHO+gis+k13cXnwj/zPJccxoSCDlY/dx2vVUSZJLWdfuIRTAlsoeGwp7p6r9nBx3oN8d3EFn16+qrvslauOovWZm5i2+e7usuf1WLj0AU65bxaurgu+nX2dM2IHePQr8Na9MO9fYOrpcP+S3v31ZcO0M2DdX3rKxsyBL73gTHvUb4bmnXDXhT3bL30Q/vpNJ1D7Hoelf4HJpzrL//xPaNvjTHGEO5zA2/B3OPlLzgjwvZXQWtNz2zkfg7V/7v+49lU61wnY3PHOiGf1A055/RZor3dGlvO+CNtfhpxSOPnLTpjljHWeUA4jVSUUjeH37DviCkdjRKKKz+Mi1lLD2nfXQmYhx04qRLxZ7CGXNTubyA+4qW3qQHe+QWakiaNaXiI/OwPefxlvsAEJd0BnkzNKnnYmePxEA4U0bX2D3IbVRDNL8E75IC5PgD0Fk9ialUeeO0B28Xx2tvk5enwe7dEmWkIttIZaaW6u4qkNT1DTsptotIH1kXrcbi9Zvhz8Lh+ZGQUIwtzCWZTFoEXghfrVvLPnHSLqPI6zPNkU+yfTGRbaw03EcKMEadMaXHiJ0eexoy5cZBCTnic2VQH1QOcEXLFcoq5mVDoQiSCeVjxkUOKdS2dHPrsbveBuJCd3DxkeH9m+fIKREO2xPbjEhcsVpiO2l7DsITN2FOGOUto6o7j9NQgxopFcNOYDiaHhPGKRPDSch0cLKM0ag7iCtHcECIZjtGkNGvOh0QDEAjhvY0aBfX/HuQEPOQEvNc1BorHeOVmc7aM5GEFVKcr04Qk3Mz2zjTHSSE04mw5vPmPb1hOMRPH7Axzt382JvIfHpfjCzRRpIznaii/ayl7PGBo0i6C3gL97ziA08yK+/5HZB/24TavAb2xvZeGfFvQqW7N0Ddx9MWx5prtsh8eDT5XSaMKbtlMWwglL4eErAXgmeiyfC/87Xz59Gk+9u5uNta309VX3n/mW96FeZfO4h9pgzwPodu8NnOV+c9B+753yEfIv/wO1LZ1EYkpxth+fRJ1RoghUvQ63nemMDK/4qxOYqs40xbQzoWCyE6B91bzjvHqY/2VnBNultQ5WXOUE+cEYM9sJ7zUJp7GK23lzG6D8QzB9EUw+BcLxq5NOPW1ITQTDUVQh4HX1vOSNi0RjbG9op7woC7dr6PPkqsp7NS20dUZ4dkMd7+xsoj0UZXxBBtl+D69vb2TtLmfEPXd8LgGPm0hM2bqnjfxM74BXXC3O9pMdcFHTvp1QNIK4g0SDZYgrjEay6BssRQFYMCWf9fURQrIXiQnb6kDcHbgCO3H5GvDmVeLy1yKu3kEbi2Qjrg7E1f91oTTmJdY5Fo+3HXGF8cXGEaMTXJ2EXD2XFnHjo5hTqa6eSDDkI9o+ma7Z3pIcP6qwp7XnvZIJhR5adBsSzSY3rxZvRg1N4XoyYzMpzBbycprJzXDh93ewJ1jLno49tIXbKPAXkefPI9OTQXukjU17N7G30zlTzoWL8rxyqtuq6Yh0kOvLZULOBNzixi1u8gP5CML6hvXUB+uJxCKUZU5C8NIcaiAU68AtXlojfc68G4RPMgAhpO2U+CdS7J9EcUYJ4YgQjQSIhLIIh7MYn5vP0ePGUl6cQWe0nabQHjqiLQSjQXyuALvbq/G6vXhdXhqCDeT58igMFFKYUUimJ5Pqtmo84iHbl01LqAVFmZE/g5jG2Nm6k1A0xOu1r1PbXkueLw8RYVz2OL5z0ncO+L4kssBfugb+9Lleo9Cjp0wC4LINH+Qa7/1O4XGfgcW/5taVL/KrF6q54dMVfPuRDbR09p77v+jYcax4u+sPRnnAdz0nu97r3v5SdDYPzLiBawv/yZhKZ944qkKdFFF02e0E25rIefTy7vozg3cRwsv4/IzuzwdMH5PN7LJcfnTRnO7PDtBW78xDJvONwFAbvPRrJ5RfvMkpm/Mx51VF4zbnVcS2F50nwzsvgPn/SvsZ32NH206m5U/DLW6e3/o31OVlatFRlGWVEepsIexyEfAE8Lv9gzTuCMfCPLzhYd6ue5sxmWOpb+3k5c317GzfhjuwEw3nEQ2OR1ydTBuTTSjWTvVe54nR469jTMYECsLnsKe+jL3tYQJeZypt7rhcPL4OJpe28Grtc4i7g3MmfILn1vr55/paWoK9f6/F2T72tIa618fnZ1Ca62d7fTv1bSEmFAYoyYEGKiH7TTqoJjN8DD6PC3egmmCnn9rg+7i8zSD9f8PapMy5RMPZZHlyeL/9HVzipj22G1wDvPIEMmQM0WAxGe58cmQq4g6SnREm6mqktq0RiRQh4bEUZWfiIYN5Y+dx4sRS3t7RQnNHjJ1722lsD9PcESYn4GFPa4iyfC81rbW0d7qo3euhJMdPfoaPOeNzmVmaw/ypRcwszSbT5wR/XUsntS1BZpfl7vPkOxyhaIiGYAO5vlwyvZmoKjGNISK4ZOCTCGMa63d7Z7ST2vZadrftpqa9htr2WtziJhgJ0hpu5QOFHyAcDdPY2cjWpq1kebPwu/281/AeO1p20NTZRCgaIhg98O++DrgDRDVKOBbG5/IR0xgRHdo37BUFipicO5nmUDNRjXJM8TH85NSfDGkfXSzwl66Bp6+D52/sLusK/NDGb7Pe81UAGmZ8ksLLbuOqP77B6qomnvvOGazatpNLbnkDcHHlqVP4QfylVltnhD++8j45AQ/ffXQVHy7ay55pfydv815+1/L6Pv3SK59CJp7Uvb7l1ZXkr76Nwkt+x82vNXHD4+v7vT+TizJZOKOEiYUZdIRibK9v48ZPHYuI8PaOvcwZl4unvzN+BtHWGeF3z23hEydM4J1dTTzy5k5++JHZtIei/NsDb/FutTOy/dqZ0zl+UgGnTC/mmfW17Gpu5NatX6Al3Dzgvt3iJqo9I87TJ5zOnOI5FPgLmJo/ldr2Wh7d9Cib927GLW5y/Dnsad9DY2fjAfdfVRDRfcpcHXMId5ThCU2kPRLFX/IE7kB1P7d3IcEZfGritUwrLmDuhCzK8vwUZWbzxvt7KS8OsKV5I2sb3mBj40ZKs0rZ2LiJV2teoSP+RqBLXATcAdojPSP9gDvAMSXHkOfPY0L2BEoySygIFPBu/buoKquqV9EQbKAh6LxB6HF5mFkwk7GZY/F7/EzOnYxHPFS3VeMWN9PypzEtfxonjT1p0PAbrmhMD+pVUioLR8PUB+upD9bTGGwkHA3TEm5hbOZYxmaNJdObid/tJ6Yxsr3ZuMTlBL7bh6rSHGqmIdhAa6iVcdnjUJT6jnp8bh9+t5919eu6f+del5cxmWOS9jtOq8Cvb2/h9D99sFfZiaUncmfmXPjH9ZBZzPcmTecvnc4IPVizmLfbl5MpnVyecyv//S+LOeO//sm88kKWX34C8+6dR1ZsFu+vW8Lr319EUXbvEWtntJMzHzyL5lBTd9lzu/ZS0PUGXHYp4ZM+j/tD38Y1wLvvqsqdL23jgdd28LvPnkhpboBLb13FG+8f2MvT1T86h9yAt3u9rTPCPau2Mz4/g1e3NvBA5Q5CkRgFmV4a20PA0P64Pblv4c17E0/2vk9KHpeHk0pPYmLORFbvWc3utt00djZSnlvO5NzJvLzrZUKxUD97dcRChbjEQ2fj8YQaTmVs6Q7GBKbw7XNmcfS4IvweP16Xl85oJ22hNohlkukLIK5OGoINFPmLqQ82snz1b6ncXUlVa8+lMAKeAAtKT6etrYCTyo7HKz7+uPlX1HZuAcDncl45Dda/7vspHirGVjC7aDazCmexaPIiPC4Pu1p34XP7CLgDuMRFpjdz0P2oKlUtVRRlFO23rjEH45AHvoicB/wPzgTlbar6sz7bvwl8AYgAdcDnVXWQ00qSG/gAM31FLKzdwtej2Rxd1BO84abj8O26kAAhxk2cwls7nJC97ORJ5E/4O3e/67zRWnnpW/i9vQO7vqOen6z6CU+9/1Sv8inZE7hk22oumXsFlccs5sonruSj0z/Kdadc57QZC/OTVT9hyVFLOKrwqH36uqdjD/n+fFD4+VMv09CUy6baFqoaO/j0SRO586VttId6RtF+j4tTphfzj/dqQToRbzMaKgRciLsVjTrnabsCVWSMv5fOunOZlXMaOxrbmVaSzacrJnLLs5vZVt/G7684iQ9OL+D379zJH9Y+RGuknrD2zN8uyL2K59+YTEc4ymfnT6a8OIun1+3G73ExqyyXf7xXy3s1Ld31xdNMXn4VbZFmcjPc5GdksnnrDMQVxusNEwrmU5Dp5aZLjue0mSUH9DsezM7WnTy84WEKA4VcNP0icn25+9RRVZ5+/2luefsWijKKmJ4/Hb/bz/M7n2d6/nRyfDlke7PJ9mWzoGwBk3InkenJTOpUhjGHyiENfBFxAxuAs4Eq4DVgiaq+m1DnDOAVVW0XkS8Dp6vqpwfbb7IDv8uSTrgvYZA+M+ckrpzx/7jxifVsrus5y+DfPuLjts3f7F7/y8V/oTyvvHt9VfUqvvjEF3vt++Syk1lQtoCb3nDmwpfOvpy73u05M2fN0jW0hlr5znPf4fmdz1OWVcYTn3gCVSWiEZ7b8Rzf+Oc3+u336stXE4lF8Lq9qCpNwSDRqIsv3/csb7c+THjvPHzFT+PN7ecUvn7ceNqNnDHpDLwuL/Ud9exq3cVXnv5K95toiY4fczy/WPgLCgIF+N1+aluCfPTml/q/HhHONNRgXyO5YGoRN11yHKW5AVo7I2R63bhsSsGYpBgs8JPxwat5wCZV3RJv7H5gMdAd+Kr6TEL9VcBnktBuv2KxwZ/A7uvzHmIg0MkFR5fx5q5tbHlhBxrO567Pz+fNlnsBKAwU0hBsYG392l6B/+Wnvty9fHXF1QBcOutSvC4vs4tms+zJZb3CHuDou47utV7dVs1VT1/Fs1XP7vd+HXP3MQD812n/xV83/5Xndz7P9+d/n3W+H+MrBF/hS/vdB8AvFv6Cn77yU7717LcGrDM+ezwnlp7ID+b/gIAnsM/2MTkBnvrmafz0b+vwul1cfNx4RODN9xs5/QNjmFjoTFUEw1EqtzUyNs9PQaaPwizfPqPkbH/KffbPmFErGX9t44EdCetVwMmD1L8S+Ft/G0RkGbAMYNKkSUno2uAKA4U0dzajqtxfs4zs6SC4mDd1FVfdezsAj3/8cRY+sJCHNjzEOeXn4HV56Yh0EIl/OnR+2Xwum3UZHlfPoVwwbgGnjj+VF3a+wNmTz+YH83/AwgcW9mr7jnPv4POPf36fsD+x9ER+f+7vqWmroSy7jJjGWPzoYrY1bwPg6mev7q7745d/DMAJY04g15eL1+3lpx/6KX63H1Vlb+de/G5/96lgOb4csrxZHFdyHF9/5uusa1jXva/F0xZz5qQzOXPSmQd07DJ8bq5bPLdX2dzxeb3WA143p84oPqD9GWMOvWRM6XwCOE9VvxBf/yxwsqpe1U/dzwBXAaepamff7YkOdkpnd2sTix4+db/1/v2kf2dz02ae3P4kMY3REuqZd/7KcV/hN1gKLqEAAA0GSURBVG/9hql5U3ns4sdYsXkF175wLccUH8P1p1zP0+8/za/e/BU3n3UzCycs7Hf/raFWXq15lTMmnoGIsKt1F//75v+yaPIizpp0FgAbGjfw4s4XyffnM6NgBnn+PCbmTNxnX6rK49seZ2r+VL7/wvfZ0bKD7538PZavXs43TvxG9/6GqmsqSZBeT1jGmCPXoZ7DXwD8SFXPja9/F0BVf9qn3iLgf3HCvnZ/+z2Ugd81d/7L13/JHe/csc/2sVljqWmr4fpTrufi6RcTjUU57p59P8b/0pKXyPEdhgtXGWPMARos8JNx4udrwAwRmSIiPuASYEWfDhwP/A646EDC/lDrmpdWej/ZnV9+PqeOP5WaNueyAVPynG/HcrvcXDHnil51z5l8joW9MeaIMuzAV9UIzjTN48A64EFVXSsi14nIRfFqNwDZwJ9E5C0RWTHA7oZNB7ii37JjlvGbs5xr5Gxt2grA3mDvM1KunX8ti6ct7l7/QMEHupe/VfEt1ixdw/dO/h7HlhzLNfOuSXbXjTHmkErKxK2qrgRW9in7YcLyomS0MxxfPf6rhKLOB2y8LudDSjMLZgJw4dQLWTR5EXn+PM6bch7tkXaOLj663zNUlhy1hCVHLdmn3BhjRru0eKfuw1M/DIDP7eP2c26nMOBcB/7SWZdy/JjjmVM8p1f9j8342GHvozHGHGopF/iJX3HoETcRjeKWnk/Iziub173sEtc+YW+MMakq5b7Ttsu36xv58QnOJ2XDA10D3xhj0kjKBj7ArPwZQM/ZNsYYk85ScEqnZ3lG3lQeuegRJudNHrkOGWPMKJFygd9FAESYXjB9pLtijDGjQspN6fT+5LBdgdEYY7qkXOB36RrhG2OMcaRs4Dss8I0xpkvKBb6ScGkFG+EbY0y3lAv8LsLo/K5eY4wZKSkb+MYYY3pLucDv9RWHNqVjjDHdUi7we7PAN8aYLqkd+DbCN8aYbikX+DHsg1fGGNOflAv8LqLYCN8YYxIkJfBF5DwRWS8im0Rkn+/+ExG/iDwQ3/6KiJQno90D6NnhacYYY44Aww58EXEDNwPnA7OBJSIyu0+1K4FGVZ0O/BL4+XDbHUiva+nYCN8YY7olY4Q/D9ikqltUNQTcDyzuU2cxcFd8+SHgLJFDk8YSbHZ+JvxvjDEmOYE/HtiRsF4VL+u3jqpGgCagqO+ORGSZiFSKSGVdXd1Bdcbr9nBssJPiaNRG+MYYk2BUXQ9fVZcDywEqKioO6toIE8dO5Q/Vu+NrFvjGGNMlGSP8ncDEhPUJ8bJ+64iIB8gD6pPQ9uBshG+MMd2SEfivATNEZIqI+IBLgBV96qwAlsaXPwH8Q3t/U8khYoFvjDFdhj2lo6oREbkKeBxwA3eo6loRuQ6oVNUVwO3APSKyCWjAeVI49GyEb4wx3ZIyh6+qK4GVfcp+mLAcBD6ZjLaGxgLfGGO6pOwnbQEb4RtjTILUDnwb4RtjTLfUDnwb4RtjTDcLfGOMSROpHfjGGGO6WeAbY0yasMA3xpg0YYFvjDFpwgLfGGPShAW+McakCQt8Y4xJExb4xhiTJizwjTEmTVjgG2NMmrDAN8aYNGGBb4wxacIC3xhj0sSwAl9ECkXkSRHZGP9Z0E+d40TkZRFZKyKrReTTw2nTGGPMwRnuCP8a4GlVnQE8HV/vqx24XFXnAOcBN4lI/jDbNcYYM0TDDfzFwF3x5buAi/tWUNUNqroxvrwLqAVKhtmuMcaYIRpu4JeqanV8uQYoHayyiMwDfMDmAbYvE5FKEamsq6sbZteMMcYk8uyvgog8BYztZ9O1iSuqqiKig+ynDLgHWKqqsf7qqOpyYDlARUXFgPsyxhgzdPsNfFVdNNA2EdktImWqWh0P9NoB6uUC/wdcq6qrDrq3xhhjDtpwp3RWAEvjy0uBx/pWEBEf8Ahwt6o+NMz2jDHGHKThBv7PgLNFZCOwKL6OiFSIyG3xOp8CFgJXiMhb8X/HDbNdY4wxQ7TfKZ3BqGo9cFY/5ZXAF+LLfwD+MJx2jDHGDJ990tYYY9KEBb4xxqQJC3xjjEkTFvjGGJMmLPCNMSZNWOAbY0yasMA3xpg0YYFvjDFpwgLfGGPShAW+McakCQt8Y4xJExb4xhiTJizwjTEmTVjgG2NMmrDAN8aYNGGBb4wxacIC3xhj0oQFvjHGpIlhBb6IFIrIkyKyMf6zYJC6uSJSJSK/Hk6bxhhjDs5wR/jXAE+r6gzg6fj6QK4Hnhtme8YYYw7ScAN/MXBXfPku4OL+KonIiUAp8MQw2zPGGHOQhhv4papaHV+uwQn1XkTEBdwIXL2/nYnIMhGpFJHKurq6YXbNGGNMIs/+KojIU8DYfjZdm7iiqioi2k+9rwArVbVKRAZtS1WXA8sBKioq+tuXMcaYg7TfwFfVRQNtE5HdIlKmqtUiUgbU9lNtAfAhEfkKkA34RKRVVQeb7zfGGJNk+w38/VgBLAV+Fv/5WN8KqnpZ17KIXAFUWNgbY8zhN9w5/J8BZ4vIRmBRfB0RqRCR24bbOWOMMckzrBG+qtYDZ/VTXgl8oZ/yO4E7h9OmMcaYg2OftDXGmDRhgW+MMWnCAt8YY9KEBb4xxqQJC3xjjEkTFvjGGJMmLPCNMSZNDPeTtqPTZQ9BqG2ke2GMMaNKagb+jLNHugfGGDPq2JSOMcakCQt8Y4xJExb4xhiTJizwjTEmTVjgG2NMmrDAN8aYNGGBb4wxacIC3xhj0oSo6kj3oV8iUgdsH8YuioE9SerO4WT9PvyO1L4fqf2GI7fvR0K/J6tqSX8bRm3gD5eIVKpqxUj3Y6is34ffkdr3I7XfcOT2/Ujtdxeb0jHGmDRhgW+MMWkilQN/+Uh34CBZvw+/I7XvR2q/4cjt+5HabyCF5/CNMcb0lsojfGOMMQks8I0xJk2kXOCLyHkisl5ENonINSPdn0QiMlFEnhGRd0VkrYh8PV7+IxHZKSJvxf9dkHCb78bvy3oROXfkeg8isk1E1sT7WBkvKxSRJ0VkY/xnQbxcRORX8b6vFpETRqjPH0g4rm+JSLOIfGO0HnMRuUNEakXknYSyIR9jEVkar79RRJaOUL9vEJH34n17RETy4+XlItKRcOxvSbjNifHH2Kb4fZMR6vuQHx+jOXu6qWrK/APcwGZgKuAD3gZmj3S/EvpXBpwQX84BNgCzgR8BV/dTf3b8PviBKfH75h7B/m8DivuU/QK4Jr58DfDz+PIFwN8AAeYDr4yC4+8GaoDJo/WYAwuBE4B3DvYYA4XAlvjPgvhywQj0+xzAE1/+eUK/yxPr9dnPq/H7IvH7dv4IHfMhPT5Ge/Z0/Uu1Ef48YJOqblHVEHA/sHiE+9RNVatV9Y34cguwDhg/yE0WA/eraqeqbgU24dzH0WQxcFd8+S7g4oTyu9WxCsgXkbKR6GCCs4DNqjrYJ7hH9Jir6nNAQz99GsoxPhd4UlUbVLUReBI473D3W1WfUNVIfHUVMGGwfcT7nquqq9RJ17vpua+HzADHfCADPT5GdfZ0SbXAHw/sSFivYvBAHTEiUg4cD7wSL7oq/tL3jq6X7Iy++6PAEyLyuogsi5eVqmp1fLkGKI0vj7a+A1wC3JewfiQccxj6MR6N9+HzOCP2LlNE5E0ReVZEPhQvG4/T1y4j3e+hPD5G4zHfR6oF/hFBRLKBh4FvqGoz8FtgGnAcUA3cOILdG8ypqnoCcD7wryKyMHFjfFQ2Ks/zFREfcBHwp3jRkXLMexnNx3ggInItEAHujRdVA5NU9Xjgm8AfRSR3pPo3gCPy8bE/qRb4O4GJCesT4mWjhoh4ccL+XlX9M4Cq7lbVqKrGgFvpmUIYVfdHVXfGf9YCj+D0c3fXVE38Z228+qjqO86T1BuquhuOnGMeN9RjPGrug4hcAXwEuCz+ZEV8OqQ+vvw6ztz3zHgfE6d9RqzfB/H4GDXHfDCpFvivATNEZEp8RHcJsGKE+9QtfsbB7cA6Vf3vhPLEue2PAl1nC6wALhERv4hMAWbgvKl12IlIlojkdC3jvCH3TryPXWeBLAUeiy+vAC6Pn0kyH2hKmJYYCUtImM45Eo55gqEe48eBc0SkID4VcU687LASkfOA7wAXqWp7QnmJiLjjy1NxjvGWeN+bRWR+/G/lcnru62F1EI+PUZ093Ub6XeNk/8M5c2EDzqjh2pHuT5++nYrzcnw18Fb83wXAPcCaePkKoCzhNtfG78t6DsMZC4P0fSrOmQdvA2u7ji1QBDwNbASeAgrj5QLcHO/7GqBiBPueBdQDeQllo/KY4zwpVQNhnHngKw/mGOPMmW+K//vcCPV7E868dtdj/ZZ43Y/HH0NvAW8AFybspwInXDcDvyZ+NYAR6PuQHx+jOXu6/tmlFYwxJk2k2pSOMcaYAVjgG2NMmrDAN8aYNGGBb4wxacIC3xhj0oQFvjHGpAkLfGOMSRP/H1jAoWlXEJWTAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["### Replay Environment"],"metadata":{"id":"tL1Ur1CtzlGx"}},{"cell_type":"code","source":["class ReplayEnvironment(BaseEnvironment):\n"," dataset: BanditDataset\n","\n"," def __init__(self):\n"," super().__init__()\n"," self.counter = None\n"," self.last_observation = None\n","\n"," def env_init(self, env_info=None):\n"," \"\"\"\n"," Set parameters needed to setup the replay SavePilot environment.\n"," Assume env_info dict contains:\n"," {\n"," pickle_file: data directory [str]\n"," }\n"," Args:\n"," env_info (dict):\n"," \"\"\"\n"," if env_info is None:\n"," env_info = {}\n","\n"," directory = env_info['pickle_file']\n"," seed = env_info.get('seed', None)\n"," self.dataset = BanditDataset(directory, seed)\n"," self.idxs = range(self.dataset.__len__())\n"," self.counter = 0\n","\n"," def _get_observation(self):\n"," idx = self.idxs[self.counter]\n","\n"," return self.dataset.__getitem__(idx)\n","\n"," def env_start(self):\n"," self.last_observation = self._get_observation()\n","\n"," state = self.last_observation[0]\n"," reward = None\n"," is_terminal = False\n","\n"," self.reward_state_term = (reward, state, is_terminal)\n"," self.counter += 1\n","\n"," # return first state from the environment\n"," return self.reward_state_term[1]\n","\n"," def env_step(self, action):\n"," true_action = self.last_observation[1]\n"," reward = self.last_observation[2]\n","\n"," if true_action != action:\n"," reward = None\n","\n"," observation = self._get_observation()\n"," state = observation[0]\n","\n"," if self.counter == self.dataset.__len__() - 1:\n"," is_terminal = True\n"," else:\n"," is_terminal = False\n","\n"," self.reward_state_term = (reward, state, is_terminal)\n","\n"," self.last_observation = observation\n"," self.counter += 1\n","\n"," return self.reward_state_term\n","\n"," def env_cleanup(self):\n"," pass\n","\n"," def env_message(self, message):\n"," pass"],"metadata":{"id":"YK8fE2VyzlEO"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Wrappers"],"metadata":{"id":"s42XsgHev0lv"}},{"cell_type":"markdown","source":["### RL Glue"],"metadata":{"id":"OlbboMWrwRV2"}},{"cell_type":"code","source":["class RLGlue:\n"," \"\"\"RLGlue class\n"," args:\n"," env_name (string): the name of the module where the Environment class can be found\n"," agent_name (string): the name of the module where the Agent class can be found\n"," \"\"\"\n","\n"," def __init__(self, env_class, agent_class):\n"," self.environment = env_class()\n"," self.agent = agent_class()\n","\n"," self.total_reward = None\n"," self.average_reward = None\n"," self.last_action = None\n"," self.num_steps = None\n"," self.num_episodes = None\n"," self.num_matches = None\n","\n"," def rl_init(self, agent_init_info={}, env_init_info={}):\n"," \"\"\"Initial method called when RLGlue experiment is created\"\"\"\n"," self.environment.env_init(env_init_info)\n"," self.agent.agent_init(agent_init_info)\n","\n"," self.total_reward = 0.0\n"," self.average_reward = [0]\n"," self.num_steps = 0\n"," self.num_episodes = 0\n"," self.num_matches = 0\n","\n"," def rl_start(self):\n"," \"\"\"Starts RLGlue experiment\n"," Returns:\n"," tuple: (state, action)\n"," \"\"\"\n","\n"," last_state = self.environment.env_start()\n"," self.last_action = self.agent.agent_start(last_state)\n","\n"," observation = (last_state, self.last_action)\n","\n"," return observation\n","\n"," def rl_agent_start(self, observation):\n"," \"\"\"Starts the agent.\n"," Args:\n"," observation: The first observation from the environment\n"," Returns:\n"," The action taken by the agent.\n"," \"\"\"\n"," return self.agent.agent_start(observation)\n","\n"," def rl_agent_step(self, reward, observation):\n"," \"\"\"Step taken by the agent\n"," Args:\n"," reward (float): the last reward the agent received for taking the\n"," last action.\n"," observation : the state observation the agent receives from the\n"," environment.\n"," Returns:\n"," The action taken by the agent.\n"," \"\"\"\n"," return self.agent.agent_step(reward, observation)\n","\n"," def rl_agent_end(self, reward):\n"," \"\"\"Run when the agent terminates\n"," Args:\n"," reward (float): the reward the agent received when terminating\n"," \"\"\"\n"," self.agent.agent_end(reward)\n","\n"," def rl_env_start(self):\n"," \"\"\"Starts RL-Glue environment.\n"," Returns:\n"," (float, state, Boolean): reward, state observation, boolean\n"," indicating termination\n"," \"\"\"\n"," self.total_reward = 0.0\n"," self.num_steps = 1\n","\n"," this_observation = self.environment.env_start()\n","\n"," return this_observation\n","\n"," def rl_env_step(self, action):\n"," \"\"\"Step taken by the environment based on action from agent\n"," Args:\n"," action: Action taken by agent.\n"," Returns:\n"," (float, state, Boolean): reward, state observation, boolean\n"," indicating termination.\n"," \"\"\"\n"," ro = self.environment.env_step(action)\n"," (this_reward, _, terminal) = ro\n","\n"," self.total_reward += this_reward\n","\n"," if terminal:\n"," self.num_episodes += 1\n"," else:\n"," self.num_steps += 1\n","\n"," return ro\n","\n"," def rl_step(self):\n"," \"\"\"Step taken by RLGlue, takes environment step and either step or\n"," end by agent.\n"," Returns:\n"," (float, state, action, Boolean): reward, last state observation,\n"," last action, boolean indicating termination\n"," \"\"\"\n","\n"," (reward, last_state, term) = self.environment.env_step(self.last_action)\n","\n"," if reward is not None:\n"," self.num_matches += 1\n"," aw_reward = self.average_reward[-1] + (reward - self.average_reward[-1]) / self.num_matches\n"," self.average_reward.append(aw_reward)\n"," self.total_reward += reward\n","\n"," if term:\n"," self.num_episodes += 1\n"," self.agent.agent_end(reward)\n"," roat = (reward, last_state, None, term)\n"," else:\n"," self.num_steps += 1\n"," self.last_action = self.agent.agent_step(reward, last_state)\n"," roat = (reward, last_state, self.last_action, term)\n","\n"," return roat\n","\n"," def rl_cleanup(self):\n"," \"\"\"Cleanup done at end of experiment.\"\"\"\n"," self.environment.env_cleanup()\n"," self.agent.agent_cleanup()\n","\n"," def rl_agent_message(self, message):\n"," \"\"\"Message passed to communicate with agent during experiment\n"," Args:\n"," message: the message (or question) to send to the agent\n"," Returns:\n"," The message back (or answer) from the agent\n"," \"\"\"\n","\n"," return self.agent.agent_message(message)\n","\n"," def rl_env_message(self, message):\n"," \"\"\"Message passed to communicate with environment during experiment\n"," Args:\n"," message: the message (or question) to send to the environment\n"," Returns:\n"," The message back (or answer) from the environment\n"," \"\"\"\n"," return self.environment.env_message(message)\n","\n"," def rl_episode(self, max_steps_this_episode):\n"," \"\"\"Runs an RLGlue episode\n"," Args:\n"," max_steps_this_episode (Int): the maximum steps for the experiment to run in an episode\n"," Returns:\n"," Boolean: if the episode should terminate\n"," \"\"\"\n"," is_terminal = False\n","\n"," self.rl_start()\n","\n"," while (not is_terminal) and ((max_steps_this_episode == 0) or\n"," (self.num_steps < max_steps_this_episode)):\n"," rl_step_result = self.rl_step()\n"," is_terminal = rl_step_result[3]\n","\n"," return is_terminal\n","\n"," def rl_return(self):\n"," \"\"\"The total reward\n"," Returns:\n"," float: the total reward\n"," \"\"\"\n"," return self.total_reward\n","\n"," def rl_num_steps(self):\n"," \"\"\"The total number of steps taken\n"," Returns:\n"," Int: the total number of steps taken\n"," \"\"\"\n"," return self.num_steps\n","\n"," def rl_num_episodes(self):\n"," \"\"\"The number of episodes\n"," Returns\n"," Int: the total number of episodes\n"," \"\"\"\n"," return self.num_episodes"],"metadata":{"id":"InGf3vAFwTIF"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Policy"],"metadata":{"id":"MCXJTVo5xRpq"}},{"cell_type":"code","source":["class Policy:\n"," def __init__(self, env, agent):\n"," self.env = env\n"," self.agent = agent\n"," self.rl_glue = None\n","\n"," @abstractmethod\n"," def get_average_performance(self, agent_info=None, env_info=None, exper_info=None):\n"," raise NotImplementedError"],"metadata":{"id":"9RsPVJpnwmad"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Bandit wrapper"],"metadata":{"id":"RbiWk0wWxTD-"}},{"cell_type":"code","source":["class BanditWrapper(Policy):\n"," def get_average_performance(self, agent_info=None, env_info=None, exper_info=None):\n","\n"," if exper_info is None:\n"," exper_info = {}\n"," if env_info is None:\n"," env_info = {}\n"," if agent_info is None:\n"," agent_info = {}\n","\n"," num_runs = exper_info.get(\"num_runs\", 100)\n"," num_steps = exper_info.get(\"num_steps\", 1000)\n"," return_type = exper_info.get(\"return_type\", None)\n"," seed = exper_info.get(\"seed\", None)\n","\n"," np.random.seed(seed)\n"," seeds = np.random.randint(0, num_runs * 100, num_runs)\n","\n"," all_averages = []\n"," subopt_arm_average = []\n"," best_arm = []\n"," worst_arm = []\n"," all_chosen_arm = []\n"," average_regret = []\n","\n"," for run in tqdm(range(num_runs)):\n"," np.random.seed(seeds[run])\n","\n"," self.rl_glue = RLGlue(self.env, self.agent)\n"," self.rl_glue.rl_init(agent_info, env_info)\n"," (first_state, first_action) = self.rl_glue.rl_start()\n","\n"," worst_position = np.argmin(self.rl_glue.environment.arms)\n"," best_value = np.max(self.rl_glue.environment.arms)\n"," worst_value = np.min(self.rl_glue.environment.arms)\n"," best_arm.append(best_value)\n"," worst_arm.append(worst_value)\n","\n"," scores = [0]\n"," averages = []\n"," subopt_arm = []\n"," chosen_arm_log = []\n","\n"," cum_regret = [0]\n"," delta = self.rl_glue.environment.subopt_gaps[first_action]\n"," cum_regret.append(cum_regret[-1] + delta)\n","\n"," # first action was made in rl_start, that's why run over num_steps-1\n"," for i in range(num_steps-1):\n"," reward, _, action, _ = self.rl_glue.rl_step()\n"," chosen_arm_log.append(action)\n"," scores.append(scores[-1] + reward)\n"," averages.append(scores[-1] / (i + 1))\n"," subopt_arm.append(self.rl_glue.agent.arm_count[worst_position])\n","\n"," delta = self.rl_glue.environment.subopt_gaps[action]\n"," cum_regret.append(cum_regret[-1] + delta)\n","\n"," all_averages.append(averages)\n"," subopt_arm_average.append(subopt_arm)\n"," all_chosen_arm.append(chosen_arm_log)\n","\n"," average_regret.append(cum_regret)\n","\n"," if return_type is None:\n"," returns = (np.mean(all_averages, axis=0),\n"," np.mean(best_arm))\n"," elif return_type == 'regret':\n"," returns = np.mean(average_regret, axis=0)\n"," elif return_type == 'regret_reward':\n"," returns = (np.mean(average_regret, axis=0),\n"," np.mean(all_averages, axis=0))\n"," elif return_type == 'arm_choice_analysis':\n"," returns = (np.mean(all_averages, axis=0),\n"," np.mean(best_arm),\n"," np.mean(all_chosen_arm, axis=0))\n"," elif return_type == 'complex':\n"," returns = (np.mean(all_averages, axis=0),\n"," np.mean(subopt_arm_average, axis=0),\n"," np.array(best_arm), np.array(worst_arm),\n"," np.mean(average_regret, axis=0))\n","\n"," return returns"],"metadata":{"id":"zAzjgURdwsl5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Run experiments"],"metadata":{"id":"58Lm6nzi3c7a"}},{"cell_type":"code","source":["def run_experiment(environment, agent, environment_parameters, agent_parameters,\n"," experiment_parameters, save_data=True, dir=''):\n"," rl_glue = RLGlue(environment, agent)\n","\n"," # save sum of reward at the end of each episode\n"," agent_sum_reward = []\n","\n"," env_info = environment_parameters\n"," agent_info = agent_parameters\n","\n"," # one agent setting\n"," for run in tqdm(range(1, experiment_parameters[\"num_runs\"] + 1)):\n"," env_info[\"seed\"] = run\n","\n"," rl_glue.rl_init(agent_info, env_info)\n"," rl_glue.rl_episode(0)\n"," agent_sum_reward.append(rl_glue.average_reward)\n","\n"," leveled_result = get_leveled_data(agent_sum_reward)\n"," if save_data:\n"," save_name = \"{}-{}\".format(rl_glue.agent.name, rl_glue.agent.batch_size)\n"," file_dir = \"results/{}\".format(dir)\n"," if not os.path.exists(file_dir):\n"," os.makedirs(file_dir)\n"," np.save(\"{}/sum_reward_{}\".format(file_dir, save_name), leveled_result)\n","\n"," return leveled_result"],"metadata":{"id":"GWc2CLhz3c2E"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# if __name__ == '__main__':\n","\n","# num_experements = 10\n","# batch_size = 100\n","# data_dir = 'data/mushroom_data_final.pickle'\n","\n","# experiment_parameters = {\"num_runs\": num_experements}\n","# env_info = {'pickle_file': data_dir}\n","# agent_info = {'alpha': 2,\n","# 'num_actions': 3,\n","# 'seed': 1,\n","# 'batch_size': 1}\n","\n","# agent = LinUCBAgent\n","# environment = ReplayEnvironment\n","\n","# result = run_experiment(environment, agent, env_info, agent_info, experiment_parameters, save_data=False)\n","\n","# smoothed_leveled_result = smooth(result, 100)\n","# mean_smoothed_leveled_result = np.mean(smoothed_leveled_result, axis=0)\n","\n","# plt.plot(mean_smoothed_leveled_result, lw=3, ls='-.', label='online policy')\n","# plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":297,"referenced_widgets":["87c240ca0fe8431096f5092d00b6d63f","6933623da13c468ea1caba8cb1e126f5","3beaa54b58654d2199153c4229b50f84","317c46945fd64488ad7b2cc0bf30b7d2","fa2e58ea24f84364b95fcc3f1587e9ad","afa78843d54d404ab99cd87d55a36bb9","0c9761efc1a04534871cb6c05f232f4a","21b40574fdb44788af67ece3c7b770c6","0a8e3b93b06740ad8cb9075dbf5d041a","bcaa037ccf1b47bf8d40956a370f93dc","2258f153219f48cc882eda53f0db33a2"]},"id":"oLbyRqGi3miH","executionInfo":{"status":"ok","timestamp":1639148611213,"user_tz":-330,"elapsed":8524,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"d2341afc-3a7d-4ca0-ff62-a1904cad3dc8"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"87c240ca0fe8431096f5092d00b6d63f","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/10 [00:00"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["## Experiments"],"metadata":{"id":"kD7w8VVTwmV-"}},{"cell_type":"markdown","source":["### UCB"],"metadata":{"id":"S9-ruIHEwmTS"}},{"cell_type":"markdown","source":["#### UCB dynamic by timesteps"],"metadata":{"id":"iSarqXSj5buA"}},{"cell_type":"code","source":["env = Environment\n","agent = UCBAgent\n","\n","alpha = 1\n","\n","num_runs = 1000\n","num_steps = 10000\n","seed = None\n","if_save = False\n","exper_info = {\"num_runs\": num_runs,\n"," \"num_steps\": num_steps,\n"," \"seed\": seed,\n"," \"return_type\": \"regret\"}\n","\n","k = 2\n","arms_values = [0.7, 0.65]\n","reward_type = 'Bernoulli'\n","env_info = {\"num_actions\": k,\n"," \"reward_type\": reward_type,\n"," \"arms_values\": arms_values}\n","\n","# batch-online experiment\n","batch_res = []\n","online_res = []\n","batch = 10\n","agent_info_batch = {\"num_actions\": k, \"batch_size\": batch, \"alpha\": alpha}\n","agent_info_online = {\"num_actions\": k, \"batch_size\": 1, \"alpha\": alpha}\n","\n","exp1 = BanditWrapper(env, agent)\n","batch_res.append(exp1.get_average_performance(agent_info_batch, env_info, exper_info))\n","online_res.append(exp1.get_average_performance(agent_info_online, env_info, exper_info))\n","\n","av_online_res = np.mean(online_res, axis=0)\n","av_batch_res = np.mean(batch_res, axis=0)\n","\n","plt.plot(av_batch_res, label='batch')\n","plt.plot(av_online_res, label='online')\n","\n","M = int(num_steps / batch)\n","update_points = np.ceil(np.arange(num_steps) / batch).astype(int)\n","plt.plot(av_online_res[update_points] * batch, ls='--',\n"," label='upper bound, batch size = 10')\n","plt.title('Cumulative Regret averaged over ' + str(num_runs) + ' runs')\n","plt.xlabel('time steps')\n","plt.ylabel('regret')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.legend()\n","if if_save:\n"," plt.savefig('results/UCB transform example.png', bbox_inches='tight')\n","plt.show()\n","\n","if if_save:\n"," name = 'batch_result, runs=' + str(num_runs) + ', steps=' + str(num_steps)\n"," with open('results/' + '/' + name + '.pickle', 'wb') as handle:\n"," pickle.dump(batch_res, handle, protocol=pickle.HIGHEST_PROTOCOL)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":359,"referenced_widgets":["6574536e49dd498bae5d87ab60144c59","a787648e15174568a65f7b5e17ee43d2","9fccdb73039f4b649f5389aa2b4099ad","701a04ff58fc492f88774aa7139dcbc2","37296861808045328c2c4eb74625987a","8a9c221fcb4a46f9af525b5b0373a598","f155f2ec83cb416d8d375ae77330165b","af831b75ff7146fc99abb8677c84bdf3","8fe71971bf8049bca4a5c6fde2bf7471","9a87fac6283543008f922a57f98b8514","ea8e8fab825a499c8fe628a021b731ca","d2a7f4cc89a44501b6423cff8f76f77d","9231d7a857624fbfa4c6dc1ba3307814","7dbee5d5ef884477be34e5a9889d13b1","ed0108c515b94090bc7ee284284b535b","ef228c5730d943149399ec81f4de4d46","0dd4ce26eb994bf89f9d429982672f1d","3c78bcaf65d745928c7cec9b3618f2f4","3f9147ea50504b5e941cf600259b64c6","3cb22a9a850b42ce8d72132b34cede77","3929bb10e2044d26869495a947ecd340","cf4c98d2628e418b84ef3c798588675a"]},"id":"Vc7uTIz8xARE","executionInfo":{"status":"ok","timestamp":1639148297436,"user_tz":-330,"elapsed":1433361,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"38fb8767-6cf3-41cc-8cd3-1b9abd760ed6"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6574536e49dd498bae5d87ab60144c59","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/1000 [00:00"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### UCB dynamic by batches"],"metadata":{"id":"82qKEl1F5dMB"}},{"cell_type":"code","source":["model_dir = 'results/UCB/dynamic_by_batches'\n","if not os.path.exists(model_dir):\n"," print(f'Creating a new model directory: {model_dir}')\n"," os.makedirs(model_dir)\n","\n","num_runs = 10 # 500\n","num_steps = 10001\n","seed = None\n","exper_info = {\"num_runs\": num_runs,\n"," \"num_steps\": num_steps,\n"," \"seed\": seed,\n"," \"return_type\": \"regret\"}\n","\n","environments = [[0.7, 0.5], [0.7, 0.4], [0.7, 0.1],\n"," [0.35, 0.18, 0.47, 0.61],\n"," [0.4, 0.75, 0.57, 0.49],\n"," [0.70, 0.50, 0.30, 0.10]]\n","\n","for arms_values in environments:\n"," k = len(arms_values)\n"," reward_type = 'Bernoulli'\n"," env_info = {\"num_actions\": k,\n"," \"reward_type\": reward_type,\n"," \"arms_values\": arms_values}\n"," env = Environment\n"," agent = UCBAgent\n"," alpha = 1\n","\n"," # run online agent\n"," agent_info_online = {\"num_actions\": k, \"batch_size\": 1, \"alpha\": alpha}\n"," experiment = BanditWrapper(env, agent)\n"," online_regret = experiment.get_average_performance(agent_info_online, env_info, exper_info)\n","\n"," # run batch agent\n"," batches = np.logspace(1.0, 3.0, num=20).astype(int)\n"," actual_regret = []\n"," upper_bound = []\n","\n"," for batch in batches:\n"," agent_info_batch = {\"num_actions\": k, \"batch_size\": batch, \"alpha\": alpha}\n"," experiment = BanditWrapper(env, agent)\n"," batch_regret = experiment.get_average_performance(agent_info_batch, env_info, exper_info)\n"," actual_regret.append(batch_regret[-1])\n"," M = int(num_steps / batch)\n"," upper_bound.append(online_regret[M] * batch)\n","\n"," # save data\n"," name = 'dyn_by_batch_' + str(arms_values)\n"," name1 = name + ' batch_regret'\n"," with open(model_dir + '/' + name1 + '.pickle', 'wb') as handle:\n"," pickle.dump(actual_regret, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","\n"," name2 = name + ' online_regret'\n"," with open(model_dir + '/' + name2 + '.pickle', 'wb') as handle:\n"," pickle.dump(online_regret, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","\n","print(\"End!\")"],"metadata":{"id":"L3KoEguL5dIz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### TS"],"metadata":{"id":"t618mNgBxKKM"}},{"cell_type":"markdown","source":["#### TS dynamic by timesteps"],"metadata":{"id":"oNTgxitH5Dy_"}},{"cell_type":"code","source":["env = Environment\n","agent = TSAgent\n","\n","num_runs = 10 # 1000\n","num_steps = 10000\n","seed = None\n","if_save = False\n","exper_info = {\"num_runs\": num_runs,\n"," \"num_steps\": num_steps,\n"," \"seed\": seed,\n"," \"return_type\": \"regret\"}\n","\n","k = 2\n","arms_values = [0.7, 0.65]\n","reward_type = 'Bernoulli'\n","env_info = {\"num_actions\": k,\n"," \"reward_type\": reward_type,\n"," \"arms_values\": arms_values}\n","\n","# batch-online experiment\n","batch_res = []\n","online_res = []\n","batch = 10\n","agent_info_batch = {\"num_actions\": k, \"batch_size\": batch}\n","agent_info_online = {\"num_actions\": k, \"batch_size\": 1}\n","\n","exp1 = BanditWrapper(env, agent)\n","batch_res.append(exp1.get_average_performance(agent_info_batch, env_info, exper_info))\n","online_res.append(exp1.get_average_performance(agent_info_online, env_info, exper_info))\n","\n","av_online_res = np.mean(online_res, axis=0)\n","av_batch_res = np.mean(batch_res, axis=0)\n","\n","plt.plot(av_batch_res, label='batch')\n","plt.plot(av_online_res, label='online')\n","\n","M = int(num_steps / batch)\n","update_points = np.ceil(np.arange(num_steps) / batch).astype(int)\n","plt.plot(av_online_res[update_points] * batch, ls='--',\n"," label='upper bound, batch size = 10')\n","plt.title('Cumulative Regret averaged over ' + str(num_runs) + ' runs')\n","plt.xlabel('time steps')\n","plt.ylabel('regret')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.legend()\n","if if_save:\n"," plt.savefig('results/TS example.png', bbox_inches='tight')\n","plt.show()\n","\n","if if_save:\n"," name = 'batch_result, runs=' + str(num_runs) + ', steps=' + str(num_steps)\n"," with open('results/' + '/' + name + '.pickle', 'wb') as handle:\n"," pickle.dump(batch_res, handle, protocol=pickle.HIGHEST_PROTOCOL)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":359,"referenced_widgets":["959b3e666a064b6d8917c8625ecd8fee","ba3a1e89d28342f683d0013065137cb2","53079098231c4906af72b59e6773052f","3415712fbd16477b8eb7bf6844c56d51","7269023608aa4b9c833466b4634a02cd","23ca025a590c4caba147cfee82b8fb82","6892149d580c4521bfbc8ce341445d61","0767ce00fdc24b709c3ca2169e13e545","c11a3dcd03f24240826793c233950bea","ff5a4612a01b47a8a805ca8a643956c3","f8e2be434a1846c797556912bad038f1","85a8b0c33c104b8eb22fe9e21dc3d543","f8a6e8d21951434b84789ff596bf188b","3825eaf811174f30b38725c44d1536a6","f0fad1d578be4cb790bcbc91e2088d85","367d61cced8a4b7a9eb3763f11856cce","0655f854114a4705ba29eabd0cc99e31","f06c08cde5e14462a35c9b411e3cb167","d769f331f73d42a1ae05b69c07023890","52ec6649d8e94615bf9397cbd9d85b48","fa0bd788f3a8421d949951e60f790c07","2464f2977ba04e8a855461a081005305"]},"id":"9POfXn2TxLWQ","executionInfo":{"status":"ok","timestamp":1639148363702,"user_tz":-330,"elapsed":55499,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"15423772-d107-4547-f6fa-59c4a54546e3"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"959b3e666a064b6d8917c8625ecd8fee","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/10 [00:00"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### TS dynamic by batches"],"metadata":{"id":"koXASf5O5GR2"}},{"cell_type":"code","source":["model_dir = 'results/TS/dynamic_by_batches'\n","if not os.path.exists(model_dir):\n"," print(f'Creating a new model directory: {model_dir}')\n"," os.makedirs(model_dir)\n","\n","num_runs = 10 # 500\n","num_steps = 10001\n","seed = None\n","exper_info = {\"num_runs\": num_runs,\n"," \"num_steps\": num_steps,\n"," \"seed\": seed,\n"," \"return_type\": \"regret\"}\n","\n","environments = [[0.7, 0.5], [0.7, 0.4], [0.7, 0.1],\n"," [0.35, 0.18, 0.47, 0.61],\n"," [0.4, 0.75, 0.57, 0.49],\n"," [0.70, 0.50, 0.30, 0.10]]\n","\n","for arms_values in environments:\n"," k = len(arms_values)\n"," reward_type = 'Bernoulli'\n"," env_info = {\"num_actions\": k,\n"," \"reward_type\": reward_type,\n"," \"arms_values\": arms_values}\n"," env = Environment\n"," agent = TSAgent\n","\n"," # run online agent\n"," agent_info_online = {\"num_actions\": k, \"batch_size\": 1}\n"," experiment = BanditWrapper(env, agent)\n"," online_regret = experiment.get_average_performance(agent_info_online, env_info, exper_info)\n","\n"," # run batch agent\n"," batches = np.logspace(1.0, 3.0, num=20).astype(int)\n"," actual_regret = []\n"," upper_bound = []\n","\n"," for batch in batches:\n"," agent_info_batch = {\"num_actions\": k, \"batch_size\": batch}\n"," experiment = BanditWrapper(env, agent)\n"," batch_regret = experiment.get_average_performance(agent_info_batch, env_info, exper_info)\n"," actual_regret.append(batch_regret[-1])\n"," M = int(num_steps / batch)\n"," upper_bound.append(online_regret[M] * batch)\n","\n"," # save data\n"," name = 'dyn_by_batch_' + str(k) + str(arms_values)\n"," name1 = name + ' batch_regret'\n"," with open(model_dir + '/' + name1 + '.pickle', 'wb') as handle:\n"," pickle.dump(actual_regret, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","\n"," name2 = name + ' online_regret'\n"," with open(model_dir + '/' + name2 + '.pickle', 'wb') as handle:\n"," pickle.dump(online_regret, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","\n","print(\"End!\")"],"metadata":{"id":"T-CjxemW5GNq"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### LinUCB"],"metadata":{"id":"L-4KgfMU0iE9"}},{"cell_type":"markdown","source":["#### LinUCB by timesteps"],"metadata":{"id":"elLqw6xk4083"}},{"cell_type":"code","source":["num_experiments = 20\n","batch_size = 100\n","data_dir = 'data/mushroom_data_final.pickle'\n","env_info = {'pickle_file': data_dir}\n","output_dir = 'LinUCB/dynamic_by_timesteps'\n","\n","agent_info = {'alpha': 2,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': 1}\n","agent_info_batch = {'alpha': 2,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch_size}\n","experiment_parameters = {\"num_runs\": num_experiments}\n","\n","agent = LinUCBAgent\n","environment = ReplayEnvironment\n","\n","online_result = run_experiment(environment, agent, env_info, agent_info,\n"," experiment_parameters, True, output_dir)\n","batch_result = run_experiment(environment, agent, env_info, agent_info_batch,\n"," experiment_parameters, True, output_dir)\n","\n","smoothed_leveled_result = smooth(online_result, 100)\n","smoothed_leveled_result1 = smooth(batch_result, 100)\n","\n","mean_smoothed_leveled_result = np.mean(smoothed_leveled_result, axis=0)\n","mean_smoothed_leveled_result1 = np.mean(smoothed_leveled_result1, axis=0)\n","\n","num_steps = np.minimum(len(mean_smoothed_leveled_result), len(mean_smoothed_leveled_result1))\n","update_points = np.ceil(np.arange(num_steps) / batch_size).astype(int)\n","\n","pic_filename = \"results/{}/UCB_transform_timesteps.png\".format(output_dir)\n","plt.plot(mean_smoothed_leveled_result1, lw=3, label='batch, batch size = ' + str(batch_size))\n","plt.plot(mean_smoothed_leveled_result, lw=3, ls='-.', label='online policy')\n","plt.plot(mean_smoothed_leveled_result[update_points], lw=3, ls='-.', label='dumb policy')\n","plt.legend()\n","plt.xlabel('time steps')\n","plt.title(\"Smooth Cumulative Reward averaged over {} runs\".format(num_experiments))\n","plt.ylabel('smoothed reward')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.savefig(pic_filename, bbox_inches='tight')\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":359,"referenced_widgets":["07d74cc8c1034e8e88e0ba68681e7d83","20755197a75045769c76c25cf9997309","3aaecc6ab5334dbaaff91f3fd3d2dc92","ccdfd084e55f4d48aa8d62ff900c54dc","602801fccd6d4538bd22a5494c8a538e","81b219b6e4414ff888646a18f038c6d2","4622f6b8c7914be18ec086e119b14510","c603ecbfec974667805abf5f8366dde5","508403ada600443f97467dde6fb0dd3d","9840772f43134cee8eabacd2509f9fd7","4e96a7f4873f4360b2118e7a3ddbfc48","c19256a397ea4e79b3a0cbf5a424a3ea","4d1f5676bbaa4b3d8b81cc7be3cf6da3","eae2220f0119401a8759637bcf0e9f4c","4aa44dd37d5243dfa24fea8f3dc6839b","2deab0a58cd04226b834a99ee4ff4c90","16acd859901c4b8aa5f6b4473b74e249","e4edb80a747a4e9684a556ff96563e7f","3d74bbdd72dc4e0aadb4bd517358bbe7","809de5d470244a2b90daffd0477c28a4","4e13cae2ba7a44daa51794431d7af0c0","2780d1d40a0644c498e860cf4839ac25"]},"id":"tXey4Vyi06pm","executionInfo":{"status":"ok","timestamp":1639148649109,"user_tz":-330,"elapsed":32385,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"77e181e9-b27e-474c-834d-4db5a099aeb1"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"07d74cc8c1034e8e88e0ba68681e7d83","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/20 [00:00"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### LinUCB by batches"],"metadata":{"id":"q7xl1u_F45d5"}},{"cell_type":"code","source":["num_experiments = 20\n","data_dir = 'data/mushroom_data_final.pickle'\n","env_info = {'pickle_file': data_dir}\n","output_dir = 'LinUCB/dynamic_by_batches'\n","\n","agent_info = {'alpha': 2,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': 1}\n","experiment_parameters = {\"num_runs\": num_experiments}\n","\n","agent = LinUCBAgent\n","environment = ReplayEnvironment\n","\n","# run online agent\n","online_result = run_experiment(environment, agent, env_info, agent_info,\n"," experiment_parameters, True, output_dir)\n","# smooth and average the result\n","smoothed_leveled_result = smooth(online_result, 100)\n","mean_smoothed_leveled_result = np.mean(smoothed_leveled_result, axis=0)\n","mean_smoothed_leveled_result = mean_smoothed_leveled_result[~np.isnan(mean_smoothed_leveled_result)]\n","\n","# run batch agent\n","batch_sizes = np.logspace(1.0, 2.7, num=20).astype(int)\n","actual_regret = []\n","upper_bound = []\n","for batch in batch_sizes:\n"," agent_info_batch = {'alpha': 2,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch}\n"," batch_result = run_experiment(environment, agent, env_info, agent_info_batch,\n"," experiment_parameters, True, output_dir)\n"," # smooth and average the result\n"," smoothed_leveled_result1 = smooth(batch_result, 100)\n"," mean_smoothed_leveled_result1 = np.mean(smoothed_leveled_result1, axis=0)\n"," mean_smoothed_leveled_result1 = mean_smoothed_leveled_result1[~np.isnan(mean_smoothed_leveled_result1)]\n","\n"," actual_regret.append(mean_smoothed_leveled_result1[-1])\n","\n"," # fetch dumb result\n"," M = int(len(mean_smoothed_leveled_result1) / batch)\n"," upper_bound.append(mean_smoothed_leveled_result[M])\n","\n","pic_filename = \"results/{}/UCB_transform_batchsize.png\".format(output_dir)\n","plt.plot(batch_sizes, actual_regret, label='actual regret')\n","plt.plot(batch_sizes, [mean_smoothed_leveled_result[-1]]*len(batch_sizes), label='online policy')\n","plt.plot(batch_sizes, upper_bound, label='dumb policy')\n","plt.legend()\n","plt.title(\"Reward as a f-n of batch size (each point is averaged over {} runs)\".format(num_experiments))\n","plt.xlabel('batch size (log scale)')\n","plt.ylabel('reward')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.savefig(pic_filename, bbox_inches='tight')\n","plt.show()"],"metadata":{"id":"XxBGmxib45aV"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### LinTS"],"metadata":{"id":"zaZs-8O31BHz"}},{"cell_type":"markdown","source":["#### LinTS by timesteps"],"metadata":{"id":"apXIaCEY4nih"}},{"cell_type":"code","source":["num_experiments = 10\n","batch_size = 100\n","data_dir = 'data/mushroom_data_final.pickle'\n","env_info = {'pickle_file': data_dir}\n","output_dir = 'LinTS/dynamic_by_timesteps'\n","\n","agent_info = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': 1,\n"," 'replay_buffer_size': 100000}\n","agent_info_batch = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch_size,\n"," 'replay_buffer_size': 100000}\n","experiment_parameters = {\"num_runs\": num_experiments}\n","\n","agent = LinTSAgent\n","environment = ReplayEnvironment\n","\n","online_result = run_experiment(environment, agent, env_info, agent_info,\n"," experiment_parameters, True, output_dir)\n","batch_result = run_experiment(environment, agent, env_info, agent_info_batch,\n"," experiment_parameters, True, output_dir)\n","\n","smoothed_leveled_result = smooth(online_result, 100)\n","smoothed_leveled_result1 = smooth(batch_result, 100)\n","\n","mean_smoothed_leveled_result = np.mean(smoothed_leveled_result, axis=0)\n","mean_smoothed_leveled_result1 = np.mean(smoothed_leveled_result1, axis=0)\n","\n","num_steps = np.minimum(len(mean_smoothed_leveled_result), len(mean_smoothed_leveled_result1))\n","update_points = np.ceil(np.arange(num_steps) / batch_size).astype(int)\n","\n","pic_filename = \"results/{}/TS_transform_timesteps.png\".format(output_dir)\n","plt.plot(mean_smoothed_leveled_result1, lw=3, label='batch, batch size = ' + str(batch_size))\n","plt.plot(mean_smoothed_leveled_result, lw=3, ls='-.', label='online policy')\n","plt.plot(mean_smoothed_leveled_result[update_points], lw=3, ls='-.', label='dumb policy')\n","plt.legend()\n","plt.xlabel('time steps')\n","plt.title(\"Smooth Cumulative Reward averaged over {} runs\".format(num_experiments))\n","plt.ylabel('smoothed reward')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.savefig(pic_filename, bbox_inches='tight')\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":359,"referenced_widgets":["a2c68a56898548f8a7e190dcf2d034f4","e009631910e042a2a07afe8cf0e6d118","a850623290b14d07838e4203a29f9ba2","9f140b090cb44361b39a6e09205c087c","36fc4560cb99439c945c620138005a1b","638ee1bca5d64cc7966556e3fb64c06e","a454e1e1460046e3a98a4b28c48ecda0","d0a15e2054af430bad52b6fc28fd0994","f9295dd431fe4587a816d6c59f464399","41ca699547f14bda8a83777d78e6bc86","c29bc574f0cd46309d8eb6ec40378025","858e460d943744248a4109a5c328d77c","700c0953693b48bea968c6c67964684d","c70158af000c40e59b588e745be4a2dd","21b1b038a25f469295ea8e2b75c470ae","c20bc14a583e452ba263c4e4f4f3147e","e1199401ceee4d0ca1aaa72020618241","57b794458691470ba34efd35f0cd8a9c","4bd9a0cc11664bf4870c7fc46d151bda","301a50e882864065b9108e0a0cf0edde","93f8c74e75504aadbf1cdec1c5b6a43f","a4d3c862e74242e4beeaaa880f4e3fc8"]},"id":"6uyZk7PT1Cjy","executionInfo":{"status":"ok","timestamp":1639149620282,"user_tz":-330,"elapsed":952413,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"72dd4d89-25ac-408a-c033-d05d6bee7628"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a2c68a56898548f8a7e190dcf2d034f4","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/10 [00:00"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### LinTS by batches"],"metadata":{"id":"pVbE43Lm4p5H"}},{"cell_type":"code","source":["num_experiments = 20\n","data_dir = 'data/mushroom_data_final.pickle'\n","env_info = {'pickle_file': data_dir}\n","output_dir = 'LinTS/dynamic_by_batches'\n","\n","agent_info = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': 1,\n"," 'replay_buffer_size': 100000}\n","experiment_parameters = {\"num_runs\": num_experiments}\n","\n","agent = LinTSAgent\n","environment = ReplayEnvironment\n","\n","# run online agent\n","online_result = run_experiment(environment, agent, env_info, agent_info,\n"," experiment_parameters, True, output_dir)\n","# smooth and average the result\n","smoothed_leveled_result = smooth(online_result, 100)\n","mean_smoothed_leveled_result = np.mean(smoothed_leveled_result, axis=0)\n","mean_smoothed_leveled_result = mean_smoothed_leveled_result[~np.isnan(mean_smoothed_leveled_result)]\n","\n","# run batch agent\n","batch_sizes = np.logspace(1.0, 2.7, num=20).astype(int)\n","actual_regret = []\n","upper_bound = []\n","for batch in batch_sizes:\n"," agent_info_batch = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch,\n"," 'replay_buffer_size': 100000}\n"," batch_result = run_experiment(environment, agent, env_info, agent_info_batch,\n"," experiment_parameters, True, output_dir)\n"," # smooth and average the result\n"," smoothed_leveled_result1 = smooth(batch_result, 100)\n"," mean_smoothed_leveled_result1 = np.mean(smoothed_leveled_result1, axis=0)\n"," mean_smoothed_leveled_result1 = mean_smoothed_leveled_result1[~np.isnan(mean_smoothed_leveled_result1)]\n","\n"," actual_regret.append(mean_smoothed_leveled_result1[-1])\n","\n"," # fetch dumb result\n"," M = int(len(mean_smoothed_leveled_result1) / batch)\n"," upper_bound.append(mean_smoothed_leveled_result[M])\n","\n","pic_filename = \"results/{}/TS_transform_batchsize.png\".format(output_dir)\n","plt.plot(batch_sizes, actual_regret, label='actual regret')\n","plt.plot(batch_sizes, [mean_smoothed_leveled_result[-1]]*len(batch_sizes), label='online policy')\n","plt.plot(batch_sizes, upper_bound, label='dumb policy')\n","plt.legend()\n","plt.title(\"Reward as a f-n of batch size (each point is averaged over {} runs)\".format(num_experiments))\n","plt.xlabel('batch size (log scale)')\n","plt.ylabel('reward')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.savefig(pic_filename, bbox_inches='tight')\n","plt.show()"],"metadata":{"id":"K63E-61k4p03"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### CMAB demo on Mushroom dataset"],"metadata":{"id":"w5S8ABzm1JMM"}},{"cell_type":"code","source":["data_dir = 'data/mushroom_data_final.pickle'\n","env_info = {'pickle_file': data_dir,\n"," 'seed': 1}\n","# init env\n","environment = ReplayEnvironment\n","\n","# init random agent\n","random_agent_info = {'num_actions': 2}\n","ra = RandomAgent()\n","ra.agent_init(random_agent_info)\n","\n","# learn LinUCB agent\n","agent_info = {'alpha': 2,\n"," 'num_actions': 2,\n"," 'seed': 1,\n"," 'batch_size': 1}\n","\n","agent = LinUCBAgent\n","rl_glue = RLGlue(environment, agent)\n","\n","for i in range(4): \n"," rl_glue.rl_init(agent_info, env_info)\n"," rl_glue.rl_episode(0)\n","UCB_agent = rl_glue.agent\n","\n","# learn LinTS agent\n","agent_info = {'num_actions': 2,\n"," 'replay_buffer_size': 200,\n"," 'seed': 1,\n"," 'batch_size': 1}\n","agent = LinTSAgent\n","rl_glue = RLGlue(environment, agent)\n","\n","for i in range(4): \n"," rl_glue.rl_init(agent_info, env_info)\n"," rl_glue.rl_episode(0)\n","\n","TS_agent = rl_glue.agent\n","result = []\n","result1 = []\n","result2 = []\n","\n","exper_seeds = [2, 5, 10, 12, 54, 32, 15, 76, 45, 56]\n","for seed_ in exper_seeds:\n"," dataset = BanditDataset(pickle_file=data_dir, seed=seed_)\n","\n"," eval_info = {'dataset': dataset, 'agent': UCB_agent}\n"," eval_info1 = {'dataset': dataset, 'agent': TS_agent}\n"," eval_info2 = {'dataset': dataset, 'agent': ra}\n","\n"," evaluator = OfflineEvaluator(eval_info)\n"," evaluator1 = OfflineEvaluator(eval_info1)\n"," evaluator2 = OfflineEvaluator(eval_info2)\n","\n"," reward = evaluator.eval_run()\n"," reward1 = evaluator1.eval_run()\n"," reward2 = evaluator2.eval_run()\n","\n"," result.append(reward)\n"," result1.append(reward1)\n"," result2.append(reward2)\n","\n","labels = ['UCB agent', 'TS agent', 'Random agent']\n","for i, res in enumerate([result, result1, result2]):\n"," for elem in res:\n"," plt.plot(elem, linewidth=0.1)\n"," avg = [float(sum(col))/len(col) for col in zip(*res)]\n"," plt.plot(avg, label=labels[i])"],"metadata":{"id":"pHKrIm_Q1R7S"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["labels = ['UCB agent', 'TS agent', 'Random agent']\n","for i, res in enumerate([result, result1, result2]):\n"," for elem in res:\n"," plt.plot(elem, linewidth=0.1)\n"," avg = [float(sum(col))/len(col) for col in zip(*res)]\n"," plt.plot(avg, label=labels[i])\n","plt.legend()\n","plt.ylim([0.1, 0.7])\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":269},"id":"ZbxPyHo_1Zgo","executionInfo":{"status":"ok","timestamp":1639152031963,"user_tz":-330,"elapsed":1780,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"0b963035-788e-4b43-b73d-9ee1df8e7cc7"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eZxcxXXo/617b+/TMz27ZtNoRStiJLQghI3AYXEScEJIsEP8DHmGxDE4v9ghtvNxjE2eE7/fI37Efn7YEDAhJvjn2AnBxo5x2INZJECgBW1Io9n3md6Xe/vW74+e2/SMepPUtLvHc+yh1d2nT31P1e3qulWnTgkpJYuyKIuyKIuyMEX5ZQMsyqIsyqIsynsni538oizKoizKApbFTn5RFmVRFmUBy2InvyiLsiiLsoBlsZNflEVZlEVZwLLYyS/KoizKoixgKaqTF0JcLYQ4IoQ4LoT4XJb3/7cQYt/s31EhxEzpURdlURZlURblTEUUipMXQqjAUeAKYADYA3xESnkoh/7twGYp5R+WmHVRFmVRFmVRzlCKGclvB45LKU9IKRPA94AP5dH/CPBoKeAWZVEWZVEW5dxEK0KnA+jPeD4A7MimKIToBpYDT+d4/1bgVgCPx3PhihUrAFBVFSEEhmFYethsNhKJRPqzdrsdXdcZDSYQ0RCoNpprXUgkQtNOs6EoCpqmZbVh3b3YbDaSySSmaaYqQ0tVR6YNVVXRdX2OjUybxdqIx+MoipLTN8Mw5tiQUpJMJrNy5Ksfy7f5NlRVRVGUM7Jh8WbaKLadctXxe9lOVh2fSzsVqp9St5MQAkVRzqmdbDYbpmmWpZ1isdicOv5lfZ/OpJ3m1/Ev6/t0Ju1kmiaKomS18frrr09IKZspUorp5M9EPgz8QEqZzPamlPI+4D6ArVu3yr17956mE4/HcTgcOQv4Hz8+ROzFnyBr2/l/fnsHvgYvtpaWM7ZTbp3e3l6WLVtWMTylYK423nLzLETmauOFhccshDiV98PzpJjpmkGgK+N55+xr2eTDnONUjaqqed8XIvUoJSR146ztlFuntbW1onhKwVxtvOXmWYjM1cYLC5e5WCmmk98DrBZCLBdC2El15I/PVxJCrAXqgZfOBci6xcolYraX15P59QrZKbdO5m1cJfCUgrnaeMvNsxCZq40XFi5zsVKwk5dSGsBtwM+At4HvSykPCiHuEkJcm6H6YeB78hzTWlqdeM73Zx/NAqUUslNunenp6YriKQVztfGWm2chMlcbLyxc5mKlqDl5KeVPgJ/Me+2L855/qRRABStg3tvxaBTb2dgps04xUm3M1cZbyrKqkdkwDAYGBojFYjl1pJR5bRmGwdtvv523nEI2yq1TrcwnT56ks7MTmy1bD1e8lHrh9ZwlmUymV9SziZjt5a2BfDQUpOYs7JRbx+fz5X2/3DylYK423nLzVBrz0NAQdXV1LFu2LGcnY0V15BLDMAqWU8hGuXWqkVnXdfx+PwMDAyxfvjyvbiGpuLQGhRrj3WszhZ7Us89dFbJTbh2n01lRPKVgrjbecvNUGnMikaCxsTHvKLLQCLNQ51SMjXLrVCOzqqo0NjbmvesqVsreyQshrhFC3DczM4NhGOi6TiKRwDAM4vE48XicaDSKlJJIJAJAOBwGSD2fnfI3If3ZRCKBrutpG8lkklAolNWG9RiNRonH48RiMZLJZNqGZTOZTBKLxdI82WxEIhGklIRCIZLJJPF4PKtPpmnS39+f0ycpZZrHspHNp1gsRiKRyOuTaZppnlw+maZJIBDI61MsFmNoaCivT8FgsKJ86uvrK+hTJk82n6LRaFaeTJ8snnw+SSlz1rFlw6rjfD5Fo1F0Xc/rUzKZJBwO5/VJSolpmnP2PSSTyfTr1mPm3/z3pJTpBcFMG0BOG/lszbeRjSfT5tnasOLO89nIZisXRy5b2erlbOo4mUyi63ra5vxr70ylYFqD90pyxckXmq/6f//jMIHnfkzI1cHnfnsL+tQQXZdedsZ2yq1TTKxutTFXG2+5eSqN+e2332bdunXnZKeYGO9Kq+NqZs7WZkKI16SUW/MayJCKm66xRs25JB0nP/sPY8Z/VnbKrVPMbXm1MVcbb7l5Ko25FOF9xUx95LLR29vLxo0b5+h86Utf4u67707r3H333axdu5aenh62b9/Oww8/DMDu3btZs2YNPT09rFu3jvvuu69on86FOZvOPffck77TOxc7+aQY5mKl4jp5t9tdlJ51/yHj2eesirFTTp0lS5ZUFE8pmKuNt9w8lcZcTMdRaKNOMZEeZ7sh6Fvf+hY///nPefXVV9m3bx9PPfUUmTMNjzzyCPv27ePFF1/ks5/9LIlEoqiySs2cr5Mv1Waoc42oyZSK6+StOcdcYkXXYLMR94fSc/RnaqfcOr29vRXFUwrmauMtN0+lMRczgrTmknNJPB4/Zxu5dP7mb/6Ge++9l9raWgA8Hg8f+9jHTtMLhUJ4PB5UVT3Nzl133cW2bdvYuHEjt956K1JK4vE4e/bsYdOmTfT09HDHHXek7yiSySR33HEH27ZtY9OmTXz7298G4Nlnn2X37t1cf/31rF27lhtvvBHDMPj617/O0NAQl112GZdddvo08dn6Pl+KqedipeJCKD0eT97309M1Nht6KIaqZf/FK2Sn3DrFSLUxVxtvKcuqRubMkfyXf3SQQ0OBM2aR0kSI7GPD9e213HnNhrMazQYCAYLBIFbSwmw6N954Iw6Hg2PHjnHPPfdkLee2227ji19MbeH56Ec/yo9//GOuvPJKbr75Zu6//3527tzJ5z737pEYDzzwAHV1dezZs4d4PM6uXbu48sorAXjjjTc4ePAg7e3t7Nq1i5dffplPfepTfO1rX+OZZ56hqampoF/F+P5eSxWO5N+VZNJA2LN38pU2YitGqo252nhLWVY1MhcXmXHugRi5RqqZi42ZOrkWIefbeeSRR3jrrbfo6+vj7rvv5tSpU6fpPPPMM+zYsYPzzz+fp59+moMHDzIzM0MwGGTnzp0A/P7v/35a/8knn+Thhx+mp6eHHTt2MDk5ybFjxwDYvn07nZ2dKIpCT08PJ06cOGvfz1SnlFL2kbwQ4hrgmpUrV2IYBlJKpJTpVKAul4toNIrT6SQajeJ2uwmHw3g8nlQoEQASicSIJ1Bcc9P4WptCVFVNhx1l2rAeo9EoLpeLWCyWTm9qXWxSynQqUovH5XKdZiMSieByudK3jYZhpMvN9Mlms9Ha2pqVx7IRi8VwuVzE43FUVU2Hu2X6pOs6brebSCSS0yeHw5HmyeWT3W5Pj+py+RSPx+nq6krzZPNJUZSK8slK6pTPJ6fTmdeneDyelSfTJ4fDgaZp6XS22XxyOBxpzlw+JRIJurq6SCQSOX2Kx+NzPpvNJ5vNlubJ5VPmAm8ymeTOazaQTCbTbSiEmPM4+1097T3TNNNtkflo1YO10Wf+D4oQgvr6eqanp+fYmJqaoru7m9raWmpqanjnnXdYvnx5uk4tf6y6MU2TpqYmNm/ezMsvv8z111+f9knXdf7kT/6EPXv20NnZyZe//GWi0Sh2ux1gDqP1XErJPffcw6//+q/P8emFF17AbrfPqc/M9MXWZzPrJ/NOKdPW/DrOrJ9cdexwONI/BvOvvTOVso/kpZQ/klLe6vP50DQNm82G3W5H0zQcDgfxeByXy4UQIr2YZN2Kut3ud3/1ZWrqxlBV7DZb+kK3OgSrwubbsB4zv/SqqmK327HN2rHb7aiqmu4QXC5XVhsWj/XFtL78831SFAW/35/XJ4vHsmHxZPrkdDqJxWJ5fbIuqHw+ZV6MuXxyOp2Mj4/n9QmoKJ/8fn9BnxKJRF6frC9TPp+sL2k+n6xOKp9PVh3n88kaZOTzyepk8/mUyWNNF1j5zK0BklXPiqLMeS3z0eqcMm0AOW1k2qqtraWtrY1nn30W0zTx+/38x3/8B+9///sB+PznP89tt91GKBRCCEEoFOKf/umf5lxriqIQjUbZt28fq1atmuOTtXGoubmZSCTCD3/4Q4QQ1NTU4PV62bNnD0IIvv/976eZr7rqKu677770D//Ro0fTdqw6s/itz3i9XsLh8Gn1Y33mXOvY+pG3bMy/9s5UKm5OvtAvlcj4b1JLYlMUME2Y53wxv3jl1Clm51q1MVcbb7l5Ko25GCkUv13MlE8+Gw8//DCf/OQn+fSnPw3AnXfeycqVKwH4xCc+QSgUYtu2bekfvc985jPpz954443pgcNNN93EhRdeOCf6xufzccstt7Bx40aWLFnCtm3b0swPPPAAt9xyC4qicOmll1JXVwfAxz/+cXp7e9m6dStSSpqbm3nsscfy+nXrrbdy9dVX097ezjPPPFO072eiczabnnKWV2mboWKxWN7Y4Xv+8ygTT/07041r+KO1dhxulfMuuRQxe0tWrJ1y6xSz6aXamKuNt9w8lcZ86NAh1q9fn1enUF6VYjYWVVoemHg8jq7r1NSkslx99atfZXh4mL//+7+vaOZSbYaquJF8ofhQMS8NZU1DA5LTklMWFWdaTp1i4qGrjbnaeMvNU2nMxUihUWYx5VRaHhibzcZjjz3G3/7t32IYBt3d3Tz00EO/NJ5imUslFRddU2jlOVU/AhCz812z0zVnaKfcOsXcllcbc7Xxlpun0piLkUJ39sVMIxQzO1BOHdM0ueGGG9i3bx8HDhzgiSeeoLl57hGplchcKqm4Tr6YXzmQGKaOqmkIVc26IarSfplnZmYqiqcUzNXGW26eSmMuhZQ7/K8U8qvOXHFZKDOz5uXKBAhgSgmqhlQUErNzbtkyARbKblgoC6XFk83G/EyA+bIbWieuF8puWChjYzE+Wfrn6lMmTzafcvH8snyyMiSeSzsVuvbOxKdcdZx57Vnc53rtWX7k88n67uTLkAjkzZCYzYb1mWw2ziVjo/WXjyebrVx6Z5LJcr5P81lycRTDU6iO53MsuCyUiUQiHdeaTb7x1DFG//PfGfIt539cvQ6bptHQ3Y5aM/fokEJ2yq3j9/vTK/qVwFMK5mrjLTdPpTGXYuG1Gg/gqGbmBZmFstCW39RdqcSU4PH4UJPZp2sq7fT1Ql/AcvOUgrnaeMvNU2nMxUihaZ9qPIBjoTIXKxXXyeu6nvd9q4JMwCYcyKCZdeG1kJ1y64yOjlYUTymYq4233DyVxlyMFLqzL6acXDYmJyfp6emhp6eHtrY2Ojo60s+//OUvs2HDhnQSsVdeeaVki5hnwvzQQw8xNDR01mWVk7lYqbgQymJGNiBS6Q0kmFM6yWAQdd4tbzF2yqlTjFQbc7XxlrKsamQuRkqxgJvLRmNjI/v27QNSm6C8Xi9//ud/zksvvcSnP/1pXn/9dRwOBxMTE3NSPbzXvJl2HnroITZu3Eh7e/tZlVVJi+SWVNxIvlCKTSFAVeOYMhUzL8NJzFDojO2UW8dKjVApPKVgrjbecvNUGnMxUmiUWcw0wpmu8w0PD9PU1JTeZNXU1ER7e/tpdu6//362bdvGBRdcwO/8zu+kFyTfeecdLrroIs4//3y+8IUvpDc9Afyv//W/2LVrF5s2beLOO+8EUhvQ1q1bxy233MKGDRu48soriUQi/OAHP2Dv3r3ceOON9PT0nHYQSzlH8qWcrqm4kXyhi14g0LQEwZiCUBWSugHm6fORxXx5yqnT0tJSUTylYK423nLzVBrznBHkTz8HI/tP0ynUtWhZtx7OypLz4YNfPeM58CuvvJK77rqL8847j1/7tV/jhhtu4NJLLz3NznXXXcctt9wCwBe+8AUeeOABbr/9dv70T/+UP/3TP+UjH/kI3/rWt9L6Tz75JMeOHePVV18F4Nprr+X5559n6dKlHDt2jEcffZT777+f3/u93+Pf/u3f+IM/+AP+z//5P9x9991s3Xr6umYxfpVKp5iD2YuViguhDAaD+Q/yRiKkYCasoCcNYmYcXcrTwtimpqYKhuYFg8GCIZQWTzYb1khiamqqYGje8ePHC4bmBYPBgqF5oVCoYLihxZMvNG9ycjKvT7FYjJMnT+b1aXJysqJ8slLE5vMpkydXuGE2nvlhbNPT0wVDKHPVcWYI5cmTJwuGUIbD4YIhlBZPvhDK+aF58mz+Z76bB3b+oyQVBphMJguGUFp/kPqBeu2117j33ntpamrihhtu4Dvf+U46q6alu3//ft73vvdx/vnn88gjj3DgwAF0Xeell17iuuuuA+CGG24AUhEsP/vZz3jyySfZvHkzW7Zs4fDhwxw9ehTTNFm+fDmbNm1CSsnmzZs5ceJEwRBKKwtlvhBKay49Xwhlpp1cIZTWdZDt2jtTKftIXkr5I+BHW7duvWX+r5WVOc+S3FkoZ2PlbRqGT+Bsbk5vA7ZsNjQ0ZLWRmQkwU7JFJ1jZAC3Jlgkws6xsNiwem82WM2NjNp5sNubbz+VTPh7rtcbGxrw+WVkQs+UpsXgsG5XikzUvnc+nXPldLJ75zLl8qq+vL+hTrjq2bFh1nG0+3bJRiMfyJx+PZWNO5sgPfjXXeDyvJGZzqlifzfWYTazRu5Vx0RrRWlwf+MAH+MAHPsCmTZv4x3/8R26++eY5n7v55pt57LHHuOCCC3jooYd49tln09/7bFkxIZXZ8qabbppzHff29s7JpGqlY85lwyo/2+g60yfLVjae+b7ns6GqajptORT3fconFTcnX9RBCyI1fjCEYFxMnbWdcuoUs9hSbczVxltunkpjLmYUWIqdlsXYyGQ5cuRI+i4MYN++fXR3d59mJxgM0tbWhq7rPPLII+myLrroIn74wx8C8L3vfS+tf9VVV/Hggw8Sml2zGxwcZGxsLC+P1+slGAyetV+Lh4YUIQWP/wOkSP0lTJMZW/ZT6ivtmLfu7u6K4ikFc7Xxlpun0phLcZB3MSmNi4nZz2QJhULcfvvtzMzMoGkaq1at4r777jvNzl//9V+zY8cOmpub2bFjB8FgEFVVueeee/iDP/gDvvKVr3D11VenN5ddeeWVvP322+zevRuAmpoavvvd72bls3huuukm/viP/xiXy8VLL700Z/Rczv0TpUodDTBn+3CuP+Bq4AhwHPhcDp3fAw4BB4F/LmTzwgsvlNkkHA5nfd2Sbz93XN71pb+Su/78Mdn7zqT80c9/KBMjo2dsp9w6Q0NDFcVTCuZq4y03T6UxHzhwoKCOYRh534/H4+dso9Q64XBYmqYppZTy0Ucflddee+0cnUpkLiQW86FDh057D9gri+i3rb+CI3khhAp8E7gCGAD2CCEel1IeytBZDXwe2CWlnBZCFA4ZyCHFRNeI2eUew67iUrPrV1oURanC6SqJudp4y81TacyliPyQJQr/K6XOa6+9xm233YaUEp/Px4MPPjhHpxKZC0kxzMVKMXPy24HjUsoTUsoE8D3gQ/N0bgG+KaWcngXMPvFVhBQTJw+ppdeEYeKx12TVq7R46GKk2pirjbeUZVUjczEdRyk6l1KVU6zO+973Pt58803eeustnn/+eVatWlUU53vFUwqdUkoxnXwH0J/xfGD2tUw5DzhPCPGiEOJlIcTVZwtUbLJ8CcTjSUSCrEv6lXY4RLYddL9MnlIwVxtvuXkqjbkYWaiHhlQST7kPDSnVwqsGrAZ2A53A80KI86WUc5JlCyFuBW4F6OzspLe3F0iFf9ntdkZHRzEMA6/XS0tLC6dOnbI+R3d3N8PDw0xPTaU79cnpAHJihoH+fhqdTjRNS6+e22w22tra6OvrA1KLHV1dXQwNDaVT0jY1NaVj8yEV8qYoCuPj40BqocblcqWfa5pGZ2cnAwMDGIaB5cfY2FjaZnNz85w4dK/XS21tLceOHcPj8WC322lvb6e/vz+9yr506VImJiYIBAJomkZLSwuGYTA1lYocsk6yHxoawjAMPB4PbW1tnDp1Kj0q6O7uZmxsjGg0imEYdHR0kEgkmJ6eBlLnXzqdTkZGRubUh9UGAMuWLWNkZCR9kIUVcmjlPM9sJ6s+Ojo6sraTNbJsbGxE13UCgQCQCoXMbCe3243X603bzNZOHR0dTE5OprmytZPP50vXca52mpmZSS/u5WqnwcFBDMPA7XbnbKdIJIJhGLS3t+dsJ0jdli9dujRnO0Fqgc3tdudsJ6fTSX19Pf39746z5rfTkiVLCAQC6dj++e3kcrnS15TVLg6Hg0Qikeay2WzpGG4hRDpc0Ko/RVHQNI14PJ4+cNput5+VDeszVvhopg273Z6OI5dSpjs7y4aqqiiKMie3y3xfMm1Y16UVlpjNhsVqXW/ZbNhstnT8fzYbiqKkQzGtTtzhcKDr+hwbmfWTzYbFkZmpMvP7VMygYL4UTDUshNgJfElKedXs888DSCn/NkPnW8ArUsrvzD5/itQC7Z5cdnOlGi6UFvSB/zrJyNMP8KPgNu6+cTvuseNsWLMKR3fbGdkpt04xZ3lWG3O18Zabp9KYi0k1LKXMO9Is5ozXQjbKrVPNzOVKNbwHWC2EWC6EsAMfBh6fp/MYqVE8QogmUtM3J4qFyJSCPzqzj4owiScMMMHIkrGtGufYqo252nhLWVY1MpeqrFLYWKh1XJVz8lJKA7gN+BnwNvB9KeVBIcRdQohrZ9V+BkwKIQ4BzwB3SCknzwao8J0FICQKkoRuYDhhKnZ6UZXWaNaOzUrhKQVztfGWm6fSmEsh55pTRVVVenp62LRpE9dcc01Rxx8WIw899BC33XZb1vdKmQfmvZB77rknPdVmSdlz10gpfyKlPE9KuVJK+ZXZ174opXx89t9SSvlpKeV6KeX5Usrv5beYB6hAeFHmTU4kHiWeMBkc6z9Nr9JCoopptGpjrjbecvNUGnMx8l6mGobU+sC+ffvYv38/DQ0NfPOb3zwnllKnGv5l6GTr5EspFZfWoNCW31QFCaQwCcTD6DaYmZpN8p94d2t3ObcgF6OTazv1L4unFMzVxltunkpjLkYK3RFYi5fnYsPS2blzJ4ODgwC8+uqr7Ny5k82bN3PxxRdz5MgRpJQ89NBDXHfddVx99dWsXr2av/iLv0jb+M53vsOaNWvYvn07L774Yvr13t5eLr/8cjZt2sQHPvABTpxIzRzfdNNNfOITn+Ciiy5ixYoVPPvss/zhH/4h69at46abbsrKedddd7Ft2zY2btzIrbfemvZtz5496QNO7rjjDjZu3JiunzvuuINt27axadMmvv3tbwPw7LPPsnv3bq6//nrWrVvHjTfeiJSSr3/96wwNDXHZZZdx2WWXnVE9FysVdx9TXOiQBCQxXScuEigzs/nkA8PQtKpoO+XUKUaqjbnaeEtZVjUyZ8r/fPV/cnjq8Bmz5DufdG3DWj67/bNFjWZN0+Spp57iv//3/5767Nq1vPDCC2iaxn/+53/yl3/5l/zgBz8AUrls3njjDRwOB2vWrOH2229H0zTuvPNO9u7di8/n47LLLmPz5s0A3H777XzsYx/jYx/7GA8++CCf+cxnePzx1DLi9PQ0L730Eo8//jjXXnstL774Iv/wD//Atm3b2LdvHz09PXM4b7vtNr74xS8C8NGPfpQf//jHXHPNNdx8883cf//97Ny5k8997nNp/QcffJC6ujr27NlDPB5n165dXHnllQC88cYbHDx4kLa2Ni655BJefPFFPvWpT/G1r32NZ555hqampjNpiqKl4lINR6PRwqmGZ22FIzGCsRBaPJlK9xr1p9O9BgKBrDYy071Go9GCqYYtvWw2rBSggUCgYKphVVVz+iRn04hGo9GCaXljsVjBtLwWT74Utn6/P69PsVgMp9OZ1ye/319RPlk5QfL5lMmTKy1vNp756V4DgUDBVMPWfHO+VMNW+F++VMPxeLxgquFgMFgw1bA1Ck2nGs4y4i4i2m6OXqa+lWo425+UMs1iHf83MjLCFVdcQTKZxO/3c/3117Nx40b+7M/+jIMHD6bT9F5++eV4vV4cDgfr1q2jt7eXl156id27d9PQ0IDdbud3f/d307699NJLfPjDH0ZKyY033sgvfvGLNMNv/uZvIqVk48aNtLa2snHjRqSUrF+/Pj3iz0w1/PTTT7Njxw7OP/98nn76aQ4cOMDU1BTBYJCLLroI0zT5yEc+kq6DJ598kocffpienh527NjB5ORkOr3x9u3b0yGQF1xwASdOnJhTj5mphhVFWbiphu12e/pCypbCVhECUv9HxwQkyaSBGhlFURW02VCpurq6rKl9M9O9ZoYy5Uphm6mTK4WtVVa+VMMdHR0FUw3nCq3KTGGbj8fa2p6Px3rN5/Pl9cnpdNLS0pJ11Gbx+Hy+ivKpo6OjoE+FeOYz50o1bPHk8gneTf+bL9Vwa2tr1jrOTDWcj8dKNVxbW5uTxyovM7XvZ7d/9rQyoXB437mECAoh0nPy4XCYq6++mm9+85t86lOf4q/+6q+4/PLLeeyxx+jt7WX37t2oqooQIp2S2fLJ6gQz6ynb88y0vtZzy5aqqqmUybPXlKqqc2LgARKJBJ/85CfZu3cvXV1d3Hnnnel9ApY/1l+mfOMb3+Cqq66a89qzzz6bTm8spUTTNEzTzJqC2Iqfn98PLphUw/OP3DpNrEoBjKQgiYmBwcDE4dR0TbF2yqxjbcqqFJ5SMFcbb7l5Ko25mFFgIZ3MDUPnUo7T6eTrX/86f/d3f4dhGPj9/vSP9EMPPVTQzo4dO3juuecYHx9H13X+5V/+Jf3exRdfnE45/Mgjj7Br166CPNnuYKzNZk1NTYRCoXQ6Y5/Ph9fr5ZVXXgHmpje+4ooruPfee9MbnI4ePXpaGuj5fmVLb1xMPRcrFdfJW79auUQRIElVUjJqIjSIGBFiRggCqUWcpN9f0E4xZZVSpxipNuZq4y1lWdXIXIpUw8VIsel2N2/ezKZNm3j00Uf5i7/4Cz7/+c+zefPmOTtTc0lbWxtf+tKXuOSSS9i1a9ecDUPf+MY3+M53vsOmTZv4p3/6J+6+++6CPNnuPHw+H7fccgsbN27kqquuYtu2ben3HnjgAW655RZ6enoIh8Pp9Ma33nor69evZ8uWLWzcuJE/+qM/Om0Rdb5ft956K1dfffWchddSSsEdr++V5NrxGg6H8+bG/udX+uj9+b38NLiFHct8vH9lAl7QQZEAACAASURBVPXN11j9mxdw3onn4Tf+jpl//Te0K6+Yc6BvNilUVil1+vv76erqqhieUjBXG2+5eSqN+eDBg2zYsCGvTjKZzNu5JhKJrCdZnYmNcuu8F8yhUCjdv3z1q19leHiYv//7vy85c7l2vJZVCh4aMvuDK5A8dzKEy+7Eq9ZRF1HBXgNSYkYjeIpIul/OwyEKfZHLzVMK5mrjLTdPpTGXYiRfqLMsxka5dd4L5ieeeIKenh42btzICy+8wBe+8IWzspNLimEuViquky9qU4BIRdi4NYWO5lZ8ziZk3yR07QDTwL1lC+HZhFjnWlapdKykVZXCUwrmauMtN0+lMZfi+L9i5oorbS/Ce8F8ww03sG/fPg4cOMATTzxBc3PzWdnJJVU9J18ohNLhcBQIoXxX1vsc+Jx1OO01JP0JjM6LiM1MENETiNnwq3yheQ6Ho2AIpcWTzYYV2iSEKBhCGQ6HC4YbOhyOguGGTqezYLihxZMvNM+SfKF5sVgsr09ARflkneWZzye73V4whDIbz/wwNkVRCoZQWlOh+UIoY7FYwRBKl8tVMITS4skXQimESIfoAekQxcwQRyHEaWGP8x+tz2Y+Zr6vKErOEMpMnfk25vNYvPl4LMnnUz6ObCGL2XyyePLZypRzqWPrM9Zc/oILobQuasgVQpmaqlGAt6ZifP/gJB90uVASDjRXLXLkMKGTJ7E3NRcMocwsK1doXqZOrtA8SzdfCKXNZisYbphZVjYbFk8+nwrxzD89Pl+4oaIoWbP3WTxW+Fil+GTd4ubzqRCPxZzPJyAdBpfLJ3h3eiRfCKWVbjeXTxZPPp8gNfLLxWOVZ7fbmZycpLGxcU4ZmWF88zc7Zb6X+Wh9dr6v2Wxks5W5r2H+Y2Ys/vwQxWJ5Mt/PDE/M9dlsPJk+WTzz/TobnkJ1rKoquq4zMzOD0+k85xDKitvxWiglqMjIXpMwJff8YpAPbq4lqJlo8Wl8SR09FMReosNyS6VjhYdVCk8pmKuNt9w8lcZs5em38vBnk/cyTn5R58x0XC4XnZ2defWKkYrr5HVdz3/BZqmbqF8ivQqDEwe4JBLDuWEDeiJBoaWLgmWVUCcQCKRHUJXAUwrmauMtN0+lMQMsX7487/uFcq9n3gmcrY1y6yxU5mKl4hZeC608i3mPAL3TESYjMQZCAzD5DtJur7gV/vmbHX7ZPKVgrjbecvMsROZq44WFy1ysVFwnX8zCgpzNX2Mtu3w/nCQyM4NHt0H9MqLBAGYyyVRs6pzLKpVOMVJOnlIwVxtvKctaZK4Mll/VOj4TqbhOvuBcVfq/766sX1ijEwpOoE3GwdeNw1ND4MQRxiO55x6LKauUOsXcev0yc1pnk0LM1cZbbp6FyFxtvLBwmYuViguhtEKFcoXmqUIwm6MsLXFXkkQixvjYAJPhaTSHnd6ht3FprryheZkhaLnC2CyebDas0CYrdC5faJ510HC+0DzLVr7QPOszuXwyTTOtny80z7KVLzQPyOuT1V6V4pMVW5zPJ8tWvnDDbDzzw9isz+ULocxVx5khlFLKgiGUQoiCIZQWQ74QyvnX8Xyf5tdxNp9y1XGmT1b9VIpPmXWczadIJJKVJ9MniyefT9Zn8vlklZ3Pp0gkgqIoOX06U6m4tAaFFiX+fd8gh35yD8/4tzItHYwrNj6+IszqydeZ6rCxakUX65vfx7G9P2b9b3+MFndLTlvlXEgp5sDmSlv8KcRcbbzl5lmIzNXGCwuP+UzTGlRcdE2xZxtKZHrCJiEl9S0tqG4b03E/gelpfM76kpRVKp1ipJw8pWCuNt5SlrXIXBksv6p1fCZScXPyVorOQpI5XaMnTZobmuhsWU5rwsmUBHdNE2E9TCCRO71BMWWVSqdQsrRy85SCudp4y82zEJmrjRcWLnOxUnGdfDGxvjBvU1TCQLVpLFu6Fm0mwpSIM+mMMRmdZDg0nNNGOTe9WId0lKOscjFXG2+5eRYic7XxwsJlLlYqrpMvdPiBlGAKEPLdBQjdkNR4HCRjCZJuOwnFJDkdor2mHUMajEXGMOXpCxblPBxiYGCgbGWVi7naeMvNsxCZq40XFi5zsVJxnXyxBy1kTtfETBWnbza/RF0dpl2jfekaNEXDH/fz/MDzDIYGz6qsajwcYvHQkPe+rEXmymD5Va3jM5GK6+TnH5WVS1TeHZkHhAlakpqGRuxS4+DkSfxRHRnQGY+M47V7sSunJzkopqxS6RSz2FJOnlIwVxtvuXkWInO18cLCZS5WKi5O3sq6ly/+2pQCBYldjQMwptuJ65NIIZBxJ/GkDpMhAsd7ceJEN3Vi0bkxy1Y2wkJx8hZP5mfn86iqWjBOvrGxsWBMucvlKhhT7na7C8aUWzz5YpWtLHj5YpXb2try+mQdSlwpPlkbSPL55HQ6C8bJZ+OZH6usaVrBOHlr00u++Ou2traCMeUej6dgTLnFUyjVcD6frNC+fD41NDQU9MnpdFaUT0uWLCkYJ5+NZ36cvM1mKxgnb0m+OHmHw1EwTr6zs3PhxskXOsbssTcGefMnX+d1/0ZCToNevRmXluRff20vbRtuZ+8rz/OCE64zWghEB2h83xbcdg8O1XFazHwoEMBpGGizF242KdXRawMDAwUzylXa0XSFmKuNt9w8C5G52nhh4TEv+OP/LFnbEEYRJpvq44R0DTMmSAwOEgqFWOJowNPcRV0oRKe7ky5v6oi1odDcU3iciQSJEyfOmacYnfmH+b6XZZWLudp4y82zEJmrjRcWLnOxUnGdfDErz6aEdQ0hPrRdp8OV0vcHWzEjUSLRGL/haGfs5YPUaipOUuFKAsF0fDptQx8dZfqF/yLRX54IkmKk0lb4K4nlV7WOS1nW4nXx3pdVTuZipahOXghxtRDiiBDiuBDic1nev0kIMS6E2Df79/GzBSoUQyqRmCggIZk0qFFTGwt6p+pI2FxEVAURiGGbMPBJO71HJzM//O4/o1Hcy5ahFDhtpVRxr8Uk/6+0WN1CzNXGW26ehchcbbywcJmLlYKdvBBCBb4JfBBYD3xECLE+i+r/J6Xsmf37h7MFKuoAWwlCgDR16hypTn48JolKlZlwFDWURDQHiCkmiZA++5G5aw/C4SA+NYnr/I3kW5cohqcYnZmZmZLYKadOIeZq4y03z0JkrjZeWLjMxUoxI/ntwHEp5QkpZQL4HvChkhHME5vNVlBHIlKRNFLS6ErNXcVREFKyrEbD4XPg9EmCWi1idjE6okcwZEpXJhJgmtTt3o1wuzHDuU+6L4anGB3rkOlztVNOnULM1cZbbp6FyFxtvLBwmYuVYoIxO4D+jOcDwI4ser8jhHg/cBT4Myll/3wFIcStwK2Quh3p7e0FoL6+HrvdzujoKIZh4PV6aWlp4dSpU9bn6O7uZnh4mPHxCaQUJIBYwmCFGAK6iYXCxCNx1jl0giKEcE4Sd6xi0j9Fb2+SxwYfY/uy7QwNDRE7egwZCrLkssuIRqP4n3kGdcsWGhsbURQlfQZmTU0NbrebsbExAKRQ6O7qYmhoML0w0tnZyfj4OPF4KpyzubkZ0zSZnExNE3m9Xmpra9PP7XY77e3t9Pf3p090X7p0KRMTEwQCATRNo6WlBcMwmJpKHXpSW1tLTU0NQ0NDGIaBx+Ohra2NU6dOpe9Curu7GRsbIxqNYhgGHR0dJBIJpqdT6xA+nw+n08nIyEiq4TVtThsALFu2jJGRkXSKXOswYWtUkdlOkLpY29vbs7aTVR9NTU0kEgkCgVQOoYaGBjRNS9ep2+2mtraW/v7U5aKqavosUmvE09HRwdTUVHouM1s7+Xy+dB1bvg0MDMxpJ8sXTdNyttPgYKpt3W53znaKRCIYhkF7e3vOdsr0JVc7WXXs9/tztpPT6aShoSFdP9naacmSJQSDwXTI3fx2crlctLS0cPLkSTRNy9pO7e3thEIhpqam0DQtazs1NTWl6ytXOwUCAaanp9E0LWc7We3idDpztlMoFMIwDNra2nK2U+Y1l6udgPT1l6udHA4HjY2NDAwM5Gyn1tZWwuFwuvPN1k5Lliyht7c3ffJTtnaKxWJMTEygaVrOdjp16hSTk5M52+lMpWAIpRDieuBqKeXHZ59/FNghpbwtQ6cRCEkp40KIPwJukFJens9urhBKXdfz/tL92xsD7P3RvXS3+Zjx2dnpGuej/7UTgKevXYI99BYt3bs5FejDf2wGV+d5bHj/Og5MHKDF3UKLu4XEqVMkgyG0Nedhs9mI7NmDe9u2rOVl8vzDCyf4UE8HzV4HMT2J06YWxQzFhU0VY6ecOoWYq4233DwLkbnaeGHhMb8XIZSDQFfG887Z19IipZyUUsZnn/4DcGGxAGcjEgHSREqwK+/Ob/nHDxOyRUFRiNV2EzB0RGCM2PFTrHR1I0hN8YRffXWOPdvSpTnLOjWZGg3E9CTXX9iJMbsZ4ejo3DMYxwKxvMzlPvKrFFJtzNXGC9XHXG28sMhcTCe/B1gthFguhLADHwYez1QQQrRlPL0WePtsgYranCUFkiRCSOy2ZPrlyYlX0OMGikvQUmNn1bYtaEaU0BuHUGci2FU7iUQUW0sLWktzuixjfAJ9dOy0Yo6NBjkxESKSMOidDONz25kKp35UBqajjAVijAVjBGM6U5H8iynW7ea5+l5OnePDxwnr4Zz6lcZbjXVcbczVxgsLl7lYKdjJSykN4DbgZ6Q67+9LKQ8KIe4SQlw7q/YpIcRBIcSbwKeAm84WqNBJ5lKmRvJCSDQRw8a7I+iQ1GhsXo5Wa8Pp0FCjYbQj+4keG8BMJLGrdiJTY9g6O7G1tKCqKidmTqA1NzPyg0fnlJM0JUdGg0yEdJ47PM47Y6m5uKGZGK/3TdPT5ePkRBgp4bU+P/Xu03PjlNr3YnRiSZMEhc+QLFzPEsM0mI5Noyd13hx/E92cmwe70k64L0YWmc9dp5JYflXr+EykqDh5KeVPpJTnSSlXSim/MvvaF6WUj8/++/NSyg1SyguklJdJKQ+fLVAxCfWtw7wFAjQXH21JjTaHlDX4mrpAmhi6QWJkFFs8QrymBSQ4VAfRd47jWLkyXVY0GUUqEPCkOsZQTGfEH2MiFGdjex0727yEh8OsrfdgJk0uX9PMoaEA7T4XhimJJJLUOQpXo9Odml87MhLMqXMuBw5IKXllJsSBUJQD/hCJArd78+3opuR4JMZEwmAsrnN46jjLmpZhSpNjM8e4oPkCTvlTC6x9gT72j+/HH/Gjmzr+uP898elMdbxeb9nK+lVlrjZeWLjMxUrFHf9ntxceEZsoKLOjeTBxKqluPyS9oKhgGthsbmK6SasHdIeBGdBRhIK0pVwOJAK4NBf+uJ9AXQvJ7ZuQpsnUEz/lZPcqljhjrNm4heN9frY3RnCagqg/SmBK54rlTQBcvLKRN/pn6OluZCqSextywjAZiSrEJ8PYtdw/CMX4nkunL5ZgS60HTcBITGMgprPCnXvThd1uJ26anIjECSdNVCHodNoYjkU4Mn2MDfUrcdhdNDub05+psddwdPooPoePRnc7hyaP4E4M49bcHJ46jMfmwak6WVW/CoCp2BQKCk6cZ+XTmerU1taWxE45daqNudp4YeEyFysVl4UyGo0WyEIJUgqEFAgjgZTQ3ZDqRCQaCRM+/MhxvviTY+gtrciWOuw1CaKTw5iJBNGTvUgpebn/ZQ6OHkQaEt00qHc0Mfno9zBVleg7b2CcfJFg/xFC4dcx/YcgMs3Ann3YnCq9e4aZHgkTjUZprnEQCYcwzVSGucwsdYZhMBmO8NP9QxzrHeDUZJhAKEIgpjM+nQopDIXDnJoMp/2Ox+P0jgdzZs2LxWKn1cvRqRkEYMRTWfVc8SjJDJ5gPM7I1NScrHmDU9PsC0SoNRLUyhlWKybNdhuET/E7S7egG4LnT/YxEY4QnmWJml4UUcvxiI2+aAKnbMa0LUc3G9jWuo0uRzd1jjoODB/g6NRRpkPT9E72sndwL9FE9Ix8mp8JMBAIvJuFMpFAT8RJhP0kI9PE+17DjIc5efJkVhuZmQCtOs6X3TAbz/xMgIFAoGAWSiv0NF92w/7+/oIZG+PxeMGMjcFgsGDGxvk82TI2hsPhvD7lquNMnyKRSEX51NfXVzALZTae+Vkog8FgwSyUVihsviyU4XC4YBbKwcHBkmWhLPtIXkr5I+BHW7duvWV+zmRN0+Zs+bWS61uhRG63GyGmkICigE3qCEWla916OHQUqdg4FZjh5YEYEOP9q5pZt/MDDLz6LyxPdBMfGcXZ2EBYDxPsM2lcn6TJ1UQ0GSVWC1pTDSHNyXkuhRalBhmM0OLuQ/e04YwcRziW4PTYWX1hG1NDYVYuaUExdZw1boJTEfb2B5lUJKt9Llb43EzoSQ4GEvzGpnZ6e3UCTR7sisLRkSChuMHu+loOjMZo9NgZi8CSOicCwUwsSiwZY1mjh5OxBD6bSpuqoqoq0aSJS1UYi+vUu9yMxnUSNjsrnXbU2bSrPp8Pr5RMJgxME/qDEeLBIJpp4qitI6AnGbc52VHnYTDwDm7h4EjoMDIEF7RegBCC8+trqQ1M02dIkoZOg02lzuFgSlfZWedACMGI3UaLXaM/pnIoGCYwdpStyzcxU6ORRGF1TW06FezByYN0eDoQQlDnqEPTNKSU9EX7qLXX4tJcICUel5PJ8Dg+Zz3xxATCjACtqePQpESVBkweT10g9hpIJlCXXghDb+CJTMNYEo80wWzC46ybcx05nbnvKKxr0TUvzUXmtZf5frbj2Swb1pyrlZbXsmE9WjacTieKomQd2Vk2CvFYPtXV1RX0aT7PfJ8cDsdpW+7n+2Sx5vMpl/yyfFJVdU4dz/dp/iEeuXzKxjPfJyvddS6fVFXNWceWjWz9Xub7ZyoVN11TTEyrlAJNgmbqIAROd6ryo6bKwMw7QGqKYd94nOsu2kDH0MuYE4K+nz9L3W9cTMSIUON2cWDkIFetupKIEUEqgiGPQLXbqXl+P9qvfQj/xH6MwSG8W5qoXbIWT1KCFsLd6sVIONH1KJOjI/hEC10NHt4cmCHh0fhJ3yQfdtkJmSYgiUiJ2+VAtWlIYCyRZHmTh1DcoKHOyURcx51IcnJ0hvbGWjp8LvzRBMenI3i9do6F4xwKxWh32DjhD9Lm9RA0TEwpWeNxstHrJqrHcdlSF09/fz+1tbWcnAkgdJ1ltTXEmho5HoowdrKPTUs7WaNBf7CfaGycehFktXsJXu9G4vFhDGngci3Fbrezzj6Dy9XBeEKnya7RZNcgHoJ4EK9Sg+Lw0u20Q+Ad5PLzeaXvMMvdTkxPCy8O9bFCSdLetoL1ODkZnYTwGG53CyMCYqP7WblkCyODb+NnGkNVGDUEXiXJUHSQlXXrMUzoS7yA01ZLk2oniY6382pQ5166suk8kkYvtKxDmiYiOATTveDygc0Nztqir69y6hRz615JzNXGCwuXuVipuHzyheRfXx/ghX9/iK0dgphnmh2dGrGWz/D7391Ls9NkcN6hK71f/Q2m3nkN9eAM/VGdlmu2ANDXO0pXVyvCjMPQ60y1ruGtwSPUhYJcavqIDceINgRw908itlyIq7UBOX6YIJMkm1fi0t7P8f0vU9MYonPlVlyudvqnDrA/3Mb5NS6eHZlhe1c9aiRJTZ2DSd2g0aYRSZosddpRgH/c08fO9S30xxNsr/MQiehMBBOc35n60XprIsj5jTUEjCSJ4f1M2RtY3bqU8YRBs00laCQxJZi6yb+8+XN+Y8NFNDtrGJoeot5Zz1NjB9nStIKldS1omkZ/NE6DNECmRgeD/qO0elpR1VoS0wdJOlMXnmnGMc0YpjRwqPXEJ/ehulqxe1dgD0wSi/aRsGs4ggFMVy1ORytqyyakUDDDk6iKAqFR0BwcTQiSwXHWdaxCBkeYMEYYDPWx3LUMb+sWlFiAuNPGVHSCcBJaRAyHZymqrZ7jM8dxqA7abHUMTh5Ad/nw2ly4iaEoGooQ2G0NqKqLWHwEm60ePTEJQgEpEULFnXQiItPQvPa0H4ZFWZRqlDPdDFVxV32hX7lUCCWopkSQxOWsIy7AYVMYDM+dr2qrS90ijXlW4655i5BST40ewW1zU+PyoEXt2KLHOGHGWVe3nMPjp2hsceFYeim2WILQc08id+zCdE0Qc6iI5m6c0xAJTXDA9gpjniDb65t54/g029fVMmJofLCzgXfGw2iKwtRomPNavejAqeFhNqxewfFIjLGETrvTzsXrm0kguaKpjhOROIqZYEIZ4uj0KF73Mo6NzuCePEVcN1je3kytESMa9NPirePoaIi4kWQ0EMfQB7ikYSU/fvFndHa1sby5hWgyym+u3MpQeAhN04gZMfqnXmdUtWMPJpmIh9jcsYy+Y71o8Wlamn04nE40T2rbtGEEGT28l/p6FUfDDkRkmsTEYUJuD+6uS7EDIec4bqedWHwCI/gmiuLEZm8gKWzYW9YB0BEOE6r10W8EqXMlaay5hEYE8fgIE/Exah0eIuEQbQ0bAEgmY6hqqt3WNqxNt2Wz2UNtTS1DoSGmkyo2bHTVdBGLDaHrfjzuVQwMDNDRsQIhNKQ0EEIlGDyArbYeMfgczvoNhOM6nsbOVIa7s7wGS6nT399PV1dXXp1KGmVWGy8sXOZipeI6+WIS6ktTgDCxGTE01YZDU7Crp68hD/tjjPhj6IYk/s4hzGU7iRipTr7T14EQ4ArE2GDzETdUtjTvYnT6vxA2B4YCto52fGvXoI/WE9APM+E/QWNkCYFImFM+SZdjBVNPvMVT67twDh5F9W5DCEFXjQOf28aek5M01jh4ZjJAt5rqVFa5nZwIBAiRBCE4vyY1zxaJhNk/+A6JaJijDV5c8adpjgp+MXGcS5pX8NxJDxfUxXCPvMwrnq0sb/Lw05PPs6beQ8K/nyX2Vn59wzbGYxqNikpS0Rk4eoh3xg8x2NCL6h5mg27gcS1nv/9VGh0tRPsiLG9fS5RGxqOS+ukZML14vV5EJIwp3NCxBRWgtgsnzImT8TWmwrz8ZjNNNRqgEI4cR6AghEoiMYHN5qPOTNCHh4TWQq1ixzAlg7KZmOnHJWw0NLy7l87q4OdLbU1quqW9JvUjNBIeoT/Ynz4QBiCZTKIoqe3iQqQevd4NGEaYZMs6YqFRPK4OGNkP9d3gPH2OFcp7OISVb6UcZZVCp9p4YeEyFysV18lHIpGCp5mbQqTm5ZNxFM2DQ1OZDL+743T9EsmhkVSn+srJSXYsb6Q3Dt4aG82uRiQSISAyehIPCUBjaiZE75G9tKz0oSd1jowfo6uzi/2nxvn0P79BkzOGzdT5XOcxgmY9Vy9dwdRAmNcvfx9Xvf3v+NZ8BFWZIp6YYGbMTtIw2VRXA8B2F4wHM0aOoVGeDIS5TptAtm1Guuo5OtpHqxKl2ediNKaixjRc8QMsWbULm3sJ21w1vNE/TZNex0Utw5ySTi7vWk50Zpj2lt/ELsdQAq/iVAxOHfPTZl9GU/dm6hvXcfTUT/CJK7F1LSNs9NLT9gkiJ/bgXLaVuOKidyLE2o4axo+9RiJ4lP+YcPDo3kEOTiTZuvxlGj12VjV7aHJrqJrK42+NMBXWWd3sIhA3efbIOLtWNdJc48CmKtQ4NCaD+7hoZSfb2k+yomMTPqEQl5JDoSixpMkGrwu7q4W3QlGW6iHqPB6m9CR90Tiba93pBdtc18USzxKmYlPsHz2Ax+HGpto4FT7FMpbN+ZwQKjZbLTZbLXHFzmRwBJtHYJMBnLL2tHKKvQZLpVOMlJOnFMzVxltunlIxFytln5MXQlwDXLNy5cpbDh8+jJQSKSWKoqQP700kEjidzvShytbtTSQS4SdvT/LcY49wcXsC1XaMXbt+C129lN0PvgLAX1+xmt/e4eDenx/gm6mX+MDaFm6M7Gfp7quwtRioehJbWGN8zxus2OxjaLIV0ajhOfpdaq74DL3+Uxx/bZTGjjY++r2Tc/iXuOBDG718e09qU5NNgScviXPE08KuJY2MJAZpariAyOgMTp8Xpy2Gq/8JYm3bcS1ZTTSpMh6doXniKG63ixPjUdTmVvqj0+xsaCVpGCjJCAlHIzZfOwYmw/4+PKaPmWCY0YSfmsAe3DUCJaHSuvxajIle/nW4gemZYYYCCiemdWrsYX63p5apqBtN8xKcGuOFYdjR7eITl28kFovx8Uf289qpd0/L8jo1wnEDs4SXRLvPCRJ2LG8gkjDoqndj0wRToQRjoQSbOrycDMSJxnWGp+McGPCjKQKnTeHilU2IpInP68BtE3TUe+gbD9NYYyccTNAfitFZ62IsEMFhEygOFU9NnC2eMJt7NuBJehGGhuaW2FQ7M9MBbG6VcCyK05kkNjVBfctS7HZv+qBm66Bl64DxzGvPOvjdOuxa0zRM00QIgRAiff3quo7D4ZhzQHTmYzQaTdtQVXXOgdTWwfC6rqf2MsTjOJ3O02xZPPF4PH3AtBWxlPl9snzKZ8PySdf19OHsuXwKh8PU1NTk9SmzXirBJ4sjl0/WgfPzbWX6ZLPZMAwDRVFy+uRyuQiFQnPqZ75P821k8ykajaYfs/mkquoZzclX3MKrddp7LvnhawM88+/fZVebjs12jF3vvx4p3sfFD/wCgP97/QVcui7BsYPP8Fv/uiT9uf/WEuETV+xCqTUJTB1lWdd5HH9tkJquVn4xHKVmYoArP7gVW209Dz3/KMsCnXzlaJJjY+FcKHPk6QumcPuWEtD8NDv7cJjLGIoLmiYnqPud32Nk7724PRdR54lAwwrwdWNK6D34InsH/4vLW9vxNvfwTsLJnhP9fPjC9TgaOkjocXpHf4qieLF5VnB85G3UwBTLPUtpcHs5NTbFLU+lpqbORrob3VyxvpXjoyGmIwnafC6ENPmjS1cxNDZJS4OPAX+MnSsbebPfz4FBP81Okwl/mNZaNzVuBx/cshxNt6c/EgAAIABJREFUEUQSScJxA90weP7AKZ444ue/3pnOWq41fs68+uqcGnEjicv2bsfn1hQmozoxI/d1atcUEsbp8cM+l0ogllqc7q634XEofPGKdhpsNSTUGK2tDTTXNJ32uULXYCl1xsbGaGlpyatTTp5COtXGCwuPueoXXotJqG8CmDoKdlA0IuEIV6xv4eeHxvB57Zimn1rXCnyOYWbiqXnXh8fc/Mn4NHG/4HDgGKu7l9PUDD/fH0Ymx/AEJhjuN2loj+EcWsInDgaI6rDEB/92+0UgTf73WzN8//GjAPzZVV3E4yP832dTW5Qvf7OBFa4Ikwbo5jK+uyvBWO8hftC6Hv/3X+L1oRVcs7SXUbmEtd1BOp2v0DczQRDBrvOuQXONc9hYyofufwmAO5/ax+qWoxwbi9BR7+a6zQpN3v2EQg5e6VvFwaEAk+F3UyT8zW+fjxDQ4nVw4dI6vvToHtqVBJs3dvP4vkHWewVbtq/n9deO8LN3/MyYKpeuaeHDu1YS1ZMkN0ikhC1LfcQNE49Dw2f6Wbq0ka1Kqkv+tXVOLlxajzMJHp+DZDJJIBBgcnwMwzCor69Hnw6TSMS5/qI1/PqmMMOTAZSkSjRm8PJogJhU2d5dy3mtXqLBIKeGgkxLSb3Xx7K2OkaGg5iqwXldjQyHdd4ORVlZ66RvJoquS5a3eWm3a4TCOlrQz8g7x2kScYJ2H/F4GNl5HvuGE4xMzDAZCHPSn+SUP44DgZAaTY0drGzNPk1zJtdgqXSsTVflKKsUOtXGCwuXuVipuE7eMIzCybMQICVSsYHDixFM8JXfOp819iNctLqZcHiSmsZu/vHao0TkJj7yg1TOlaT/EJNaHcqAi7HVk3zuxWleG5mmxgWfXKkycvgEEwdcfGN/jPjsusf9/201R2M2goNBvrh9BTdvqKHRpvLzUz9nvf08dpzn4mP3peyfiJqQWqbkuucdwGboA4gCNo5M1aX+/aZ1eLjlZ9/s40tz/Dw2lmrowWmdbzytz+obwLsZ6jZ11vG137uAVS1epJSMnzrJyE+f48u7NmLr6MBdW8cHepYiIxFihw9z4eVruPJSO7HRUZbFZ7AlA8RaWql12tKjZ1URTI8MYeo6iUgYZ01qbUEIkP4ECU3BadgYPTVD67J6+t8cwC5M+oaHqa9x0+z0MvbGOKbPRq3dhbehBqeM0NnYgGlKahsczIzFsKteNp3XQHA6gqLamBkOs2V9S7oDdthtLKt3c2g4wCqXjealtfiMBEfeeQd/OEgUQcPKLsYdLiLBCMODA3SFpnl/s4L3vFaGAgmi46O0tC3FK2xMhGK4En6EyL7geibXYKl0ipFy8pSCudp4y81TKuZipeI6+cIdPDCbF97QapCanYSM0lLr5FOXrEBRBHZ7E6ZdEhg8QXPjWv744g6+9YtBngtFWCujuMw27nnidZ4e6QDAH4K/fNMDzE03/NWPeFlS14Loj/F2XMcZ1EkYY4wmFLaLHrobVvB67E3e+OOdjDpUvvmTV6iNzPz/7L1bbGxZet/3W2vf964bi8U7eXju3T0903NrjUbxRJYEy5bzYMMXOHFiI0AQCwlswECAAIERJECCJPZLgARx7AcDQRJHHjnOQyTnYll2BMgSNNOSZjTT092nz408vBdZrNuuXfu+8lCsOiQPyaruPkORVH/AwQGrvlrr/62197fXXuu//ov/bevk6nnBFLxdg34o+bgdE2YvY3x3dYr7swW+/d7g5J+//1e+xltaE33GwzE9Co7HR3sR7291+N+/85h/547gwZsPcCyTe0vTtPf28AqDOb7GxjpW1+fun/wF6psv0PZ2kZqG7RXANPG++U0A7gHptDugVn7wAY5hovQqKsvY21gnjUKMdpdkd5fNj58x/eAulQf3aWz5lGccDEun/kEDV8LB8yZzxYi4KbBbB9jSQnUEU7pA6lNgG6jtp6hSCbfxgjjXaW/klDxF5jh4C3ewvSJpFOGKgNb3v4csl9GnprCER5T2WGg2URWLvQ/2eXGwy9JijamluziWQ5jm6P0MX7dYnLEJXJNnjSZJr8NDy6D41ts0NvdYo8tcpcZBolF/fsjyYpEZ6+wR1WWqDY6bRrhsPON8rhteuLmYJ7Url+QnWSPIlADE0VRNn8NWjMoVHNEoLWuGuN+kPD9Ha3edYv8uAH/rRzX+4e3nRHmFXzpK8OfZ3/4Lb3FrZpeNWFJN4N/7ygqin1LMPJpxi3vRKtl2j3t37uH0Lap2j7/19h1MV/IfavvsFmf4+vIdOodrRFFCSxM8Wf8ubyYu5cAgTgw8RyC1BPvtRf7rP/dFsixD0zT2HreYcubY/fXfRHt4j7ncpGA6/Om/9EXcqTJxz2f3ycesH7ygPDvP3rOnOKUSptDwFhZRjsPiG2+R5zn1tRcs3HPxW4eoLCfLUqbmFwcPyTjDuHeftd/6AV6hiezv4BSL6JZNW1vAmF/GWyzy4vd/wPZvr7F8Z5HwaZ/89gOq8x5JFKK/2EYuLFN6twjqDkLTUGmK0HXCJ1uoLMO6fx9pWailJawkoWxZhD2f4OlTgt99j7DTxrRddKuKsCqoQ5/u8zU0u4Q9PYcrQg5/sE6pVsNdfYOdUHC3WGCvGdKTMDfnUjM0slCjXC5zb6aIKwQ7KmNGSd78ygPE0ZRTkiRkQqIdhKg5/cxpm8vUDR8ee3cZdb0On+uGF24u5kntWiZ5AKSOaQy0WcKwT7cZ4povufJCSpK+QLNAHB4ynBr5n7Nl/vnWyyfpP/zzHttGynf2Z/mGF7Os+dy6JajN3abe03gRZZRmHUxDI+0lLHVqLOY1xB0dw3bopg2Q8OzxR8w+WGK+epf/9Zf+Cd/8Ez9L1lxEGU1ma1+n+egR76z8DEavj10pU//dfZqJxv0vTrP7ow/p6jnTtof/Bz8ijR2ehN/Fmimy/5u/TbU2Q+2b75Arh73nPkv3qtx/95vkeYamGzilEoc7Aarfx3owj8hzDMMg7ufkmcvT3/suxdoMluuRpxmNrQ0KtVmibsr2k3WqiwXySOegP8VSdZ7uQUJlpUK9tcfMQhnv3S8Q9tqIkotTukv8+AnpXp9MSvKFJfx4F57tInWdJAwp1mYoVqfJbRfpmcgjrQ6lFNKyONzeQmSC2sO36dUPMasZbqmEUXMQhiSpBxTSu5C3yf0u5r3baEtLuIUC0tIp9BNedCPu35k6cUms7TQpl8uUjjbBnbWVRCmFbWioaYf0oI8x8yqV7TJv5sPDw7GKg1cpAV03vHBzMU9qVy7JD+lFF5kSoNsNJC65EqhckSU52sxLAR/DtHCLBdy5h5jNg9Hn/3zj5Sv6L31lnzuzOsGhz7cevEkhSxGP/imhd59f729Qmaqi8pTbnsXu7i5ToUPajBCmpFOvM3//ITY2B3GTu/1lqlO3EELwrdWfw/4oIXB3sB+u0u/3OdjexLy1yvKDB2x++D73fuZN0iRne62D0AvMK5+DwwY8eJva/bewnv2IxZ/4MnEQkOzvE+xugXebQtqiVbeZXiogEWRZTt/XMFotcttlf72L6UpqywZRP2FmdYagU8QpWHQO+gih0HSfJ9/9IZZjsPzGCmk7pBe3eOOrb9Ko9ykueDgzLsQaWTPEqZQp3Jrl4MUacauO92CVfrtF3/eZKtmUS3MkcTQaFQftNtuPH2FYFl6/Sre1R7DfAk+iDCiZVYyCS+YneDNVcuOlSBOAMTtMvIXRZ7aVIo98TMfgvjN+8eqi60sY8swEf9xnknI+q88kdpl4Xgfm64b3ddZ1mZgntSsnNZwkyYVSwxw9BaXIEALSJMd1XaIwIVMDadEoTGjsdilN36af5pTK8/zzv7J6Ase3v7lIYUWwWd/lS+597jx/j8PWI2bu3iHs2MymEbutFqVmlw+fP+LJkyd8+PQR7jszqLuD5NDtdCgaRb6z9R3ymZx+p0t9/Tmzd+9xKD4mXN4l3lM8/cGHZN0OrqbRabcwbIckj4iCAxrb36XblwivgLN8D29xkdKMxcJPfAWExCoUOQx8PCVwO2to/Ra973+Xrd/+kEffWWP799dJG9tUV6coP5xl6c0Kwsx4+r06mgVSVwTdmMZOB7dsIAzoNQxq925z7+vvIEwLr1ylYNdQqcIxMtxFjyiLKRQK5FM6yh3wgQuzczilMs36Hk6pglOt4ZYr9MMQ03aI0wzDsjEKRRYfvklxboFmb49e4lP+4grF6hzl0hzmcpm8KJFTJrFISdN0rNRwEAQvpYbPkXsdqvtdJGGbJMlYqeGz8JyWew2CYKzUsO/7F8YUhiGe542V5c2ybKws7xD3RRK23W73wpiiKCKKogtjGgpnXRRTHMdXKqbjbXye1PBZeE5LDff7/bFSw+fhOS41PMxzF0kNl0ql1yY1fOV48nmeX/ik+ye/t8k/+z//EX/q9jqF6QrL934e0jnma8vUlgv02hGaJgmDGCU22TrcpxvdZ8Vp8S9a8J//H08B+M5ffkBa/YDf3qjxJ5fKbHz8T/l47o/zxmIZ63kXlu/w4ft/wBeSAtqDGlMzs7xYe8Hb73wRKSWNzQ3iMKBUm+WHuz9gOZ+mVd9FSskXfvrnePL9f4aQAo+H7O085d4f+0lSv4vUdWzPo13fI0tTyrNzSN0hClKauz1W336Vtw0Qb26hlYqIQoF0e5vG0wNKhRwlNXTbxHrzzdFI+nQbqiPBHyEHC9bJdg+FQgUpQhMYy0Wk+epiUBzHF6rhjeury/YZh/ey8dxEzNcNL9w8zJ+UJ3/pI/lxFkXRWB9xxLFJpUaKRCLR9OFUQUya5GR5ilMsI2wXLYfQsLgz4/Lf/7TJv/qb7+KuTNEV0ySJidaYQs89ZlyX3wqmuf8T32T7t37IT8hlhGsSdlps/fB7rNy+xcbGBt3DA9ypKbbW1gj9LredW1SXlrn1xXd484/9cQDufPHnyPMa6/uPWZha4eDggPb+HofP1zFsB4Rg/t4DnGIJyzHoNkKqy+dvkDCXl9BKJaIowlxeZv6nv4z7ta9S+OqXsd9668QC4uk2FEIgpCBaaxM9bWPMuiQyw7pdxrpbOTPBA2xvb3/mvrpMn3F4LxvPTcR83fDCzcU8qV25OflxO8FGbx4Ccm2gYeMHPvmxvfhpkmFZFkKEHOaKliu4NbdI8fvvsbIyQ2bovNf6kNLUOzyc38OeqbF6/99kqtclNl2efrjPUnked6HCbrPFtFeh43dJOy08r0CWpMQqwqzW6Pd6mLZNlCumagPaU5IkdDod9PIUZZUztbBEEO5z6+13SOt9ku0eM7duk7UjtPJgimFq3sMpjp9nHraPEAKVZOSpQtgaeZCiecaZbaiUInrcwlwtIgxt8IYxX75wQ9AkNq6vLttnEvsc82f3uUpY/qi28SexKzeSH85tXWRHR3iTaAa5GoiNhd3BztO+HxP2Eg46PkGakXRjpq21wVFS7SaO6yG9ArvRFr912MURoFdtnNIcU+kcXzNM/uV+m9VvPMCdm0IUPESe8/CnvoVdKJIkCR89foJhGKzeu8/G1iZJkpzYofb8+XM++ugjFhYWePOr7yJdHbPNYKuuJgZURj8mbUeoZLDryi2ZE8Xe7/fJ44w8TEnqfeJtn+jjJlk7IvjBPvG2T+AHJLuD+TylFNGTFta9CtLSR1TCSeo6fYLNWVgmwXtZPuPwXjaem4j5uuGFm4t5UrtyI/nJ1dkEuZA0W21milOEwSDJZ0lOx+/zXTPla6UcYUms/GPga1TmFrA8j987XKOtrfKv16aY91++AWhlE7cZsXx7Cr08EGOamppi+dYtACJ8Zmdn6R42Broqrsvth29Sma7h+/5oHq1Wq/HgwYOXwkiWRtUtEz1vY92toKKUeKeHVrZIW9GI5eG6Lsl+gD7tjJLxWe2TtSPiLR/rXhmhS8TR/gBj1iXrxphBhnB0VK5IG32su2WEJl4pZ5wtLCxc+P0kZVymzzi8l43nJmK+bnjh5mKe1K7cSH64ojzOFIOFxDhNKE8ViHovNw+4JZP1dg8/zegCkb5MrhS3v/J1puYXgZwVr0IUhkwvvWRT5/0UleY8rA12rD5//vyVszxVnuNYJkopdF3H9gp873vfGykXpml6NFX0MqlKS2fPHBzcLTSBdA30ioVRcyBXZH48il0YkmTLH9s+9lvVwcj8mI6+0CXSNQhJ0DyDZNtHWtoJn0/Szuvr6xd+P0kZl+kzDu9l47mJmK8bXri5mCe1K0ehdBznYgolAwVDhSLLcnKORPi1nCQeUKR0S/Jlx+b/OWiTxC364R6Pmh2EEPR6PVYsjTe8Re6ViyNqUxzH5I5E2ZIVXY7oVoVCYfR6dVjf4+nv/g5WqTLCZ9s2q6urI0nSDz74YLAR6RQ9Ko5jrPuVUQyRPaBCRSoh3vQJWj6O4xAnCVmeEbaCMylfVqrRT6JRLMfbp9/vgyEwyja5UOQljdyRZ1K+hqv742hsF9ENh7K0F9ENhzKrF9ENXdcdS6HUNG0shXK4oHVRTLZtj6VQnoXnNI1N1/WxFMrhg34cNW8c3XAof3tRPw3xXETNO43nLLrh8Do+L6bz2vh4TLZtX6mYxvXTcYnhi/ppKH98EYVyaBfdT0MJ5IsolEqp10ahvPTpGqXUrwK/+u677/614xtgYLAh5rig/tmnlg/EuTTNQJMatm1j2zalaYlAYrsWUkr0WJEphWdqFO1pwqM5Ls/ziEOXL5aLhP0+9lEdmqahDIUyDaSp4/s+X/rSl0ZJCKBSq5GFfWZnZ09cYFNTU1iWxcHBASsrK2eeDG8YxmiK53hMXq0ENUgPQ4L9Dl61iJjRiNbaGBV3VEba6GOYGr16m9KDmZNlnDpdfpgUtdKrr4VD3YzhAvbpMo6fLi+lPHNucBjTSA74gtPlzzsgYViGpmknfMbGdIbux/CzYbtfFNM4PKcxnxUTDE7uOd02x2M6bufFNGzjs6hywzKO65yfFxMwSh7nxXQWntMxDTXwj7fP6ZjOa+PjMZ3Xxn9YMWmaduY9OcQzfKhfFNMQz+m4jscEjHLCeTGdvt7PimnctfdJ7cpN10wSyFCkrCgErldA0zS8ikWa5GjGICRH6vyl+SqeAEdK/P5A+VGpwUKnFOKVupRSSFvn6dOnRFGEpmknfJxSmeW3BmeRHp+OGfpEUXQut3V1dfXMz4UuEbokD1O0Zo4wBhfNkHWj0pzwcRM0QdaJ8W5NnVnOcZukDSfxOQ/z667nsvBeNp6biPm64YWbi3lSu3JJPgwvPvxiuEyaCImGxPG8QcIViizN0XSJEILytMFiKpgxJfOm5KvmNnkeEwTrZMnBmXXV6y9VKOM4fsXHMC0M61X609Dn9u3b5y6qHC/7LDNmXdStl2VL1yAPU+IXHfSqjUoVQhfEeXJhOacxfxafcZhfVz2Xhfey8dxEzNcNL9xczJPaREleCPELQohHQognQoj/5AK/vyCEUEKIiXdjnbZJqUM5IIXEME2klOTZ0fhegFs2KVRc2vU+VVuQpC0MIYnjBrrucb8wdWZdeZ6jlMJxnNFBupPgGfpIKc/lno+jTUlbP8GflZZG1o3RZ1z0aQej5mDMeZ8Iz2f1eR10uquE97Lx3ETM1w0v3FzMk9rYJC+E0IC/C/xp4AvAXxZCfOEMvyLwN4HvfBZASTJ+pCpQKCUQQkNK7WgBcLAgIaTAK1ukaYLUJdLQMY0qSdIiz0P6/Q2mC6uv1NXr9VBKsbe3h2maTE9PT4xnEp9J7HQ5eTdGK5oX+nxaPK8D82Vi+XG18R+2zyR23TBfN7yvs67LxDypTTKS/wbwRCn1TCkVA98G/uwZfv8l8HeAT3fY6JFNemJKriSasBFycDjvcCHRtAeLGFJqlKZtbGsBw6hgmlXiuEHQXzuzrv39fbIsY39/n1KpdGKh5HVgnpub+8TlmKuvnmB0mQcXjMN81Q5a+DRt/Iftc90wXze8cHMxT2qTsGuWgI1jf28CP3ncQQjxNWBFKfV/CSH+4/MKEkL8IvCLAMvLy6ytrQEDdoppmuzt7ZGmKcVikdnZ2RFXVAjB6uoqOzs7NA4OEGqw6zUKM9rtNi/CF5AMtut3gybNnuAwUHjeOlF3hih+ga65rKws0W7ro3pnZmZot9t0u12CIGB2dpY4jke6EYVCAdd12dwcLNrqus7y8jKbm5sjUf/l5WX29/dH1LKZmRnyPKfRGLCAisUipVKJtbU1HMfBNE0WFxfZ2NgYTQndunWLg4ODgRSCrjM7O0uaphweHgJQKpUoFApsb2+Tpime57GwsMD6+vro4ba6ukq9Xh8p9y0tLRHHMc3m4CDtSqWCbdvs7u6eiGXYFjBYU9jd3R3NGXqeRxRFtFqtV/oJBoyhxcXFM/tp2B61Wo04jul0BvsEqtUquq6P5hxd16VUKo3aWNM0VlZW2N7eHq2LLC0tcXh4OHqFnZ6eRkrJ/v7+qJ8qlcqojc/rp1arRavVQtf1c/tpa2uLNE1xXffcfhoqUC4uLp7bT8djOa+fhvFf1E+2bVOtVi/sp/n5ebrd7ohyd7qfHMdhdnaWtbU1dF0/s58WFxfxfZ/Dw0N0XT+zn2q12qiNz+unTqdDs9lE1/Vz+2nYL7Ztn9tPvu+TpikLCwvn9tPxa+68fhrGnyTJuf1kWRbT09MX9tPc3By9Xm+kLHpWP83Pz7O+vj5K4mf1UxiGHBwcoOv6uf20vr4+ovKe1U+f2JRSF/4D/iLwD479/VeB/+HY3xL4DeD20d+/Abw7rtyvf/3r6iyL4/jMz4f2y++9UH/tP/076n/5n/6W+pf/799T6+0Nlee52t3YV34rHPl9tN1U+zsfqTzPVa/3TIXh7ujfWXW1222llFKHh4efCM+kPs+fP38t5VymzzjM1w3vZeO5iZivG16lbh5m4HfVmPx6/N8k0zVbnDxkZ/nos6EVgS8CvyGEWAO+CfzKp118nVg0Swgg53B/f/Sb7uHgibkfJ2iaRAgHIQS2vQxCI06aaJp7rIhX65qaOklRnATPZxX6et11XRbm64b3ddb1OeargeWPaht/Epskyb8HPBBC3BFCmMC/BfzK8EulVFspVVNK3VZK3QZ+B/gzSqlXxeInsOEr17k2EqGU9BREWU49SkhTRdLPSJIW3+sEKBT9zuA1UkoDpVKiaJd2O568rtfoc1oe4cdZ12Vhvm54LxvPTcR83fDCzcU8qY1N8kqpFPgbwD8DPgT+sVLqR0KI/0II8WdeG5IjO70L9jzThCRROX6W0Uoz0jhD6oJWsEumFJahE/VfNqZAIEXpxAG5uq6PXemeBM8kPpPIi76uui4L83XDe9l4biLm64YXbi7mSW0inrxS6v9WSj1USt1TSv1XR5/9Z0qpXznD92c+7SgeJqcXCSXIUORK4WcZvu8T9VLqUY+KoWNKhXF0sLdS6uhIsuRE4yVJwsHBAXmen/sK9booUcMFms9azmX6jMN83fBeNp6biPm64YWbi3lSu3I7XifbDCVQ5OQIhIJumqNEShylhHlO0N/E0QS6NQiv0+kQBAGaVkJKOVrtrtfrRFE0EtL6tHhel/bzVduQcZWw/FFt49dZ1+fXxY+/rsvEPKldORXKXq93oQqlOpqUV7lAxQLSjF4cE2Y+fj8hTVKyuMGH7/+QKAkIgoCPPvoI3+9iWUsEQTCi4zWbTfr9/kjd7SyFuSGe4zhOq8O1Wq3XotjY6/XGquYFQTBWsXGI5yIlwCFl7yIlQNM0L4yp2WxeqZjOU308HtNxPOcpAZ6F57QSYLvdHqtuOKTsXaRuaJrmWMXGYTueF1OWZSM8F6kbnsZzlmLj8FyEz6KsebyMqxDTUBX2IhXKs/CcVqFst9tjVSjPw3NchbLb7Y5VobRt++Ye5D3O/vF7G/yLX/lH/MydALM2T3nlS+QrX0Q+/pA0WWDmYZ0oV+TP+pTLkocPv8F3vvMdlpcLVKt3EUKwvr7OO++8w9OnT6nVajx69IhvfOMbP4YoP7fP7XP73F6vXfuDvCc9NMQUIRg2eTZ4snXTjPhIgRJhIMQWljXY+FQulykUiuzv748O9hg+FYUQlEqlz4RnEp/jm1l+3HVdFubrhvey8dxEzNcNL9xczJPalUvyQw3l80yhUAikUKBJsmSQ5BMliGwJCDyjjG3bdDq77O7uUq1WkVKQ5/ngUBFdZ319HcMwyPN8pFPzafBM6jOJva66LgvzdcP7Ouv6HPPVwPJHtY0/iV25JD/pSB4UQkr6SYrR2aQvBHFBw08zqqZ5dKpMxvT09IlF1W63i+d57O/vUy6XyfOcmZmZz4RncsyfvZzL9LlKWP6otvHrrOvz6+LHX9dlYp7UrlySn+QpJ1AIFLoU9DKJ0dulrxtEek4zSRB5zvz8PKY5S5ZlVKtVNM3FtqFcLlOr1dB1nUqlQrn8qghYfqwTXteT+fbt26+lnMv0GYf5uuG9bDw3EfN1wws3F/OkduWS/MQ6ygI0IRDSIc1imoZFe3eHqmkgsv7RmYxT3L59e6A3n1uUyxbFYhHP83jnnXcIw/BMRbjo2fNPhGcSn0l4r6+rrsvCfN3wXjaem4j5uuGFm4t5UrtySX4SDumQD2RJiaYb5HlMC8GCZWDkOVl/C8exmZ6eHo3UG40GpmlSKBTQdR3PO/8ADlnwyI5UE18X73WS02CuGld3HObrhvey8dxEzNcNL9xczJPalePJh2F4MU9evQRuKEVVNyFL0ZWiYJkkUUie+fT74eiAXuDoEGj9BAc2DMMRf/U4rzczDBLfH30fBAEqy87l9Xa73bE8+SRJxnLKwzAcyymPomgsp3yI5yKucrvdPrOM47ze46fdnxVTp9O5UjENJW8viuk4nvO4ymfhOc1V7na7Y3ny57Xxcf71EPdF/Os4jsdyyofSvBfxr0/jOYtTHgTBhTGd18bHYxpivioxHb9uzuPJn4XnNE/e9/2xPPmk1cFdAAAgAElEQVShLPdFPPkgCMby5C+K6ZPalePJZ1l2oaj+L7/3gl//1V/mzz5oUF59l9j7ArZ8zG/ykJ9wNcp2i5p6Qblyn6nKV0a/q9frlMtgWbNj60r26oDCmJsjyzJEkhC/eIH98OGZmMZhhsGTeZwexSTlXKbPOMzXDe9l47mJmK8bXrh5mK89T36cQtvwmTQ4vFvjMEnQhcTVJFq6xrJbIcv8EX9+aGcxaM6qKzl1gG6WZag0RegGWTd+xX8SzDDZ69dVU8Ibh/m64b1sPDcR83XDCzcX86R25ZL8RHrMo1l5SZzl2JpkOk+whUAXGkKanC7mrHLP+iw9GsWDGPmku7tkgSD4vQ9P+OZHHTEJ5uFr3IVxXTFN63GYrxvey8ZzEzFfN7xwczFPalcuyU9ukiwNCXPIgUKaUDF0QGGZ5x2RNX5qSpuqkLVaSM8l83vkQUB60ECIwWHh4UePR28J0eMnry2az+1z+9w+tx+HXbkkP8kagUKAeLnztZfmFFFoR0ncNOcZjsQ/aV3CMFFJilYoELSekXaayEIBY6GGQNHbXqP3O79P5vtIxyY/WiwZZ6dPnJoUzx+mzzjM1w3vZeO5iZivG164uZgntSuX5Cc5yRwGh4AobHQgBgwhRqvSlrWIab56soqUDln2cq7rrLpUHGHdvTOqI1Ntus0XCF0jXmyBkCSyS94LkKUSWac7EWbTNMf6XLUT48dhvm54LxvPTcR83fDCzcU8qV05CuVQ3/18qeGXlmUGpgCV54g8J0lSkiQlz3OyrPpKGbpepNPZARhR5V6hUIYhqa6PPgvjgMw2iZND0ooHrkkUR/TCJzRa76PSBN/3x1IoNzc3x9KjhvFfRM2L43gs3XCI5yIaW+doH8BFNLbhKfHnxdTtdq9UTBsbG2NjOo7nPBrbWXhO09iG9L6LqHnntfFxat7Ozs5YumGSJGPphr1ebyzd8DSes+iGx+mPZ8V0Xhsfj2nYp1clpu3t7bEUyrPwnKZQ9nq9sRTK8yidxymUw/gvolDu7e3dXArlUHv9PPv2d1/wL/7pL/Pn3mhjLX6LyF3ANTfQ4hks8YK7d7+Kac5SKBTOLCeK9rCsuXPritZ3sFYXiOMDmi9+E0PNI7UCudFHL9tEvQZOukAU7YAGJfMhyjQxL9C/gYGq3LityuNiv2yfcZivG97LxnMTMV83vHDzMF97CmUURRd+P+S9SMMjTXvcsk08XRJ0uyjVnKCcwVy9f/g9osbGiW/Cp5tk7YS8nxKG28iCS1ivI4UBkUDXyxQq97Dm55FWAepriLky0QSCQ+edPHXcxsV+2T7jMF83vJeN5yZivm544eZintSuXJKfNDiJTpZHOFJgxRJNCNIsGJ2fOCzn4ODg1C8VQfCcvPkEJ26c+CY7GJzq4rc/AqBQfpuivoS7eAu94uC6qyRJC8OoYDuLlBbukmb+RIfuzs7OjvWZJPbL9BmH+brhvWw8NxHzdcMLNxfzpHblkvwnkuFUg8MACzloEuIoZHNz60Q5Zz0107QLQP/UhgMlBFG2S64SDKNCmnYID1J0y0XaBgCOswKAadQw9BLkGaEQZN3uhVDX19fHhnPV5E7HYb5ueC8bz03EfN3wws3FPKlduSQ/kdSwUgheTt24Ssey9knSbKStMa4cISSWZZz4TJoOqh8BiiRp4Tgr2MUBS6dQeAMAwxhQm0TwglyVEL0GhdnaCXnioeV5OjaW43Yu5t7BeJ9JyvmEPpdVzx/FwyGSvTpps4k6WlT7rHW5rvvKbu1PU87n18XV8HmdduWS/CRPuT+v/X+43bUTu1oLhSJ5luG6HvV6fVSOlPLUNuLBjwytTLe3P/jo6CbLQ4l0BjxW05xB14uoe2e/NklLQ3lLmMY0rdaLl18csXcAuv6PXtY6wS63M2PPUmiuQ9w73+eU+f7FbxWTljMO8x/G6KextUH38IDm7vYrPp+6jS/BJw9DMr9Hur9P/4fvIwwd0hR6vcGO6gt2OPZ6PVSek0cRKsvIWi3SZpNke5t4c5Os26X1e7+HtCySrS0y3/+xxXWV2/g8u6mYJ7UrR6F0HGesGtvb2nOm9t8DJcmylCRNUSpDCIHjeARBgKZpKKWQUhIEwaiMMMyR0qYXdTALZfwn62Q77w/eACoa2r1VpFNCRhXCMMTovdR/PkGx2n80mu6RUqJ0nf7uC6L6FklrQItTuUYQtMnznNnZ2QENauP7Z8bU99tHsXeJou5Lypd/QGRVyfwDoo3vnVDWPJfyFT0hToILaWzD1f2LaGzLy8sX0g2llGMplI7jjKVQThKTpmlkWUaSJOiWjWG77G9ukKbpKKbhPOZFMdm2PZZCeRae0zS2wcljF1Mo81YLlaYD6t3ODv7uDug63L+HrFRIi0Vufe1rqOlpUiB48YKo13uFmue6Ls333iOt1+k8f45KEoJOB1kqkRYKpL6P+6UvoTwPZmeJw5Dg44/pra2RhiFBp0NyeEi6tkbm+3QPGidiCoKAXpjQefoUub9PFIbnxjTUgLqIbmjb9ivXXpKkbOy32Wr2CIIAz/NeKcPf3yd89DH+2hpxqzVq44vohsNkeLyf8jwn8H3ivT2C/X2WlpboPX9OuLc3KiMJAvq+T7SxcS6e0xRKwzDGUiiHdhGF0rKssRTK1dXV10ah1D/xLz6jKaV+FfjVd99996/p+snqdV0nCAJc1wUY/T98vXFd98QTzjRcNCkxDB0hEqQUBEHI/Pzi6JBux3FQShFFEZ7nUZBVenkdozhFFCg8FDJN0EyT3DAQloZl3SLrRBimie8GFI8WSoY4PM+DqTu4BQ+lDPL2RxglD/VsB/w6xswCmCZKzaBUgJRlWq0WCwsLOCJ7JSZ6BzidDYJgCq3soFRGmu7jeXdBKPSZO9BvoUVtgsMt3OnlV/EwWNDJ8xRdq5KlTRxn6WXj5hlIbbQRYzhF4DgDbWtNa6OUg+PYKBVj2zY7OzssLCy80ofDfhveZGf10xDP8f48qwxN0074nBUTQK/VIo/6TM8vYhwtdHulEu39OqXaDFJKWq0WjuO8UsawbNu2x+Ixs2yUWE/g0HXydhunMpi+y7LsFV1wTdMgTTGzjLwXkPd6pAcHaO0O2nSVcq02qOvI/0QbmyZUKiT1Oqrfx67ViDe3kGGftt9j6t13EZrGcIvM8Oh5F6BSIQiCER6jVoOjutKDA/QgAE3DuHMHkgQ7jog3NtDCkNR16WKQpBlBoiOdAvMHh1iGjnAdpO8j5+YwlYI05eCgQe4Orp2IGIHB7r6PqUn6nR4FS8dL+pSLhUE75YqtZp80z5kuuBiaoBUk7LQPuTt/NPVpWLyodygEEXvVRVxTw04iOk/XWXBNmpqNrEyR5SmquY6TJ3jTKyS7uyT9HrFtYeQ5sWFjWBbxxia65yI9Dyv1qf/OrzFz6y6yvED2/ENEHKKEhmY5GHfeQBwNBM+79obEiiiKXrl2hvfT0HeYn8669ob+p6/B4bU3LMN13RP33vH76dPYpSf5cTYukOPTl0IlR40qCLIUPeviuh7T09Ov7BgbPmmzVoSo6OjJNKlRQEWKqJFizzTJ2jH68rH6sxx3qnASQOsFuDXIE6RrkHVySu40ceMPULGJLFZPAg2aIAqEwye9Nz2YY/dqL8tMIxASx9SIASmPNl9FTfSjaZrccpDzX8Jpn6R9Di0InmNZi/R6jzCMAqY5SxwfYJpHN/vGb6Pf+hYIQZI0cZxBsvJ7H2GaNeK4gYq6BGkd217A8+6fTfXqN0FoYJdO9lUcwP5HsPS1E+6flm2Q54OHYdTr4R82yLKU2vIq+rF+FVJSmZunc1Anz3J2nz2hWi5jnZHEJ8GTNhpk7Ta6aRJubmI9eIAQgqzTITs8RKtWiTc2UEmCfYzDrJQire+T1vfQp6cHZczP483OolcqGPPz59Z5uo2N2VnyKCJ68gRtagp9eQnz2Ij1k8al115eZ7pSCNdFOzpIZ/iQDjYOWCnqaCtzIASNXkw7Sig3Gnirt+ht7ZD0+ijX4ekHz/nKnRin4JBqBTrC4N5MgSxXSDFIcnvtPvtrezieQyfOmdEypueqCCHIg4CFskNeMNhpBiAlwWGTBRXRMV1WplxMXdILdcr9jIO0T9zrMnv4CD+EQmGajlOm8+wxRtmhMlPElCnYUxC2QdcxSjUIGpD44EwRLr2FvnwL/D3kbBXKSxB2QEhorYNm4KgcohxUDk4VdAuMk216meyaSWiWk9qVS/JRFE1ESQSQAjQpQCp24g7zZgfDMAdTJUqdWY5wdBx5G6EfEnS7WNhkrYDo8TNg+qWjgninR17WcDj2wJAG9OpgvzwbNvZTiDKMahU5f4ck7KEOPySPW5jOIom/gXqyCasrYBYhDaG7C8XBzR/1NtHnvkTSjxHBDubUW2iaR7L+G5DPIqs5Xf8jSsV3iIwK9rHfDi3LQqJoByFNpJxD0xzStEun80M89z59K8NqvE9sCHKhUKqJbRcRaGjSxXFWkM0NdGsF3V0hDAdz3p3OD5DSwvMeEgRPcYIUaZUgbBNhYFfmB28JvTrMvAnNNSgugm6e7M+jNwkAkhCMl31zVp/Xnz8DwHQcphaWiNPkRIIHBnW505Q8g9DvMrW0Qp6lBJ02bumof7q7g746umGP15UHAULXEaZJ1m6T9/sYy8tESYK9uEi6vY1SA6kLrVJBK5WQngdC4K+tYSQp0nVQcYwsFnG+9CUAjMWBQF4Yhp/qBpOWhXX//oXtc9o+jY8Qgr1OSHW6jO4aI8y1gg0Fi33ToNMOORQeS8s1elHKnTdvMXX0gMvabax+m6ybohWLA7Zba5OykkzpOVJo1Pot9IJG8qSOKE6R+x2kZRKuf8BMrYbq94kNF2+xhqdnEB1Cu42XBITTd6i4LwdZ7on/Byy3nUaHpmbgCg27NHgzMLM+cuo2aAYf73XZ77SxuzG1wjyGdjRDbR+9C1mD8qOhfrtSg4FMHEB3B8wCFGZ/rP3w47aJrkEhxC8A/x2gAf9AKfW3T33/HwB/HcgAH/hFpdQHnwaQYRjjnY5MCh1NkwhdQtTHNqZJ0wyl1LnlSFNCBnmQDl6TjggwyW4P/faxXauaQFoamnPqqK6dP4DqHSgNlS4FIvFI8iomMYkWEHQfoaV97NQg7wvSQkjJ2EXtfoCYuQ9Sgn/EhCjO0/PXcasrGM4cvca/wircRmo2eukWfv0pYk/HrNSIol0Mexa6bchz8I8SmOlhmlW0XCLdZZQajPqsbhtFTrz3a2gz90nzDK2xgTf7E3TzQ+J4n0LhrcEoMc/BnkaTOiQhhlGhVHqM694lz2PCcBOACB+7+BDReIZh6BD50HgChgumC3IRdn8A3gzYZQyjMLhxtr8/SLSlBfD3B7iLc2f2ea/VpDw3j1Mojj4TmnakO6TQMjVI8KY3uCE1E9sQLBQ19PYm6dQtOlvPKORdpH00akpDVJoj+hnKsgYS0o0GwrSQtkXu++gzM0jTxNQ0hJQYS0tknQ7SnkEcPWDE0eu5u7o6WJPo90EI5Bmjs3HXchKFzM/NnfhM5Tni1G7IYTlZro6uOJBSnOlzkZ3lE6c5cyX7TJ+ZooVSioWyjRCCKc8kLrxUeNUcHS3ukO2uET1NQAiEVUQpgVZ20HSFXkrAdtFKZYg6oEuQOfKn/tRoqmP06M5zCFswfQ/yDGOCzfizFQ9N0+hFKc1gMLhTvZx+MKhrca7A7anbJ97sk24EcY40JLJoIo4kyvMwRdo6uEdv4970ibo+bRt/Gp/FxfOUdD+5jU3yQggN+LvAzwObwHtCiF85lcR/SSn194/8/wzw3wK/8GkApWk6sUiZJiSGFAhdIlSOZy6gjtaSzypHJTlpK0K3Y9J2iih7pOwjSx5ZMwKlCMMdLGsOrWCStSJylaEPmynsgMoGr3TaIPnLokF6YGK694j894g3dZSWENa3Ec0FrNVD4lgQmzH9ziHuogumhzp4Qipzwm4TkVQg0Qk6e8jaGyj/gCSYwixNQzUGJbDNReKsQRwHOOVlaG+QqgQ9yFCGC+0t9ATw+kRaAS0fTA/ZcU4+9y7Kq6FpFrirkIRYcg6rcEzEbef7g7eDwiw019FinzS30Q83obRIlggEHsJ06fc3cGv3SaMILWoMEnZ1IOqGbsLyu5DGxNs/Qpkumi5h7m1IgsGNrFuDB0J3F9KQ1F1AKkGU7qLrZaKgR3Vx+WjBagPHuTVYCGx+F0MZ5KGPcffnSba3EJaNXpkm83t0th5TkiDa72NKQaOTUnl4CykM0rWnqG6DzHERcYLQJCpNMVdWyDodhOsij+a0h9dOZ79O2POxXI847DO9tIJSCu1oQdCyLMQFU0NpmpJ2+2weNDFLFUK/xfLSHCKN0QyDXqtF1/epTk+ThCFS08iyFE03sD0P03FH5UgpeX+rTcHWcU0NQxtsAJzyzJFPfEQiMzSBrslXts8fvyf6cUY3TPCsl+sraRyRI07cNyemifpN/CCnWq1C/QOwSlC9gzbzkON3WhRFaJYFWYJCQxyNnpUzRRiGOI5DGkWjhXshxKAeKV8mWKkNrq8xuWDYNq6u4TiCtBGilW20pdJg8bub0NxpUJkelJtEMc1Wk8JCBdMwMToxKh0cwWhPeaTtaBS3sDWkqZ2oaxI8r8PH9/1BO78Gm2Qk/w3giVLqGYAQ4tvAnwVGSV4p1Tnm7zGJcPs5NmmCB9CkTkF12Q9DBCYl16Fv5qNy2u326CLVNI00SVFxDv01oIJhuIjZEjzaROUpem2W3DhavW6uAdMn8TSeQO2NwUjjyIQQ6LaBZhqkzNMP1ijV3oYdwKsQPdlBzizQLtQo+dsMeZ/dgoSsj9yvowclpO8RRe9TqrxB1uojwy2UW6LofgGtZJG2IqxKDd/fIElThMjpdT6kNP+z9F/8GpZWgqkH0NlCMzJor8G9nwWlkMdvVLsMdhnt4DmoELxZaDwG3X75dlK5Bb194mePYG4a4h5ae2cQd+xj24uE4Ta6Pjv4zVk8b92kq9WIOj4lM6JQs19O0YxGSGLwiv/iGaE1Teq0EGUTmdRp1Z8gdRcrzAl2fkBqljB0A2v+myTxIZ0f/Q6WVcMoVwgfPUJoGt2mz8y3vgVKkTz7kJmf/ALN588wkgSruoDx1pdp7myhwn2MICXO+si2i2G7YE3Bzg9ACDTdI2UJoUlmVu+MQvIPG0Q9f+BjmlAsYdrOaOSdxxlCl+R+QmenTr/TRQgw5xcodtqQCrY/3iE2UhZmKmiGQabpmEOiQblC1OthFwqEvs/GxjZKM/Bsk82tJm+vzqIfJcwozeiGKdutPu1eSLvToWCbCCmRmk4Wh4RJjm7ozJRc1veaeKZE03UQEoHA1uFOSfHiowaO6yENk34YouUZXrmM5RbQDZ1efZtiwUNkPr3dOtWsBNP3Bw9rGDFQhguEmqZxsLGHYzvoiUDzTPq9Pt1uB9fzCJIOeZaRqhzTMCHNccoeh41DvKJLaoIWKpIowdQNbNtGswxEqhBSoHKFMDXIFMictBUgDA2hCYwZZ/Bmf3RvaiUT/zBCUwFxP8KwTBbfuPXyOh2+gKUGUtcHI/lhX2on36guU4Wy0+lcapJfAo6v9m0CP3naSQjx14H/iMHb18+dVZAQ4heBXwRYXl5mbW0NGGgnm6bJ3t4eWZZRKBSYnZ0d7foSQrC6usrOzg6NxmBjkEJg5AZ72+v8KOiRuqAyE99v0+9vMjU1RZZlI5U6KSWOZuE3DyGsk/cTKvPT9IKI9HCTpOnidX6EIWvU62uYB+9jzL1LIdHZ3BxMVdi9PvOrK2xubZGmg3me5eVlWjIg9EPSMKSkL5P6ZdppGfQMt6hwSy7hXpXd6UXMD77H4uIXqO+2yZohZrrPrZ/6eQ5e7NIKEvwf1qlOVZHpLgdbGiQ+pdVpXGGx+WSLOO+g60+ozczQ6tjU4/cxuhGrX/4Ge/sH9KMyWTdkcfZt4nabZnOg51OpVLBtm93d3UHHS8lyus/m2hOUZpG5M9wGdnd3R0ePJfYsLb1K67CHSFwqiYZpzbD3Yos8D9H0A26tfOnMfoqiiH6nzcLyCr0oYuMPvo8jCxTnSmiRopUN3jREY43azBfZ2fk2pnoLFa1RmkkJ01VoPKdZucfC7a/R3H1E1lbk4TrVcpnULlFXh+j7Ol65zFStRrPdZm1tDV3XWb73BTY3N0mFJIpjFnRJ++CA7c1NvKkqZtzGrNxm5/FjZOxTqxaw5h6w2wroHTYw+x9x662vsrX2GNHeJDcLLL35LkE3Jmh0yVVAqHXJdeh1+/SDDuXqLFaxwvbBHpYZY1ZrpE6VoFkn83TQFLfvrLCzvsMPH9exFEgr5zBt0D/cZaFWGdzYYcjuwQGPdztURERquMR+k+9tfoxXKrK0uEDL7xFHKSKFcq1EoSDJsgiVKyzDwi5ZNBoN+gdNdvYFD1ZW2KvX0RyP7sYT5udrBHHK7/9oG7tcwdCmifMyQRih8pwoP0Bsb9F8+oTM0rGsWSrTZR59tENjTuEWnnDnwT02nq/TjLsUKkVKuosf9Oi0OpSnKiQqRZga9fXnSCWYX1rAcmyefPA+Skgsr8DK3CL1RoO9/UMs08RWIeFOn1b7kIiI2cUlmq0GnX4fqekUi0Vc12VvfQ8yhYpj5hdm6XYiNE1HHQxons8ffUh8RANN0gz7zbcI45h+HIKAQqHA9vZg3cmyLGq1Guvr6yPW2erqKvV6fUTYmJubIwgCukc720/fT7ZtMz8/z/r6+iiJ3759+8T9ND8/TxiGNBoNNE07kfdgsCA7zHuNRuOV+wk+3TTOWBVKIcRfBH5BKfXvH/39V4GfVEr9jXP8/23gTyml/t2Lyj1PhTKO4wu1lH/pOy/46V/7k7SNEs6f+B+Z09v8dpAQJNv8a7MziJm3MM1ZLGtwkReLxVHHJds+BdtDjx6TiWm4vUTSfI4ZaSQdk6y2hl6+N6Aebv0+LH3tJJ4nvw73/8S5mJXKB/Pm0TTJQR/7XgWVZQhNo/nRLsX5Kfqdj0jiBBGD695F6jnGcm1UDnsRsmCgFyCLNVSYok8Phhv+d3+A+ZU3EaKHUhmmWSNJWsTx4YBuOWEbnvBJ+oN57dKrF0+n06FUKp3x64EFwSEijMkyH692D8jJshBd9+h3O4NRpWFimiZRp4e/1aDX3ENUmtRmv4xK2wQvtjno7TL38KvEkYaIYqaX7qGVTq6FDPFGGwcofx85tYBWc4iCbSxracAdz7Nz8SZhSHN3G6tYpjx9cq41iUKCdovQ93GKJfx2i8X7DyE4HLSNMzWY//f3SKMi+twUkd/HKjioXPGDp7tYeYSyinTaDVZniqROmTCKuTdXPpMZo5IMlSs+/PgZrjAp16bZabTRFPSIiTSDpbLJ0uw0eZ5jmiZJkhAEAYHvEzaaaL0eojyFa3sUtEMMu0QSt+nGJrKfktNm+q2fQmiSrBfTa9ZJu1uUq1W08iyk8YDlJQQqz1H1TZK0ixZraFMl0mZItriIyWCEmyQJ3W6XguvQbjTIAdP1KFsFDp6vYXlFMi0hTxSao2EXilieR3N7C9NxcUpl/MMGlbn5Af8+HCTuPM+Iej5CarjlMpquo3I1KN80Ic8J/C5Rz0czTKSmYVgWav8xcWMd3fFIe236OIj9D/D2voubHpILA6kScn8fGTbJam/R+jf+GwrVBaQ0yVVCnkWYZpUw7CBliqbZmOYseR4CAk2z6bWaBL06UXhIoVKjWFlB085eOI2iEMu6eFF1kvvzonvvk6pQTjKS32K4lD2w5aPPzrNvA39vUgCnbZwEpxrNBClAglII00YPY/rq5Qp8vV4fzfm5rsvBwQFamCELIYgSmq6hpCT3fXJVRn9QJm9pJMkhTj8avIomIVLqcPAYag+gcDYVbohZCIltL5LnCVp5kKSGC3VGxUaakrwnwUwhlIhSiFLFE+VoSwXE0aKaZoByXnaRVvbI9g6xZmqoqEe09Rzz9m2y/KQGz/E2VEmOMF5t05GP4ZygiuW5Gpyba2gj/m4/znCO5iZVrkb4DK1AdLiLNuvQO3hKZgaY5jRJu0eY9ZiavTNIDI3H6Ds2Mt/Dq7UolN+m8eIxve1nOHNfZemrX0S2e5iehXd3njzOSFsheuUkEyTaPEQlEfrcHYQmUF2FbE3hux+jKZdcWcQqR/QN9JpDlgdI3SJNOxhWldnbd1/ZtAL8/9y9aYwlWXbf97uxLy/eni/3yqqu7q7q7pmemR7OSKKGIiWTEEWKkmj5g2ADEgUYMmDS8AcaFmBblizYMCDIhmHYhgEDMk0TsAGbhGmLFCmapM0hh+RsnJ5hr7VXZuXy9vfixR5xrz9EZlZlLV01PUNh5ANkZdbLyIgbdzn3nnP+538wbYfm2jpBbw1N1/E6p2ay1wWvi8xKNMOA9iXU/QPu3Fug+z3i6TGUOS+1FI7jYcoILl09RxGNlw992lEUkaYp7XabPM8JwxBN0+j2g3M8dLvj8WCVcPXUNaE3LIShkS8SyjCBUuLbDna8RDdyzLdeh2ROOr5HVPawzAaraUI3CDA2Napil+rwPgIJeYRjCrS115CljkoNVGFCnNSRXKnQm5uosIM28KiiAmPPeohGoQ4YOo6D5bisbe9QFgXJcsF8PqR5eRPHr5EqZz7nMs9Y7t+ipYUYCWS338eb3YZqhRnPsL1ODceVBV48hZN3HiJahIZSFVQ5RCM83cJzu6AZ9ed5hMiWPKpOH1WJcvszaOmcyu8gB9eRrg+GTav/KskyISvn6K5Lt7VHWYbouo2h9wHF5Ogd7MCnSCTpaojb7NHo+nTMqxiGQVHMKMo6Q9m2BgihUZYRWXaElDpKOTjOk/klT6y9j5DHc4i+E3mRO30FeEUIcYVauf8N4F9/9AIhxCtKqRun//1x4AYfU6qqesEX1DB0D5ggNBNLVSRKnCv5qqrOO9MwDISofRSM2eAAACAASURBVHmGG0FWUGrbyKpCc3w0t4UUBVWV4nufhvkR2AEkMyq7i5GFNQTwUWz7R7RZ80xkcTEzbZzOubzRxvH3KMQ95GyEtmcjXPOZ9wHOFarKc8xei/hwBmEFco4wDbIPPqhxz4/MqbP7KKnIj1bYl548ETz+rPuTmO2Oy63R6pwXSIUjrr38EsfLFEMT7HY93vvWCZ01n3GSs6YLenvbmKZOleSkJ8dYnT4Ht38Hx+uTVCcsZ7dodC8RO++huS6B/iZCKQZXP0m5fRVnfbPOAmz3KcdjquUSvdlEJjHlRKF3HVSakt6/j55YaF5BbAQ0LA/dMdDbNo6oA8i33v8Gmggxg4ByGFLoC4Rps5xP8Ltz8jxntRoRBJtYlkNZ1slgmtZktVphmiatVouqqmqTWwJSUS1zpFFS6IJ7WoPttkk2vMl24CEaa2jxBJqDOjaxfABGvWkG9kP4X57nBA2fxeEttGDAYDCgqqrzIhxQU2XsPvI3VZijKkWZFbi9BkoliNkd9J6NCF6vYxx2gNPeRWQZRbxi41PXH94PgBaqyJEpKEthPJbAdS6ygtH7aHlG9d4fo/tbyNBHR8DiAEYf1MHQo1sgYtB0jI03CaxGnTuSLVBVgQz3EdEMgYZZZbQnD+sgnz/ZbiKsBqxO6liQbtTWUu8VuPyFGmGThUi7hW65IEvQjBrwUOYIzQDLQ7UukXRewjNtKBPIQorWFgyuYfqbFMUCTbM53D+mu9XlaHVEq1iBC0mp0agEN+5+i0JUlHGK1fBoW20a3T2icE7L79LuXznXI1mWYRgGptnBpA5WZ/kJssrQNPM8t+TxJLnnrb2nyXA4fC4H/ovKc7WpUqoUQvwM8OvUEMp/opR6RwjxD4GvKqX+T+BnhBA/DBTADPhIV81HyYtCKAUKTbehVHX8riyQj3DUGIaBcVrhCaDf7zM5OECYLqRzjJ6D0gVC71D5FUJpCKFhCKuefJf/LCyPTtujauhk5/ILt9loPX2gjZ6LnmwiX2lh9HoUx8fktyc4b7yO8YwdPpyOMY+H2C+/jBe0yN5+B2NtA+vyOsW9ewjHoVpFaK6D0PWHkLtFhrXp14ii9sX2GIbBIilAQcszsVAczhM04KW+zyQuuHlSUkmFoQn6lsHb75zQb7tEUrJtGKRZwQfDFWuBzXrTwdvZoSpKWjufwHQdBClO7iIOlzT3Po3Zu1hYxThNyjlrr9HvU85m5HfvoqoKLBvoUYwm6MLHfWOTw+ExAbBIQozCwHVdNE2rrSCnTXPjMmWZ88C4x3prwPG9GWudl6mWJUU4IzeuEY+XrOSEUA9oNWPM7AgcSbP5KvP5HFuYlPNTbK0mMDoOh2HK6jimtxmgiRIVbHGUZQSWQLfXWR0fYxgGrdY6pszqYPXqEOgTz0d4ZYJVGnS7fShCyHR0O+CjRA8slFRYmUaS34fpLbT+a+h6A0jJowMoM8T8Adrt38MOTyg1A83w0YItMD2UbiB2Po++GlLu/w7l+puoMoFCxygW8M4vIcJjyFcQjXhe0bkmoLwuOG3Eu78MgDJdUBLlttGCTZRSVEEXVEV1/UdQ/aso3UDzN5DtHaTbQAgbJRM03UXXfYQwUCpHKYkQ2qnbr43UTDTNIM0moEo0o8M0mxJlM6pqhZAVpmGSVk1a7mcYhu+xjqBY7mPpFlLFHCaHWLnFq51Xzy2rpExwDZe1bn06OkMihXlIWmVYQYNRPsVKLPIqxzVduvbFQKgQAse+aN1/tyCU3015IZtAKfWrwK8+9tl//MjP/+53q0Fn/DXPE8FDhR7YASd5QlJIKFOELSiKgmazPqF1Op1aEdinUe3GOsLUSZMEQ9Tn1jQ9wG+9AdGoNhFPJT96D8dp16cJ7+nR7hdp8znywDfBf8haKXQdveFT7O8T3btP5we+cOHvVFWhqorF0SHrr79OkSQ4r7+CMGxUUmKd7vb5wQPyO1Oc69eJx2Mcx0FGBkbHQXMUMq4zdAFWWcl7BxMurbUQQFpoTEcxL20EFI5GOUzoegZX+k32j5aUlcL0BZcHDVprDxn0kiShpdX3HC5TslLSESmNfh/jdCLnwxDntd2n4sif1n9Gp0OlaeitFqqqKA4PQSao9U0qoWi1WjQaDZRShGF4zkvUbDbxPI+bwxVZWWHJFt+4MeGt61ewjJrHaDpx2HJgVTaJk4LLvkUyShgpiZxnJNWYrFJkE8mgbeJ3A/YnIbveGroGm5vH5NmYWLUR2pCNjSsIYbBaDel2+5imRxRFJFLieT32xzGdYkq6CtnYuwaL/ToRTjfrhLBkhm/IOi5iunUCjumeI7CgPswUH/4y1ge/hKF0lN1E6gJx8h7+4dsgi4+cd49GA2xqY6NQLivZwxIJwg3Qd69jmAaytwvWAGvtKkWpY9qnp2fdpNx6gyp8wGgV0tt+haqwSMf38Dwfb+ONC89KTzmLAJ6FI0mSBM+v40D5aV2HSmaYRoeinOE4myyWB1gmLLI58yLHNn2Qx/i6wZbbxLKukGcSy3aYHC1R6ZSXgh/AVDpOs55/RVWg93QGjYtEg65xcT6eoYMCKyCg3nybZos8KVGA51qMj+Y4jovtGZh2PadWs4yqlDT7LpomXkgXfDv64rsh33MZr99+JpgisFvcLTWcRrdeKH4dPT+jHQYoDiP0jotSGaK1ff6scllHy4tijtF6C0Yf1kE2AN3Edlxi4aPG75NmHr2d3Sda8CJt7vcvunq0U24LY22NQkrK0Zjm576PYjjEPCXaKqdT4ls3YXMT9/p1yrzmlDk7jZSzFOHWrijNayMsk+JkiDabwsYG4hQDrTkG1TJDlZIizLkbZ7y+0+XmMOJTu21u3p6y1nYwdVB3jpBOiTC26VpNilmGse6i4orWxkWK1EfbkqUJcZbz4dERb731iVo56wZuq/1MBV9VFVVVPdF/leuiU2+ASbNJrBR6lTA9mrM96MB7/xRx45/T3Pk+2Pkczc3Xzk3pZjJksHWF+c0v84qXMLwXkjd2cbwGnkgJbvwazb3vh94A5ndp9jzW2zWkrooKFsdTrGsZ0yTgQQR+w2E4eptVkrHR2sMxbbzlV2vK6Q9/jtJvEnztfwHTReombjJDbHyGxOyyd/DFuoLY7B6M3nv6vPD7ICtUEdca2GkhWrv1QSOeUi2HJNUGCWBqBaU08LUJVm8Drv8YrL0Gncuorc/U8/o0WFx62yzHx4hCRzv8MsrpIfuvIqIxwnbpNjJwWlTdV4mWBVE5xjLXKLIKcUpPoJsaZV6hGxqm0slEm96GDpWAStHc7FFVktUsxXQMkmWO6ehoqmIZLslkRKPRRaDhNEwKWTCOx/TcHo7jEOYhlm5xmCzRhU6lKiy9QskKa/kekWlhLFeYXo/r7Z2LHZeFML1PFRbEStDvNdGEDQ2DshKsZhlydg/dsmnaOizqMGKubKTuYzo282GMrcfkuUCzXeTsAZXVQTc0srjEcg00XeA0TCaHK4oEPF8QTlLCWYrlGHhNq84DeWRNPE8+jr74TuR7Tsk/Sg71USKEqg88uomhWxQkaKZR466VQkpJWZbnSkjoAkezSNIlXuvhs84MJ/vM7JIlbLxZ/+z3yY4/pGy+jDB8pg/2aW9s1ljjb7PN9+/ff6aPzVxfx+j3SbIMs6rOE0SO3/4jDM+junmDrR/4QcLphLws6W/WJyC9bVPNMzRbRyYKoftorqRYzLFEg3I8xFyrTyV602Z2GPLBUcjepRaiKri2Uf9ur+EgDI3i4A7Cb6IHPtXsgMPxmCuf+1NgaCgrp5zNMDqdC+/tGAaaZaGGRwwGA9qXt7h7+wHdPMK7tEM8GvE043Q8HteKWdfQkTRYkd//Gmn3dZTboamWZHYPzW6wtt4nfe83GPwff/PiTb7+P52O0wD78hfQbv2/mGl9KjxL8zo7vylvDRGPnj44rd06qUdWdDSDyvDYvPJ5BqIC08MqNbR7v4eKp4iTd2qf8ak8elJV7UuI+X24/2XOtkPp91HtXXjjr0Bzk9Lx0BfD2vVw8HViu4fbaNe5C3aHanXE4vZ9ZKHA3EbbfBPz6ucJvv9v1TGhMicLY0Ll4Pgm5ql1msQxnu2BHaDal1gcRrT2XkfTBOJ6ndVcE2Ndu/DqOtDsGcDuOQPi40SAAGVeYdo6d2/fY2dnF8MUWI+AAqJVwsIaU1QFxdEBbuDhVDqz7BAlLLJFmzAPubb9EmEeMlwOWWuuscgWXLG7kC5YjWM0B0Q6pWpcQhse0Bhsko0TotltbFejVCYkM+wgIFQbyFYNPTyzUglPMGRJgxIGHXA73Hr3XdbX1im1AE3G2PKQbKnT8Wv0Dc0m88M70NjE0yaYqkDXdFy7wXTpoCTYrokelEzyEcqVaAH4UkI8ZxVBZA0oVEmVVzT9Jl2ntvqlkmjiohv2O9UX3678C1fyQoifAH7i6tWrlGVZpyGfUgKfneqSJDn/7nm1Cez7NYXwGbhGUNPOlrpBWRSAQhYFhVTYQpzDJs/w7Ck5dmEyC0/w1q+cPyMej9EqAwjq0oF5gqokulbf37JtVkWB3btM002Il3M02z1vz5lPuKqqc1TB4+90RlF6Rhf6+Du5rkt6ekrPheDkD75E5/prqEaDwnVpeAESwDCR0YrhwT6Dnd36b22L1XBBsN0lGi2xNRtr7wrFPGcpSuS9e8jZHPv11ziSJVdfCmjMlhRhimtbLC0b3wlYHd/D6/Yogga265JLWS/6oweMb98ksByU62BlGVGaMLi0R/7++wjLIgai8QmmZdPe26M8eZ97laT54R+y9qX/hCI8wEwuKthW52UMv4M4+Mr5Zw4XERKO4VK1L6PP79IoH6Jiqr/0j5Hbn0M8+CpaMkH8/n+DeOeXiIJrtKko7TbGK3+BdPPzODKmGN3E+PCfojIb9QM/i9RttKO3UTufRyUz9HtfhP2vQHMLMb1VL4ob/+zJzclwqTY+hbbzfVSmh1h/A6nbCMOCKz9YB9R0nSJeYJdLEi3Aa/eJogjLKknTEs9zyXOBYUBeCYaHh+zs7LAYJXhNqz4N/wUT15NYlkO0OiFobdfzBYiyAr/TpopjyrxiMV0RtF00obGcxmRRQSUl3c0GlSzRdPOc/vdMnph7aXpOf3v2/SwL9SxIWMr6d1KUOL55SoNrEq5Ccj3naHXEK/1XUFmOuVZSta+cbxQqjynjIVumTRmWyExnR7UohiGNUjEuDzD9HqqzS6PlE856NLs+erOmSDDdNoalkYYJeVbQ6F9hOM1p910qVWB75vk7DaXAsFxkltP1ejU1sGORByaWLimkjtK2SWXMKA2JZiWlOqHl9kjFBE82GI8TWj0f4zjEcw4xygYPwjuYuk/P7WBpPjLNCYuIlTug1XKospSNzi5RFGGaJnfGdzBsgyzNaHgNsiyj5/eoygrf9s/1waM64ozT5lFq86eN07crz8XJ/0nJs3DyZy/2LPmFP7jHD/3GXySybBo/9su03ZAHep9vvPs/8uaVt+j7Hbq7n+Pu3bs4jsP6+jq6plGFBWL+HqvpfZqf/onzZ1mrCGPQJwzfodn8ZI0kaG6f+0WjyQOWsxXtjS2KJEFoGkHvoin1vDYD7O/vs7v7pKvn8ft4nsf+l36XRrsDuk7n2vULp6rJyQme56KbJtYpL0u5yC4Eemd3TiiSkKUhaOuKdquNqiTfDOEzuy2yGzeJFzleJ0DGAufaFuX4GGGaWDsPzeL9/X3Wg4BodILeapFEEeVsRrC+SbkKUbpOc/cSJ9/8Bv1Xr2KmIbz9v6J/6QK10blI3UGrarinclqIdAFA4Q4QvZdQ2YrK6VIIC+/6v4J++DWY3QFZUravYHziJ+GNv0ZaVDimTlpUdXp/ETGMS8aTOa+/fPnpnasUyJIozZ87VtFyjp8ewc3frFk1m1s1F4/h1qn3vNiYf9Q1Z+tuf3+fptPHC6zagtMElnPx7PW8ZyVhzmy8JGg1CLrPdgV8p20+a+9ga8DR6ghbt1nmSwbegI7TqeMM01tE/iX8xlOCykrVVBaqIpxNcQdXEEWC3hqQp+W39d5FVXB3eRejNHA9l7zKUSgszcI16oPXJI0wKLl/eMBrl6+BgiKVhGlIVVm0RJNOx8H2TEaLEYbbZlaWCAQt02CaLUkrRVqmdEyXJMnptbsUaQiayYnU2HUsCgXG6RKdrCJe7bSwHgNRKKWYpBO6TpckTr4jffEngZP/FyovXhrrlL6g0UauZH2ilsW5QnQch3a7XUPhHnwdgk+g6RUMXrvwrGIVIYSGaZ4a+K2Lvr+i0KiKArcR4PgNVrMJq9kU3TBwg+YLt/lpA7aaTmh0e8iqqjNxUZwcPaB97RpGpVA8pJitZI1+2T0ltFqOh1iOy2J4QmtwkeTK8DUyw2NR2uiegaNXRDlcCo/IPjxBu/oSXdulOIqwX/bIPngX5/p1xGNR/93dXaLZlLa8i2h9P83JV1Bv/3eIe1+i2PsxzHu/irRa7OQL+JXHRufyn0Ob3yXa+2HkD//Dc4z4ZBlyMI3YG7QYjUZsbe8SuBazKOdokfLqegNNwTDKCD77d3BNnW/sz+k3LKKswjwJ60CYpZOXEtvUaTo2y0I9W8FDvWnrJr7/fGSD32xDs31hrjxxzXPGvMiqp15TuxIV4/srvJaFp3VxA+uC6+PbfZYbWLjB8324z7vPIlvQ8ltPfC6lIopiSjOnCAruTu5xpfkSmJLNxubZi9WEeetv8MynCFET1AHB+ToLkEqyVHPiZUzX6ZLLnK7TfWZ75+mccTLm5fbLKCCvMlJloAtBLhVHeYGtafi2S6EUu+td4kJnskjwCo2stUbXNkmF4qtZiplnaMKlijMGloGtaRymOaWyGdgmwgmIK0mv3SCXCttpMitKrrsmKeo8d6dvmqx12hiaIJOS23FG29TREQxsk77bf6FxgKfri48r33NK/llFHR4XDYVu6AgdqkLW3Buajm0bpGn6kEtjfh9QD1kfH3vW2ZI/K9C9HA1prj2MxM8nY/pbdaBWCEGj0+Pk1g38R3glXqTNh4eHT6QkV2VBNJ8hq4oiTVmFSxpBE9vzsb2HE+HeJCLKKnRNcH8459Kgjd/uMjnYP6fWjeazut5tu0MUrzjU2nxit4lScDKecbDI+NNXdqmUYjEa4nR6BDv1JuW8dg3xT34EDv+oplI+RWxkm5/HP/ryhTaf2RTmvRpspeWL0wExYedz8OpfhDd+krkK0HWdNE1ZDwJ830fTNNZ6XXqdNvuzhGsvXz3vv47v0fbM801ts+UyWWUczGLe3GmRpwnrTR+lwLV0pFRMopy1wGYW5XQ866l9/Li8yFi9yDWTkzmGZtWnTwFuowYfJmGObmokYUGWp5i6he2ZGJaGaetE84wylwz2AoQmCNNDbLfxkc/6brX5/Jo8guVp+UTTI3ZbxFVKKUum4ZRypTEwe2AVzMsYlULqRPTNAeH9nCuX9lBSUSwlpZ7jOQXM7lF6GxjAfLKk1Q2eyYGflil3J3fpBT3KKCfRUgbBOn23T5iHGJrBcXRMsooRUqEZOiQV5IrcUiw0i9fXL3NrtGKcJFxab5NOYvRS4ijYc02SWYJhabi64Hfv3mF9rU2/bTNyJFYccj8siFkyLx4QZlPW7U02mgOO44RFviAtc6Ii5IZuseFtstHYomVt4AiLapbRN2tSxCCox12VJaOiYJUklKaNq2tc952n9sHH1RcfV77nlPyLVj+xtRDTtpBlgpQaltGq0511k2z4IbZ9isk+K4K9OoFOAI/W3czSJ+6bJzFlnjM7PmTt0mUc10E9UnLraYP2Im1+FOlzJslyWSvl+ZS1S1cQtotyfO6FJS87dWD5eFkTTVmGxnbbZZHknCxTbEPDa7UwHY937x5hCw1bFxTDY2zXZztwsY06KNdqNZHlBBo+xa/+fda+/t9S9l5DugHi6G2EYdc0sHABkmc/puB582/USvy1n4Cbv1m7XBoDaO/VCS2nMp/PQUl83z/HBD+a5adpGnu9h5vY41V1zqTXsHFPGReNx4KBmiZYC2oX1RkT493xk338uHw3ijpkSYmQJq11j3iZY1o64bSeS1UhsX0D2zVo9ts1H32liBYZWVxSFZL2+sMF/rR5cSZn2cXn7YkmUJzO39buuUuxklWdTa3qOMoiX5BOKjSl4/o2nu+g66JOt0+XEB6j+q+wzJccr47oTseIPGbN32Z+J8R2S2LzHosopWcEWG4TMoGb3KuBC7GBiAv0QoJrEE0VuX0JozAQswwhDVaTFDSoCoXl1iiUUpYkScy4GPNy/2XSLGFUKHYbfW4fx/jGEsvUcHWdfGmjhE3hGmSzFBoWpqYIC8XLlsnBh2MOuUfDFtz9oGTICcfxEUflkDhPsO0azXN3sc+702/B3ecO+0fKn9/68/yja/+AdHGI0XFBClRpUM1SpFLINKXT7tCUFavDA6xmk7QoMbtdlOtgWg/dqR9XX3xc+Z5T8mdUpM+UU1+mEGBYJkUWAham0UT3HAwjIEo/ROqP8pMI8mGFuxGgLSbnGY2z0ZC+F1wIsDmNgHQVUhW1stMdl3i5uOCHt30fr9kiXszxWu3nt/kpkoRLGt1ezTwYRxiWRSJMorhEALdGK2xD5/405q29Np51OlQVvDdK2Gg6NIIGJ2GG22yTFBXNwGYWZeR5waAs4Z//R3Dzt2j7PdrLI5jcODeljclDSJ8ybNKX/jLRZ38ao3cZoemkq5Ds3f+Lwc5lnE//5JMvcO1HL+ChHxUpJc1m8zQF/KNx3PDRY3723k+7ppwkoOpCMPoLuGHOTqqD1tOLs39Ue2bHEX7LRjME85OY5nr9PK9ZbzCmrZMnJVb34ZI6q3GLJmj2P3p+FCcRwjHQXAOVliipapZFQyPLM/y1JuQhNNbrLNHlIaBAt7mfjginio5nMUsr4jLC9gwyI6FVtjm5ecJOPidLG3REhRy8ySj6FruDLbbTTYTSyJKSaVjhXXLwB2s0hODxXpJRTKUN0TfXWaYpgdPkwfg+QaNJq+FQmpKj6Ag/tGk32hRlzkl6ghdb3L4fYpoWUWFg6x5/ePeIPibaps+xUOxsNzGpGIdz5oZPa9NiPBpz4+R9YmvJH99+h1zVJF3zbM43R99kmS+f6Mem3cbWHZSqSMqYneZV/ur2X+WVzVfQhEYpS2zdRs8KmnqDlzpXaUqb95MjyDO0SmBJnY7ycPprZHnESXZCM4ZcLvGv76GfHlxm9+9hNHzS1QpvfUCUZWRFTufqVbRwRRYYLKcTPMuqwRzdHnrD/1j64juR7zkl//zo8cNAsWHpFEnFfl7SMRpIr4mu2UiVs7GxcVpWr6azxfQgXeA7FuFp8kyr2yOdTnm0u4s8u1CByLZsGhsXeSg6m7X7pqrKC21eDI9pDZ7Ob7O9/bDeqpQV0XyG16ytj7VLlwHQDJPdpo2pC947Ciml4nNXOucn8rNnvbltY+ga+9OYUipeWT8NclUFHc9i+d5vM/i5v/7sLvx3vo7MY8Rv/aeIv/xfMIvAdBx6nR5KSm5/8D5Xr79GufszH5md97SxKsvyPNv4Wde8yH2ed40qJegamqODgNVoTmfQIa9ykjKhZV/0LS+yBQfhARveBkerIzzTwzEcLO20GMjpqbjIK8oUlnGCbmhYgYaudCzXQErFapyxfrn51ILKj/vVn/ZexSjG6LmUoxjNNxmINuU8Q/NM9MBCxgVKKvSGhsoFwjVwVjHl7W+htBaUCqFnKNlFszTGR7cokxhXP0EVWwRmzHZvg57mAIosXnEFm1XzM8RrFWO1wFAj1lST5e0lJ+Q0NwN0X2OzvVEHhB+zqEZ5QdPQmYmUqllxc3VAx+kwzEf0+usoFCfhCVmUsGFvcENfsjSmfBiP8Nwulxt9qqLDNKpoaApll1BUvB/d5w9u/hb357dZZgvuhs9mQ2lYzVo5C5OGFfBm5y0+3f0svc4W0WrGILjCy4XLJbd2EyWWxahUbGYxQtPq+agUwrZReY7oOmi+BwiEadAdBWhNHeE4LOYzTNelnM8JggGXrB7llkeRZUSLGZqmkyUxrc0tDNMk6NYHwLSoaOqitlptB10pZtLC8S0UAuuU/+lF5vuj+uI7le85COXZ4nkWhFLwcHHlRUFVVcQZ9Pwmhv0KWTpCahbxYkwzOybNUozIpPA03DIhLepM2SRJ6orrp9Xoq6pCCMHw3l36u3tYrsdiMqHIc7Isx3Xd83acfU+SFF/Kmpmv0SBLM1bzGbbfeAIeNRqN2NzcJEkSdKitiN29C/CoaRjTdXVQOtcGbp3kdPd3kP/330doOsVn/03kG38dWVUY+Zje7/33HF/5axS//79hfu1/qPvt9AtA7n0B+UP/IQgd6XXRm5sUGFiWxXK5xP3Xfp7RaEQzCBC6jhCCw+NjNvcu17S0uk6z2XwmLDQMw/Os4jPaVU3TzjNSzyhWNU1D1/Xz4uqPQvPOTvpSygtj/bCPE2zbZrVa4ft+TVMhQS4zRNNCmIKDxQGWYzG/NcZac1DAqpizMFa0jTVszeRYHbJr76IrAyKDg9kYxzXPccyVVuAZPsPxhLVuh0TGSKdi9WGKp5rsrHVxGwHLdMXRyQzfrTNsn/VOtm2zXC5pt9ss70/wmj7RNCTY6hCPl9hNl7woCO0MxxJYusX90fu0rTZ9DfLDB0izy3I/Y55P2br0BkGrddovDrPRgvFyzP3RiFcu7eJmWzSDFsoUaFpKKXPy3MPQA6pLu3QdFz1ccKlz5RzFdTA94Er3Sl2m0DTYn+9TlRVSkxSywDB8wtLCMuBbqzFbQZfZwmBvc5P9+RFNu4QcYtPjIJqyLIf8wtFvcH/2Db45+RZREV5Y+2cJT4/L1dZV2k7AT23/FD2/xzie0bBc5knED136s5iayye6109Jz3LUcEgpNPydbaYffEDn6p8mTlMavR7JaTZplaZc0XUKKc/H4WycUA54EAAAIABJREFUZFWhPzJO48MHOA2f1WyGrSpEI6C5tkbSaqPbNlWe45kmi3DFrBBUpcJy2piF5Hg0rRklqxLdtFjODzEdA025YHqsexqrtKQqCyZoyKoky0suD5ofCaE8g3j//xJCeXYSfJb8wpdu8UO/+eMoN6H3t75FPj/it5c6n9gxafgbbMop49k92sEWRjIGIchnLrpTovvA2jWWUUKz2WQ+GqJmMzqvPkwQmR8fYXkeumGSpzFVJWmvPd28V1Kymk+xPB/DMDj88H3aG5vnO/ujcvfu3fPkhttf/wqXP/0WmnYx6fvXvnXIj37ytAhHPIU//t/hn/37Tz7YCmrT/aPk3/5DGFx/6q/iOCbLMmazGZ1OB8/zLijbZrNJVVW8++677O3tYdv2UyfXeDzG8zxmsxlCCDzHJ18J2gPvHA73vPH8ONcUJxHC1HlPz+iYAkPFVKqiGBdcGmxTzjMKUyOch1QKZu6UXX+PeJrjtkym8Qy7KzA0g4PjE0ovoSU69aFBL3Ath7RKeaP/BqZebwTHhwdEyQplwl73MsPZCSKX+E4AlsDXfYyGTWlU3Nu/RWutR8fssArrwjWtdhdOM0nzKsfST+MId++idwVOFtPIY3TdYmYYzJYGJRK3pdjG4ygTKAkdo0uiYu4vTzgWCX9qe49Nf0BaFMwlpElBkVWUp4eZZt+joWu0TIMkL4gQ2JogqSSTosTUBI6mkUmJIQRlWdGyNXR0bkdjbp58kbvLO+RVzr3pbYqiwnEdvjr8Grl8ut94w9vgc5ufx3d22Hbc2oVj+sRljKVZ9N0+Simudq5yzdimGVZ1rV3HQWUZwrYx1tYopUTL6mfIOELlBTKOsHZ20LyHVbMenztKSsoixzyl/L354QdcvnwFw6phqqPxjFTCYrHCEGC4HjoVmuXi2BZZeXEjsg2drKzIi5LLawG6ECjqeNl2+6EfoCxXJMmIRuMySlU1g+3HnO+P6ovH5V96COXTzOCLoh75F2QmAZ3dwGaWVUhNIRBIpWplqek1bXC5gsFnTmlg62SD+ckRjfhi8NWw7fMi0OlqidCf7a4QmgYK4uUCWRS01tYxTIsizy4EWh6VIkvpbu1cVPDxlOWv/D1+9J1fgF98yh+99Tdr7P7/85/X/z9T8G/8q3U5woOvwb/1O+cVl57HV32WtLW+vn4O55KnJ54z94yu63ieR5qmD10ZRUGj0aAsKqSqLZ/VasXm5uapFRTS3fTJ0xJZKRzffIHxrJ+drHLKXBJ0HZRSRPMcr2Wd1zKVUjKcxhDl9Fout0RMnEzQpYFpdrnqB9wf36vdHqZOFuZsvLKOzErWkxZ4JkHLYRgNKfSIK41XmaQTPvPSJ84zEsfJmL7bP++/cTKu2UwzRSVLNi7tUsiCB9kxzW6To+SIVK8tgYPkCLWSJPMVu7tXWE0W3J1+iNvwWdvYJI6PiIsYIQSWZrEqVsxP3iaaxXyWl+msvQ5tH0yXdUAvlvQ3m+fjeaVnIaXkJDphHi+Z+IK/svVZRkXFjSilKgu2fI9O00FDoGTBOB2zZvrMipTfm+6TFoKmrXhv/E3+8OhLhNkMS7Poul0qWZFXOTfmNxglY5SqT/NnogudveASVAIVTvnB3R/kcxufI0wWTKIVTavFy41LmMaATbePlJJd38NyXPIkptHroWm1RZjfvk2pFJYRQFliXF5HPKL0lFJU4zHpbIbTbiGEht5oUFQlzs42UkoW8/dJliGy0jENH8NoIMUKqMiikvbaHqvFnMXqPncPF+SGhqHZxFGG3WhxqWXS3x0QRzX7qGl6YFh4lsFisTi3SB8N9ud5foF6ebvtUlUpeV4j94Qwse3tU4vh2ar1RdbEd1O+55T8s2BX579/5Oe8nJCPZ0i3i9AFTSE4mofYSASiJlfSzPq74Jzn+0w0TcPUdPIkPq+n+ahIKbHcjw7oGbbN8MM7NHt9vMEGhmkSTsfE5eyCf77X69UkWQ8OCPqnyJ/lUV1T9R9d4dmlOYCf+K9rP+kP/l3K+1/G2P5MXUv1WX30nD4sy5IgCC74213XfSIYNBgMCILa3x+GIVmWUZYlR3fHmJaJYZj017qM91c4vkmjbWPaOqatc3s8Ya3wkKXE7JkX2rTMl5inxGazdEYUxrTyAZ1Nj8mDVb1JuDofJik7DRN9kvNeOCf1FL674N20g1Em/JnBq/UmIOBBVlAaDfK0ZDFMyNOSRsdGsw0026BcZBRaRSYzdoKdujB1HiCzHGyDcp7Rdhsou85NUJWk7/ZJxyHjfMrubp2OL5VkN9jlaHnEJ9c+CcDJbEyaKAb+gKE/JKDHxs4Wm+s7CF0wz+bo6DSs2o1FVdCJp1zb+XPMNgwyq2IoJG1lMD2YYekWVqCRVRm2bj/MlahWZOlt/ngxR5YT/qujX2GaTtAFlFJnVS6J85CTZMRhdEJWfTRC4xPd64TVjIPwPqZuopTkcmObt5qv4JkNLq29xpXGFleDDWJcrrR2OFgkuL5PqRRrlokuYD/NueRYDJdLGo0GRlkgpMRyPfI0wWu1iRcLlJLIkyH+Sy9RJCmrNEYzDbTFHL/doZIrKplg6AHG2hrCzagMSVGELA/v4AQe4QqKbIHXuEpvcBl0HSEyymrFKvHotXb44OAB4+E+QuQkcp2XLvdo+CaonLaX4/sFWTYjnhR0ug10zSYMUzTdJKrAsjI0TWc2qwO7Sjn0ej1AIWVOlh3XNSN0FylzbHvjvIjIiwANnrc+gdPnfXfkXzol/2jgVcqMbDQkX7OBDo6mMU1KNkWICKhPuf4arHKUeiSYatukaUprbR1PQpYkWK5HOBlT5tn5de31zXOqYqDO1gsuBlbdRoDXbNFef5zTRhDNZ/jtmusljyPmUUi0mLGuHcN/+eNPvJn83N9Bu/pD8OpfOs+sfKxzYPuz8BxT76P6MAzDc26c58mjsMcgCAiCgCzL2NzexG9bpKviHB6XrAr8lk0uJXEl2S+POcptVObRiZesawF+2+K+kIwfPMCQKQNvwDJfEsRdwt6M/bv38FWDrf4Gh3HOWgXfvP0hq5XBYDPnurfN5mCPZb5gdCKZn8RYbs0IuONbvBsW/O7oLuv9PhsbLuE0ZSFnKLuk63b5o5N9XjK2mBg6tq44iXO6ax6LSULRsWhoAjVL8DUoohStYaFLnZ2tvboPrIAgKUEz6etrqGSGKBLcqsO1LQMdSX/jZVZFSbzIa9dH32fDr+eMVJLp5AZ9dBi8WXPBRxE936eUJe/eucHW1hq5DPmd8QMMUpJszCIZ0bAcknzBz3/wi4wfoYjQhIZveOiahqu7NK0WW94u284ub65dR2mStMjpiyaJlEhZstdo86r3Muud1/BabaRKWE4eoCoNJW3a6xtUusG98ABb5UyNbV7zHQ7CB8R5TM8JiFbHDO0+oVJc9TtEJzdpeBYim1JIHUFFcnuEtbFNcWcMvobMFhA0mY0/QOgmXtMFZWHZDRaTm1SyxHW2WCzuIQnRdA/L6DFdGWxfeouDeYrv6Wi+oNI1bs8zdJHRcGwWkY2tWQznR9iGyfWXP4GUNQw5juPzeI6maXU8yu3SbtuPzPE5hmFSVTGWtXVaAChGyuxUsZ+QZXMsK8Bxdp+5zl5kXX27a+87le85Jf/iRUMEQpgoSvJsQV2wCtBMhCyppMRQEnQDrddErh4mnJwFxSopqdOqzmCZgomTcEbB9Wgwjbu/Cz/343Vhg+/72/BnfvqcIra7u4f5iM+60elxcvsmfrtTlzaLY26+/Ue8fO0VLh3+Ivzyk4Wzhj99g1az9V0pOPC0a8qiQiDI8xzP8c+veVo6+ZmMRqMnsvMsy6I0a05s6UkOkvtcti+DVwI286IiKxMC0+Wt3kt8MLlHpnLulgnj8Zye3qHrN3Aal2ksCyw8lFuxub7LMorQDZ1b6ZD3spvsWCaf6lyl8VKfxXJFaM6IF03u3B1x6dIambOk5Q1YnmRMHqxIxwt2LluU7oobySGTUrCuN1mLfD4sEvp+h2UFryqN929M0IFUEwRti4amEVeSwIebwyO6rLCtK3Tbj9lY8ZTl0YTRfEhmW1i9qxjLW4wTsJtbJPObeNkczeuhhM3kMGbdNPDNJfekhV7ZzFvbtCyDbFXw3p1DTEvjF4e/gqkrju8f8MHkm5xEh08dk4bV5Gc/+7N8dv2zXO9dRypJGcYsp5OaLM4w0CUIXcPbqN1oZblCqYo4HuG6HTTNxjDq9RDNZ2RxTGvtag0b1nXuFgpHVuwEO/iPFJ3ea+1yZ3wLr93FVjqkgjUlyYtjbNlBZCYqkRRSoicV7qXPUB6fIC4PMMwmhlHPpbKsaQXO5npRLPBabWx7nSjJMbtXmCYVsixwdB2rpbh9PGUzMJFlhtB0ZouEzZZPHCe4gGnXeRm23TlXomeuvrN5fFZ7tdV6MqtXCBfDsM/bCKDrHrr+qIXf/hNbn4/L09bex5XvOSX/XAWvHvrkNWEAFVUVc/7h2X10vYZNCg2hgRY8yemhi4t/M54dYa+16vTuUwieUazg538MHpwGiSc34Nf/g/oL4Af+PYxP/RuwyM8DnUII+tvbaL/2d0mbL2Hc+nW27Vfo3vwqYvju6cMt+HsjiMZkRoMorOi9wOb2Ihvg49dUleTk9hLdrdAdRTjOMR2NwoI0ynEDC/8ZRU6UUqRS4Z76IrO4RDn1z+/FIQOryYPVA+IipuN0iKRFlh/zWqs+/b7c3gEBcVmyUW1BNqGteuRpxdgSrK03QEp0Q+OAkpZmMcwifmTrTVpWi+HoiKYPq3JGR/Z5MD3G62vYmkugB0ySCXkF/sAhLsZc2/gMvumfc8MIIbgVpWjLDKcSNHRBlFdsBA69rcY5J7imK3w3xJwPuRYEGO03iVYjjk/GWHqMb/tYZo+jtMGk6WGvbfJaw6OqJIumT8+rLcVjU+Prw/uo1QkHq5SOyqmUIFFtBt4Gq/wGX/7m/8xRfsgqG3E3/OBCf7uGy2udq3xh/VO80XuNntUmy3TW+ztsN7YZeIOHxF9SMvvjd5BVSWdtA1HkaKaOFniooqAcnmZ5CwFlia1czKCD0DRUWZIsl+SOixu0MHSNB6Vknha85LsExkXXZnFygoxj1P4+sttFGAbGqUvBiJt14PQsDhTH6M0mmnGRC+nh/AwQoraQpZSYZoui8Hj3zgNKdLquRtexwVBAie/7rAUXg/+tZnAeNzIM44Wt02fJx1lXf5LXfDflew5CeQZlfCaEUtVBCwFkeY4UElnJ2hdW1cyRuimYLRb0uusk0sQGMqPEwL4AzZt/7StsXX0VtdYnTRJu3XmH12UXtf+HVG/9FIXTR//t/wz9wZMooHP54j9G/+I/fuLjs449Owfs8PsPf/nD/4Dozb+ND8TCI6802lZJlmXouv6RcMOz/noe3LDRaFBVdRBqehSh+yWao1NEOq6rUWkZKhWYjTqRZ3Q0Y22zcw7Xen8eolsOvz+Z07UsNi0DS8Dvn0xptTwG0iYKp6i1PWzp0AwGzMMlt9J9vr99FUevx6+oCj5cfMjAH7DurqP5A8JJim4J9pouX58tCUtJW18RVSP0zOL1oIunPEzNpK21mSdzqqyiMwjwXAfD1BBCkEY5RuwhzZi5mtD3+/hm3R+a4zIMQ3LTZhknXF9rchTFLKQkVjkN38DKMpbplJ7nsJjdJFttYiz2Ec3LFLOvYhoWCItluuQmGen4y/yRSgm1A3734Is8iB7Qtju0nQFJGZKVGfNs8kLrwNEdLjWv8IWNH6bf6PPG+vfzye51dEwyleLpTURVER4e0N7ewV0t8QydRTrF931O9u9hTCZogwG9vYewSKUUlVJovo88heYl4zFOp8MyjkmPjhFZCkXJHc1g3c+I4oTIddnqtOnIAv1wSt7rodk2+XCEpiQV4G5vY1sWRqdDFEUYcHHuSVnDkpWq4z15jpSSKIrO3Z6PrveiKDAsm/+PvfeOtiy76zs/J4d7br733RfqvUrdXZ2lbrVQQCAUQCSRpNGARwwMYDAGw2AQY88QbAzGyzYMWjNkrxEY0OARIkgkyUhIAskooO5Wtzp3Vb16+eZ0cpo/zrv3hXqpg8pVhX5r9eq67+57znfvfc7v7PPb39/3d7E5JGcaFAyD043y9D7o9/uUy+Xpbyb+QVEUgm3V1tFoRLlcnoZkDlLWNAyDIAgOpbq6rosoinied+D9FAQBipIpXU72rdI0RZKkTKlWVaeFQAaDAZVK5SoFSd/3URSFKIqm5z2KQmlZ1s2rQnncE/ndf/s4X/7X30xieBTf/hF6/+1TfMQo8G2vuw/ZL/HY5jK3qsv48mmiJKG6dDvx0Ecq7B2cQb/P8OknmV06gzwzw6jT5pmnH+L+D7x1p9H3fwJ+9dXTj098/0c5+6l3oZTPkl7+FN7mFXLjR55bx7/13XD73nh8c+hNU/SPW42cZMUyaTPqetgjB6tk4EcOmqbteQXcwxzwIkI/pqOCJoo81lulpirU9RxiGnLJ7jPsKeS8AfNnK4xjkZoiUzWq9GPouC4lBLrDPhfmtvcnhuu0JImSXkEarHHJE5gpVumvfp7FC69g2bbx0xQ5arLpDgmFeS6U8kRpSuKl1EIwKgaCLB7a7zRNGY0/jyKXkaQKqprdhJ8fu1QUibIso4oC4mT1m6bbiaQCT3efJC9J9MKQ91z6EFe2PoWuzxHh0RpvcK58K27osja+wjODvSvuU9YckqgxDsb4sYshm9xeuYPF/Cm+bP7laAQ4sYgkGyiSjh/7XOo/RcVo8LLaLRQUC1OfJUkEun4HvXkZY/YC47BJuXAb7miIMxxgCTJiY4ZWGJOkKb12K1vBDvpQqmBUKogCCCkgCPSjCAkBUYCyIqMKAqM4xk8yYTRFEkhTGPg+dxR3lCIT1yXqdEjTFKVeJ3Fd4n4f0cyhNHYoxLvpf9NauPtsUkqv1+shyzKiKO5hcYliNp9hnHKpbXOuntvDWnk+1/pRdhLK4ot1rmuB+YanUB4nqC9sP5RMO8li6UK6R5YgBRTJouu2STlc+EmWZQRVQ9lWdRx7Q+JdhBUvyaH9ypciCJCceoDVt72LOBjRf30Wpine9wOE4xAaZiY4ZK9mRR30EvzMdlT/Rz6fVZ43SkfyXrP7UziRcNFJCg4MuiPCKECVDAQ1IkqzlZFl7YzH5Fxu5BHGAaNoRBTJPNMLmbNi3PUVNvyE6FyRhdwpykONnrDFzHwVSzZZXXkQmxHr+hzmqEdPMFG8EUuzZ3hktc/p8ix4DhdbT3BX/hY0waCy0eXJS49AfZHWxY9TjiW0nES1eAd6P0Cij7GVIFfKjG2fgSFlm4VBjB4G5A+IUQ7GT7Lm+pwrV/jwo3/Kq8/fSSdWWNArGAj4sYIu7cTVH918lP+29THcsM+HVj9B1x8w8Ae7jvjg9F9rzgp2OKZuNvjGW7+Nvt/nKxZfx/naK5DCmFtKBQxZwQ+6uOMmvm2TRCGSoDAz/zLciUjetn35qS/fg933m1x5+EEalSJG7VaiUR97eY1U94hcl2J9Brmcx15Zoa7rSOUy1cDPSkYuLJBaFm6cYEgizZGNaRrcYu6EJUdRTHM7W1WSBPQkyAqLAPP6XnaWaBiop07hOA6ariPqOnK5zGg0InLd6Yq81WrRaDSmq+LJalZRlGnYZDAYkM9nmacTdpbtR8RpSkFXuNgak0YBqaROC9ccZCe51k/SZnV19dgCHC/Wua4l5pPadefkT1rbUN4O26QAkc2GvcWivC0XrNdRZRe/5x76ezFNkJWdC90NXXQzo5ylD3wP0Sf/kDhVyUk9+l/+DobBkLpRn3Kp4zBBNWR6mzbl2RyYt+0c/F8N9p/uRHaSvh/XJk1Txu0AyUjxkxG6JTPUDOZ2lShzwuz1bxSM+GRnlTSNMRULTdYpqV3StkZVLSKOZQabCVfsZ7lraY4FPcctuSqPb13mpY17ubL8SeqqyoaUA0FHmb/AQ8mIdLDB8sYKHbPMTP12rrgpfmpRPVdjRruNirNF2ymipAG+3cTtfhAVFU2aRyhrjFuP4dhblI3baV26SJK/wN9s/Vfunr0TUTT47NanedcjvwGAHdl7+p973MIOx3uyK88XzyEgsDpew4t38iLqeo2XVO8mEQTO6jO87Z7/hTlrDlEQUUSFOInxYx9TOYBeGwQIkowgiKhKmeFoSOP0HSSxjyhpjHsd4v4ASTdQZ2dJw5A0jIg7beRajTSOUaQckqtTuOWVAMS2RuVCGV3TEXe9lhdK2XUdtdtos7PT0pHAVLf8VPFqZ5mXpb2xdeX423339dXv95FleZrxXK1WcRzn0NJ0YRjiuu6BKfldO6CcU9kYuNTyGgX9aOXN/VheSJuT2It1rmuJ+aR23Tn54wsbZCt5kYQkERgikERDul6PRSuL1fdcGU8UEMzyoUdpLV8iCXclJYgCS4/9EQCCUaL/Lb9G6U/fwcq3/xfU+hmKcUCQBCRpQl7NM3RGzNZrxFFyomIMJ9lsOWlRB9MwEcSd173J61/gRaxfbjFy+izNziOjI6oJdhqzHsFpZTu8EYz46/VlLBV0pcQrK1k4oCBL5MpVtqwmYjPFOqvSVYuc7ofE4QhbmqW/eYmKLRF5HXq5lxDWityV0xi0YmxpyBnJQlTuwBBTqknKeGuTmUIdLx7QcTa4vThLrzfEDIEc5K15Wo/qnD5bo29C6m7Ry4t8sPUwn/673+HB9ud2Ov+5vWOxZC5wm3UGAYlKfp5nNx+iqpfIVUoYmBRkDVGM+JvOw2iCxNedei1LAXztLW9BNAwKXoxq1RGTEf5AgHZMurmMI0tYMzNI+TzqwCdWEqTtt6B4bCOqCp1HHkE0dXTdJCnkM/2hKCJa3UCQJNQowhnbBMUi9pNPYBgmsqqizM2RuC5urw9xiJfP0VldAUDRNQq1q7OrJ9eFvMu57n/lP+ramcSDJzHm3fQ8QRAYDAZEUYSiKNv0wizkZZrm1CFNkusOu46TJGW552NpKsvLTWqlPF6YYCgSYZygKSKWJmNt1x1+MQqYnLTNi3nvXU+YT2rXnZM/rvPCtpOXyDZcNwQly/CbaGIIAqHvo5d1BKGM67qoXB3vS5IEpdebfu5rLucfeU/24ZXfTzh0SX/wU6jqAF3WCeKAxfwidaOOGMl4YRcAs5jxxdfGa9SN+jRdfb+dOoBlABBECY+tD2kU9BNRpnK5HK0rI8qzJrIq0dvMVrKyKjJouUT43HnvbQRhQBJAd5zSlzqUZYkVocaj3Ysoach8LkfHbVFSBOzQZG6bTTRwh/SCLoXSInpBQ01TGlZINPaY8QM6rkVIxJKRcKbWwHISxr0USRFZmtmrf10Gwlqd1pOXKJ+eoSg06Xf6vG/zozw8fpqPrn4UgIJawLnoEKXRnt9r2wlTC2aDc8VznNKrqILBLfIpbtF07r7wJogDUr3AcPlxii//UVKrwbjbQfba6KrISCjxv8pKpvbpbxA3++jFBRLbpnvlEZRTBQzFQi7qKI0GaZKgiyLxaES4sUHiOFmx9Y0N4sEAqVTCHw5Ji3nyC6eIogjv8mX0IEuCUebnETWNNEnQJs60WmPc6xLGMaYgMHJt5HyONE2449z5Y+Oz+6+LZrOZJfIpCrquI4ri1Bn3er3ppqaqquTzeRzHQZblachF1/Vp0o6iKKiqSmn7baFQKByJZ3Idp2lK3wlpj30sXcYJYs7VcoiiQMlU0JWdN4iDYtAnvdZfjDaH3Xv/vfC8WJhPatedkz/2Kbcdk1cJiEURSFFFk6HfgzzoloW3uUZs6OTzMkEQYA89aoWd6kmB66BoOkmpNL0AhZVd2um5GuJKC68worc14vZzjWmGJp6EH0T4o5gkTRiEfTbbHcoNi67XnSa+7LfV1dUDJ27khbzyXPVkfd9uYxZUnGGAqssIgkCvN2SUDClRxczlGI8d/EBEVQS6UkowCpg9tciG3SYvRgyjmHyS8JKFl+PFHuORy0pri7HWQw9zzClLPBOH5DY6FBqzEAfIjdspRD55QWKr2+WymacWhxRqFt44xMhf/XCLo5BO8zMUZmcQ1TFP9zb5xYd+k882P7unXcNocOHUBURB5PHu47xi9hW8fun1vLzxwE4ZxsnYuH1Su41QPT/9TgCKtz6QjXFe2JaFzla9BTJl0WFzi/FTT+MDtTtqaKUS+p0vpVhvYPd7uFGIlMSIopSdK59Hyuen14dUKKDMzRGFIZEsoSsqqmGiAuY995KmKVGrBdsMim63myXcFYuZkJkkUyhXsPs9VMMkVywdeV3sn/NcLkcQBLiuS6lUmq6sPc9jPB5P2R2CIFAsFqep86PRCFVVp6yRgzjiu23CUjnMJnhXey6WJnO6mkOV9y6i4sAD5eAN/v19Okm/X2ib5zLG10ubk2A+qV13FErDMI4s5D1hAynE2HGUMQZkAycYEkQhaZKQiBJxGPCM3acetfHdCsYuuuFoYw2xP6C6uESn06FcLnP/n/1LAOJX/gCDrRHFqsnQ70wpVqZh0u8OIZQZj2zKepkr3SsMkyH5XJG19gZzpQZu4CIL8lX0KNd1DyzkPbYdKrnSVHvc31bFlGX5QMqXaZp0NgdUZ4s8/fAapXmVDlvMFxbw5CEFvcyo5dMvKDx++RK3LmiUNBGnF9AbdtFkjy+t34qKyrjr09lqUShXCUOHvFjmmc4Gwdka9Ric1sNUlAGRJJMaWWgqDlzkMGRRVRC3F96pFAHqVZSvjz/x51z02rz74k+z4W5Mr4GvOfM1/PBLfpiqWYWEnTHeRwtFEKYhBkmSMgqdZCIUT8P2tbObxuY4Wb7E7mOYpkkYxeiuh3Lfy9AMgyiKiDwXxcgcp2bliaKIUadDEATkLIt+p4MQh9i2TbUxS6/VwrQs7OEQ07LQdZ0oiqaU3zAM6QUB0tYWAI1GA8dxsCyLVquFZVlsbGxMQyVh2se2bQaDAbls+H1rAAAgAElEQVRcpjE+DARU3aBgquQUAd/3CYIgC7GNRojbkrmTvk7CKoVCYU8R+SAIpteeZWVFpHevpo8r5L2bbmh7AaIk0Rq4LNULrHeGBNqYNPIpV6pZnWR5L91Q1/XpMeBguuFuquJhdENZlg/0Ebvphifp02Qcj6JQHoRnP4VSUZTpG9BhFMqJHUWhnOA6ikIZRdGLRqG85k4+TdP3A+9/4IEH/vH+uJMsyzuFFtjZoJg89UzT3CN1rWjadvEQmZn8GTzRoy6rbBg5Ur+DraQs6ipPjuHM9spHiCMQBJQ4IQwjVMtC2hWjlL7qZwmvjLFKMpqi0R9kr7etlRGCIKBoAoqkUV/Ks+U46Ois9deY15eYL9XZtDf3rOYnfVQUhdCL0XWDra0tZmay2OuGnXBaEKYPN03TaLVaLCwsTC/gyTEkSWJ5dY1CrpAVwMitcip/L/66T+WMxUZ3RCiFaDOgSAovOV3BUGJmtRmurC9DP6Jen8Xf6jEKQRI08vkUSbKZX5xHkUSaqUZFlTFHTyL6MSgxFBZRFIXuyjKSLFPb1r93o6ym7v55MgydB5sP8gN//xPTcViwFri3di9vu/A27m/cPxUFg71sg8kxdo61l5eM6xF3O8izs4jbczqh8Snb1XRyuSwZSt7cxA9DlGIxqwtrWdPra//NIssyuq6TJDHdrS2sfAFRNrO0/zih2pjFdWzqi0sE2/xvURTpdrtIkoSiKFetvCa46vX6tC+Zw8peQpaHCV48ZhzCTH0Gp+tQ0RIcz8X2FTwvYLFq7fnt5JroBwJOEuBHKWUxRUiSA/u0ewwnv919P+3+XtM0HMchQKZoZIuMtutRzcnMVfJ0nQBBUjhXt2CbubZ/niZO6qCiGJPxmFzr++d6gkfXM4ZQEAQvSp8kSdoj2Lf7fpr85iA8u/s0wbO/X7v7BDtyBIf1SZKkq8Znf58O8nu7v3+udt2Fa459Uu3j9acCqAUFJS0QyDHGCKI0pWwYhGEAqoi+/bRWFIVhq4lmmviSQM73CJOY8FdfvUPDFEX0nIxnh1RmK7Rzbex+QBKnWCWNcc+nNJuVoqvoFdbGa8wV5qjKZXw3IvYgMiLkfTKjp06dwrdjuoMBpmlwcesirZ5CobBXbmEwGFCr1RgMBtM46W4bjz2kYsrj648wavqsV9dpFBu0220q+QqJktB22mw5TeYNDccOuLLRpet1uefCl9BrbjEOZUzZxLc7zJw5D/6I5Uf/iqQI1carKTIkDUNm734jsm4RBG2aK4+hW5U9omuTuWq7bbzI49Obn+ZDVz40jbUDfNdd38Vbb3sri4XDCxMfNOeJ7+9hmIjtNunsLHG/j7q0RNhsEicJymyGJ2q3aZi5LHbe74Moop49i6iqpElCGkWHnmu3iaJEpTG7Z3PScRz8IMCPE0TPI0kSkiTBcRzq9fqhm2SapmVFu8c+OU3G9iM6dkBOlQnimDvmS4jzBaJUYL3vcX4mY8gUyVaCmDkGgUCcxLijMdtkWzRZpGAoGIpElCSEUcrYT5ACj0ZBw9smFGiyiBdltYEFBBRFJU5SNoceiigQJikFXcZQJOwgJk1TNvoh9YLE1tAnJaVoKJS2s3kNVaJy+9kjx+8kY3yt25wk7HEjYj6pXXdOfvK6d5gJu3QIRDMr5K0XdJIwq4dJmiJKEoIgEoY+CdIe6cpsNa4jlKsEvSb5P/wqRC/bgLXv+icYcYKsSCRxiiIpqIIKpBh5BWcYUJ41pxWAVEnlbDEruqDpMqOuRy4qsNbe5PTM3knq9/uIoYEvu2yurfOM7zGvVjHSHdW6IAimK0pBEA6M3dnCgIowR38w4O7T9zEajDhdaqBZEk2/yYK+wJbtEUaXUMVTDJtbuGWVl9/2OgDiUgVZN3Bcn3PnLuB4W0Ttx7FLGvNiHXPwOKEYE/YGeMmA2XkLVa2haiO0Qub4RsGI33v89/jdx36XQXA4XfQ3v+zXeOn8y46cz0m/d7cJLl8mDUPEXA55bg6ShDCKELtd0u3VujIzQxpFhGtrJEGAaOawdY363BzK3N5KXoIoTlPuj7u+DmoTRRGWZaGq6vTvnucde5zLzSFOLLBYNoniFFUWuWXG2pP40253qdVqnK3tnefpqnJyrvLB51IRQQVNjFFUja2hT7y9EJJFgThJiZMUQ5XoDGx0XWMmr6PKImGcECUp630PQ5WwNJn5vEzROtwJ9fv9QymUE3s+Y/yFbHOzYj6pncjJC4Lw1cA7AQn4T2ma/rt93/9z4HuACGgB35Wm6fLzAXRUubnMdjl5VQY5xSzkIIDRQ39G9e9+idJ3fhA9Msl7z9KJanu4NWmakpKSK5ZQ3v3yvUd+/U/hDEMEkYz7DpyrnCNw4iwhx1KuKvG2G7OqSzhBjJjI+LHP2niNeWkRxZC49Ow6t569hc1wjaJUw9F0UmWAFOVobXazmpyCj5ZmN9gkFjhJNLEHPoomEROzudUhF97C4kKDnqcR4WEpJXqjAM1ts+J1uL92AboOhcZ5So3t1W6SEusGpw2NsSRyZdyl47TQ45iF+ZdT0QsMNlZQxuskxXvotduUCkXiKERWinxi6zP8x8/8AivjHfGsb7nlm8jJCiW1QJw4vGbpa7ldvoCQgFw19qp4HjHn4VYTeaaO7/pIqop65ky2mbm5SdTpot92656yjACCLKMsZEwZMZfDvnyZ+gnOtd+eaY4xVQkniCgamW57sr3hYCjZXoAfpYiyQhgnKJI4Pc7uEMrmICvq7EcJYZwwk9c5palXbUzutvF4fOzNfPw9kbWRRIHZ4uHOo2zIezJUpW3p7QmtEUCVjnY+Lybea9XmZsV8UjvWyQuCIAG/DHwlsAp8WhCE96Vp+tiuZg8CD6Rp6giC8P3Avwf+x+cD6LBU6R3bcfJpmiDAlDNuPPSbSMPLGL1nSHLnKJgaoqIRj3ZWy2axiNtaw9xOppnaT7QQxun+aFC2GZxX0ZJ0qmp3GGbNVPDsiPnSDCutdexhRK/Yx3BN/Nhja9SkVqxRnp/lwoZLo7LA41vr3C6WWXcu01kfc9ed53Evw8xSJu3b7XaRE51N22O+kKPXHXFrZRGnnDKMYipGmbbbZt1u4fopsegyt/EUldnbsQ3IlSvbY5Vy0fVRnBaMbNQoolwsUlSqKJUKpp5lhRbnFumui5Tn5ul6Lltuk48vf5Sf//wv7unzaxe+lJ9+2Y9QMMskiY9hLJIkAel2bpK0zbYJtx2wsG9O1/vuNIRRUEH1PdqXVlhfa1O941bmkhRJFFC2V+ZHVa8Xc0czFfbPlSiKtMcBW0OPgq5QMhVqlkYUJ8RpSmcYEEQJQzciCr3MyTtjGgWdoRdm3O8wxE8y+YA0hZwmk6Qpp8omQZSgymJWfOQIB/9cMB99T1zbNsfZjYb3WuN5sTCf1E6ykv8S4Jk0TS8CCILw+8A3AlMnn6bpX+9q/3fA258voGOLhuzywmmU7VALCAibjyB3nsja+CMES0DXNVAt1KhLxw1QRAFRksk9+z6kR393epzl7/gQp2WV0HeQVWmPMuV+2dLjMBfr2eaIkqoohs/yxXXmzlehkFKuWzwyGBIrMFvQSJ2I+XKNtjLGW/GZm5thbb3JbQu38vTGiNsWCnQSlUGrQzeNCfQ+mAVGBYFbVJXHxh63WxrLgYHXepAHNJ2WUyEwlhg98/cItbMwWIHSEle8AL1/mSVVAKOEF3ZQh6voUgkq5/b0x2rUefMfv5nl4dUvY3/7rX87Vej0xx6atrPyS12BxI9QajsbROHTTyPNzyMaBtI2ZXVz6BEn2SDHV5bZchyss2cY6Qov+bIl0jRla+gjSwI1SzuxwuBkg/MoEwSBzaGHJkvc2rD2FEmXJREZqOd1ut0uRV1HUGTy+fL0txONoSDINlsPwjVx7C8m5uulzY2GF25ezCe1kzj5BWBl1+dV4BVHtP9u4C9eCKijzEscvmZJ4T9uGVRWusiRCwLkPvhTCOJ2opS9RbJdWEWSChgsY7s27c0e8x//EaTmozsH/B9+i01iFqKI3qbN6XtqjDreAWd+bjbXqCOvaFTOF1m2L3HL/GlmCjVqA58rts/rZ8v0NmxqtRxRLLCxZLCAjjansjzaQMvXeexyH0kSyMsG7fgJnthKUYM8oiQR5SwaScqzjo8Tp+iuiHrmHsSLF5mbO4deKNJeeZSmJFGOV3AEg9sNDYqnSEUZRjai1QC5DKJEmIQookLTafLV7/3qaem3V8+/mi17i3e+/p2cEuYQRZnEi0jjlLjnE4Ug5hTivg8CKPWdlO00ipDnF1DmZonHNsHqGkGvh2+UqJ5qYLhjrLMLxKpKIsqY8bbCqJCFHbwwZq3v0rMD8qqAqkbIokjJVHDDmIK+97X3pGXVojhlrng1rz8MM8rkhIuuaZlq6WHFIV6ItO1zxXy92I2GF76I+UXdeBUE4e3AA8BrD/n+e4HvhWz3+PLlywCUy2VUVWVra2tamm5mZobl5eXJ7zh9+jQbGxtc6j4FwE/Xq/xEawOjf4mtrVlui2IcNbvpRqtPsGq9AqsU0emuoYzGND7wfVjjMVKyo3XifO8ncbU6wbNP80jncUpmHdd16NktejZYloVpmqytrWWDJctZIsjqKtE2W+PUqVO02218P6soVa/XSZKEJ558ClVV0VyFcrnM8tPLjCoj2mHC+dkSa2urxHHMaBWWlpbo+k2eGXapaAqxZfLw5qc5E87z1DCgoHRRtYSym9IbdWkEZdZsgfp4TC1Naa0+xZlyjk8++jgj18VMJDZTEcGYJxz2sd0R88kz+GdfxebKGnE8ysSkSmO+9y//ERveDod9t733Ve9lZmaGfr9Psu7S7T6G1qjRHTggCEjpiIWFl3LliUsggpAKnDJm2Or3CaKI5NIl6vfdR7fbZTgcZjkOlRlKukzr4c/CaIR1770UFIXNtWwd0ZckFhcXWV9fn4ZoLiws0Ol02FgfYyoiq6JJo6DzxDMtZFFAMwxuOzXDU089Ra5QRpIlbj17mtXVVVojj7IhM5aLEDgMh0M0RaaizZMkCZ1tbny5XM4S52x7ysuen5+n3+/T6WTywUtLS7TbbRzHIYoi5ufnp4lPkPHVLctifT3bs5C2+7K8vDzN7zh9+jTNZhPXzXSVwjAkSRJ629nXpVIJXdfZ3NwEMupdpVKZ3isAZ86cYXNzE8/LFiOzs7OMRqOMkbPvfoKMeje5nyb5F5P7aXLdzs/PMx6Ps/CgLFOpVJBlmea2Jr1pmtRqNZ566imq1eq0b7vnaWFhgeFwOFWfrFariKJIq5VVsrIsi1KpNL1/dF0/8H7q9/uMx2OiKGJubm46T5BVKCsUCtN7ctKXlZWV6f7P7nkCprkGh82TpmlUq9Uj56nRaGDbNuPx+NB5mp2d5cqVK9NQzEHz5Hke7XYbWZaPnKdOp0OtVjtwnp6rHSs1LAjCq4B/labpm7Y//0uANE1/fl+7NwL/F/DaNE2bx534MKnh4+JV7/zAe/hPmz8DwM+f/XfMji4h3n43F37rO7HVITNxjHPfd9J+w8/z7OZDzPZCGn/z3YwSgVxgUwt2nHz8kxnHeXljDTVUKRQK5EoaTpxgSiLtIKIsCQfiaQUhdVU5FPOTVzaxXZ/7L2TFMx599FEWFxe5MrKZKVRoFK7e4Fp3PMqaymO9S1hChCiKfL4Jt5V1FNljJl7A0m26oxaCqFDTdYg8uisXsc6/jMgoseb56JLEgqERpulOan0SZyv2sI/trvKTn/51PrL6kQPH+Ntu/zZ+6L4fovX4syzOz2eZnJKMoBeRqzni4TDTCpdlJEFALJaQrIybHq6tZQUkNC1jyGw7hIk9vTXi1sZeMa0XEsecyDRvDj3WV1c5tbiIKok4YYzjRxQNhShJMVWJgi7TdwIsNYuXT1Q5e70egiBk9Ull+ZrGXo9SJ32xz/VitLnR8MLNh/kLITX8aeBWQRDOAmvAtwL/aN9J7wN+Hfjqkzj4oywMw6MHYNczyY9CpEIVqbe6p4lub9LzAxBz1D/7k8j+EJS96dzh295HvOtcu1+9n7Bd7i/kWPEyh3AQnhUvmDr5MAzZGgXMFnbEn6IoomCojLyQvK4wMzPDYDQiUQq0Rv6BTr4iwlYQkqAxa9Xo+30c9yL5xZczcNcp1k28VhOndI7Z4TJprgaDVdSFe1ALNVTgVkXG8zxEQUDbHU4QJdI04aHNj/NdH/4X0z+/YekN/NLrfmlneMOQuN9HcCJygoCYz6OaJmkkkqYikqUhGgbxYEBimii6TtTrEW6NidrtLP1/NgvPCNsZgpIk4YUxqz2XwgGF0Y+d811tgiAgjuNpir4pRjSbAyzTJCenjGyHhqVQNzVGqY8YRyS+jyAYtMfZaqg1CGk0GgwGA0Qxm99CoXDVuU6C54W2mUjxXotzvRhtbjS8cPNiPqkd6+TTNI0EQfhB4ANkFMr/J03TzwuC8DPAZ9I0fR/wH8hS4N6z7SyvpGn6Dc8HkKoeLPA1MSXZKbSdkhKpOv3Vv+L8rjbCcINoGHEmN0e5sEDUCZHSmPHL/xm1V3wHtmijOsXpuQQE8iWLQWdErqRRVWRaQUhFkRghsNsdD8KIQRgzoyps+SENLRN4EnyfD6/3uVsOqNfreInEXUtzrPVd8rpCN1JRdY3QjTldPZgNoqoqS4LAJ1tj5qwZNEXnzrICcY9UzBGsP4yg5NEkiUCt4nR6iGKeXKly6Bg+1HyIdz36Lj688uG94ygq/P7X/z63lW8jarWmPPKo2QRJRq5VyZkm4jafNx76SFbmoAVJQq5UELfjhnK5TBoEyJUKwjY9TLKyPgrbbYZuSM1Sp4k1h+Hdb5NVj+M4hGHIcDikWq1SKBSmmaeTENmZ00tTiYEwDNFVBUmSKJd31EgnmaqQ1fqM4/iq+Odx1+CL2Wb3w+ULfa4Xo82NhhduXswntRPxu9I0/fM0TW9L0/R8mqY/t/23n9p28KRp+sY0TRtpmr50+7/n5eCBaezpMLtt9Q/2fE7SBPOJTD1STFMiQNh8GGHtIeoSSE++GylOKSV9mo3X4Cpl4sTdc65+MCCX1zFLCl6cqQeGSRbqGHl7N2G9JOUpxyOFadLJx1c7jL2InCLRQubp1SbV7ZqyM3mND6/32NzcpOklLBT0Ax3dBE+apuTVHGVZYm3kc/d8g7ncHOeEmKZosj4aIqUJgetSmpkl9P2r+OOTfr3v2ffx7X/x7Vc5+H/96n/N377lbzkblghWV4lHo0y9c3MTudFAO3cWqVCgtZ3JGXVc0jBB2Fe9Z/dcCao6dfB7xsvzeGJzSM8J9/R7OBweeJzdZts2zWaTwWCA7/uYpkmj0UDX9anoVqFQmGq6bG5uomkalmWRy+XI5XJXJZ3sP9dEkuCoNgfZi9VmElu+Fud6MdrcaHjh5sV8UrvuMl6P02eQkh052rd+7DvplW/j6cl3X/97rD7yf3Pm0seZ/9j3oi1/GQBynLL2rX+E4lh4Xh9Jynjjk3Pl5IwRIghwxfURRYGcJJKkKXnTYBzFtMOIM4ZGN4w4Z2oYgkBRkWn6IbKqkkgCZ/Mmjz51mdlyia4bcwZQJJEZVWFFFlBTmCvu9K/n9ShppWmoyDAMPtlZ5f5SlZwssSgrSHJWz1QWFXKzS3x+MMZcX0E2DARRpLZ4+qoxSuSEe377nunnd9z3j3nN4ldytnT79FxRu008GiHqOuq5jEIpba8e0iSF7f/CloOoSQj61ZfKZPw6Y5+qpRFvc9sndqltE0QJcwWDornjSFutFrquMxgMUFX10DmP45hGo4EoiuTz+T1SA8/XTqL/cS3bnMRuNMw3Gt4X81zXEvNJ7YXfNS+yTRgCh5mwb5/Y6j01DdMn0jzdC181/U55+o+n/45TkVxhhji2EV0Fr3uJte42a2ZbZ8ayLETfQxdFZlWFKE0xw4ANP8Tf5nU/PHKIUwjSFEUU6I1GBMMhuVzmxC7MVOilEned2gkRKIbMuXKOamXv5AZxwIadMVtWRitsDobYoU1NM+iu2yhBsrNX4Gcr3/zGFXKlMvnqwTzaOIl55btfOf387jf9Om+/+59yWt2rWz5aX0eeWUSe27tbH/V9orZLPAxQbBBNBamgIapXxxBt26Y59HDDmNWew5ObI9b7Lrs38+dywh4HnyQJuq6Tz+cpFouIosjGxsZVyU4TRsLEsU9YDkfZSV6Dj7u+rnWbGw3zjYYXbl7MJ7Vr7uQFQXizIAi/0e/3p5XLgyAgiqKpXOdElne3dCxkQlEHMZNFILLmCfyA3szdpMre0mKrD/w4UdIgX6vjui5CKhDXa3TjTPO7t+5M47thGDIjZs5oRkgZiDJXRmOqssTaYMgdOQMjDCjIEpvDEQ1Fomn7rLk+w1YTSxVZMGT6vs+m49FxHPKiwFKtiiawp0+CIKCnOi2nxebY50mvj5EqpHGMqKSkJNm4uDaxoOB5HuX6DKJu0A67vPfx9zIOxvzK3/8K3/wn38w9v30PL/2dlwJwZ+UOPvrm3+Fc/jxB38fvOXjrHexmC+/SJeRcDtKU0UaPxI8YXumQhjHuyEYsKEQmzJxfIJKTA+cpSbIHUJKmVDRYKBnM5QRmCzrPbPRoDj1UIqycie/7RFFEEARsbW1NpVnjOBPFmp2dZTgcMhgMGI1GDAYD+v0+hUIBd7u+6ERqeCL1MMETxzHetmjYJPY+uV52Xzdpmk71ZiZ4DurTRBHzoGtvIhWdJMlUBneCZ3KMCZ790r67/z85hud5NBqNI/u0Wwb3sD5NpHOP6tNBePb3yff9PXIaB/VpIpp3VJ8mUsPXS59mZmaO7NNEQ/+oPk3kgI/q0247rE+TMT6qT47jMD8/f2ifnqsdS6H8QtlhFMrjBPU/9qtfzw+YGX/+kUtXCIGHdI3bH/gpmsJL2LpdYv4v/0/C9hPc6l6hJ4oM3/qnPBlEvPLu1zAcLGPZY/R6jZWRTb1Qp7M54vyZTCXxQ2tbvG6uPl1BXuoNGEgyVSWjC7bDiHvzWXhn3QuwAo8PN4fcU87h+D4XahVUVWXNCxCAME0xRJGN9XVOn1qgtKvOZtNpMmPOMPAHfLrXZUbSKesRi/lF7L5PrqTRXV+joMfIuQrDkYPn+byv9wHe+dl3Hjm+H/uW/0LBPEfwZBNBBPXsAt7nH0WqzCEoIYGZx5A0pIJGPA6y8IwgQApSIVtFrKyssLh4sHpkzw5YbffJmea29Oxe6449cqpEt9PGMAwsy8pkbIPgKk2O3XM+HA6nGtu7C1ycpNDCUXify3GuZZsbDfONhhduPsxfCArlNbXjy//ttckjSpJEkiiloBWIzn8Vdi97EPz9vd/FYuUWgrXPkROhr1o4rQ0McZZiLo8f+8j6zguNbpiMx+Pp7rZhmlQlkVEUk5NE1vydh2JFkVmzI/KqST+KmdFUXEFEBeY1hc8OHe4rmGz4IXYU4SUpQZKgilm8P06y5A1BsnhpJYfhxiiKxLC9E/JQdR13uIGRnyP0e/zhxvv45cd+bc8YnLJO8dKZl/KTr/zJTHxNyeH7W0iSDmmEevYUUXMdpVFDnqsT9320OAU5G03JOvjV8DBxMS+MWR+43L1UPzTrM7QHBKk5LS03Go0ADhRd2j3nh7EKTlIy7SRiaNdbmbcbDfONhhduXswntevOyTuOc0w18x2n0vUNLM2FFKKl18PlNoIg4J1/E4GQZ6s4SyAJdKOE02qBp4YtSkYFUTRR1Trj3mfJ5fdKAt+S00m97FU9TVM+1e7xDQszqILAKI5Z1Hccoi6JjOOEspByV71K6Dj044SikvHu782bNIMISRAYJimzmsIV12fJ0Phsv42VDpljjnEUM6+rtLtjXGcLWZonP6FZCgL5SpVBr8sfLv8Zv/x05uDf/03vZ1ZTSdMIEND1U1OH6zgOkgRhs0uaJIiairq4SJokCKKAXNEZd4dY+ee+AdR3Atb6Lufr1p5iH7ttUsFmUl8Ujub9Hj/nJ2tzEnuxzvVFzNcHln+oY/xc7Lpz8sftPO/eRAgSabqSTyUtq7iDQCoJiOffRGioiBufQSATn4q2W8tSESdysIcOtYrBIGrvOYemaXieh6xqVCfVYSSRTpg57D14XR+rWEYTRTTLwkhT1r2AeV1FEQWiNKUdRHzVreforI+JzS4P9pukgkQvSQnjhIuuz7yeaZUrRgln2EQUzxKFIUkUgQTf8Ynv49nBswD859e/i0VpHte+RK5yG1E0wPNW0bQGcWwjhQn+5RXErobxsgtTrMIudkqufHyyxdLS0p7PyXbBiTvnsmLPqXzwXNm2TbWaiQddS0bCfrxfyHP9Q8V8o+GFmxfzSe26Y9ccxyE9KFwzKSQy/c6QCEcekqUgahJxGlFayFgkuijiN5us9lZxYgfXcRmMHyZNU56yM0bHZHMuTFPOKztDVFNlOmHEbrvcHBDuYoZIgoCxHd5p+QF2FHNv3qDdbqObCskYUkKkeMTd1Vv5+FNP8FLLIPDG+H5AHHuUaudw3RXc4QCPIW/76386dfD/9lX/lnsrLyEehUhSjsQOUZQSatTA7W7gNrcYbl5CNWcx778NyTick3+ctds7D78kSXl0fUDZVKdvDIcdYzcz4Fpyi3fj/UKf6x8q5hsNL9y8mE9q191K/jhB/d3bxBOnrmrlTNRbFFBEBTuJ0CQDAYFGZREl8JCUIgQxJVmi4weoqUqsmGwNnkFV6gRBE00sYW0n/IiiyEODMXdZWTLNYDBA1/Xp93EcMxwOOb8ww6nq3lVxWZG54vr0vTY5EURhgUFvjFqyUA2XM/EC5dlsU+XO2jxOewVfSAmDGEMyWA6Xebj1MN32M/zik781Pe5fveEvqM3NAQGClAmZC6JIsNGDREGJJFJfJKdUkQwF0Ty8ws9JChdMQi1+FHOxZXNbI4+u7FApJ8cYjUbk8/mpSNbulcq1LLQwwXstzvUPFbSWQscAABo3SURBVPONhhduXswnteuOQhkEwZEUSnHfWj4F1Nu/jzAISJIUHZ2BN2AxDum5XcIgwjQVBuMBZ3JlPvf5B1HmF/B8j4paZ2B3qGo1unYHKYlR04QgCDBNE9EeIUQRV65cQRAEer0ep41MfjaKIgaDAeMgYTweT+lRkz7NiLDuRlQkk+aoyeXWFQJrTBja5PLzdJsDBi0XTZLIF8p025tYlRqf2HyYN//xm/mJj//E1MHnZYv3fcX/S6FaylL7R5vEDPB9EDUZe2uNNBrhJyFiGBJWK6TbOi+HUb4mG6EH0dge3xjw5FoXJ4hYaQ+51BzRsGTENMZ13SmVazwe4zjOlPY4Ho+pVqsoijKlfAVBsIdCeRCNLQzDq+Z6P43Ntu1jKZQTrv1R1LzdeA6jsR2EZz+NbXINHEXNO2yMd1PzJriPouZFUXQs3XCijHkUNW+SZXwU3XAyRof16bAx3t2nyX18vfRp93VzGIXyIDz7KZQTaY2jKJSH4dlNoZxc80dRKI/q03O1645CGUXRoYWRAT71q2/mu83LAPzVY10KxpjNr/wdKrN30u3YGGcKPJO4PKCe5on4Ev3YoKEEREnEPfV7ePiZRyklOklRIoxC3GSDhrBAbEWEUo3TVpkg6KBpMzzV6nC2VKDT6aBpGqZpkiQJhmFg2zaqqtK2I+qWchXmJE34r8sf5k1n3sh6Z4vLy2vcddd5NgdPcGHmFbijAHccUqrqJMOAVBboJQPe8P437jnOt5/6Jr7/7m9HlDTEnIwklXDcJygVX5aN17ZErWiaJI6DXC4fO4ZHjbPtR3hhTNXSWGn1ScSsalJuu0Rcq9VCVVVEUcS2bWRZPrJM2QvB8lzbnGRD61riuRkx32h44ebD/FwplNddTP64h87+b1uSRBOH1HOonDkHrktezTNQxiiiwKXRKikpmrwduhAEFm45j6NquGOXexZehettISklnm0/ytjvE4SZ7rRlWTy6/iimadLpD9E0baqNPUnQkUThQMxBHPDy2fuztm2V8qxBUSuyYC0AYORVzIJK1HZJk5Q/WvuTqYN/w+xr+Mjr/oDfPP+/844v/XGM3BkEM8EwFgiCHoa+lMn9AuHqGnK5jKhpyNvJQCd5cB/UZmvocaltU90u5FzURE5Xc1MH7zgOsixTLBYxDINcLndsHcrni+X5tJlokl8veG5GzDcaXrh5MZ/Ubjgnv9t+r6bzdUsLfM8zP8dTG5/mJx7+P3B8H1mUieJskGRBwJAMTlkZVXLL2SBOY7rBgFw+h2+DIGpU9Aq9fo/2uIksWawPLzOIfGRBpuf32BoGfHb1EQzDYGtrizRNeXh1gCyJpGlKy2lNcflBGz/2UdKAJIkwLAUjyOLUSdyftpPTFNFU+Dv7M/ybz/wsABW9ws9c+CeUChZ3z5xB0PPIloppngVAVWfQtDpxr0e4vo48M/O8xvCgNq2Rz90LOwlIkyILEwuCYJqgJMsymnZ4zP+FYnk+bfbj/e+N52bEfKPhhZsX80ntutt4PU6ESti1ln/XzM7rzHevZZror819CXfMzCKIAlIicUpNsVSLcThGc0JqqsJToxYCAqZl4tkhleJd+I7PfOUcvr2JVFwi9S4RoFArVmgNR8zVq1zsX+TUbEoYhGwONpkt3kbJUEiSmI32BmpDpaDm6TqbIEiUtQrtjYvkrNOMehDGIb/+yHt47+Xv4XzpPJ9rfW5P397xsnfwP9/5dpxH/xLpzEuwzDL7TRRFwq0momUhFYskB2i6nETIa9JmUnT6meaIxfLVr4eO4yCK4rS6zu5jP5fzXIs2J7EvYn7hba4nLP9Qx/i52HW3kj8u0+u4Z2A/HLDmZxrhFb2CpZpEy1cACFZXEZOENE05nSvTDSJUQ6ZQNUiShIa1SFEzaDodECRkBLzIo7UZ0XY7nK7NcrG5Tn22zvLaJlESIIoCcRyjotIf9+kOHmQUjBkFPp7bwzQXENQWhUKBb/mTb+A/P/2H2KF9lYP/6Vf9NG89+xboLzNSZrM/qlc73ch1iTbWkatVBFlGOiDJ6CTZcnEc07UDNgYuyx2bkqnuERKDjAXgOA6DwWAq3ft8znOt2pxEg/uLmF9YmxsNL9y8mE9q152TPwm96CjrhwPcVMFPfQQETlmn6HfWSdIEuVbDsmOGfh9VgFjcoQMuhwlRGFGt3Irr+DzbUTCFCFlSOTffoN3tk89plIvzbNlb1EvnudzO6pJe2ryEqIqU5BLDSCJxUpyNmGFbpjcc8+8/8Wu89v2v5fIoq2D19lv+JwB+4w2/wSPf8QgfeMsHeMutb6G/toIfxqjFq0MwAPFwSLq+jn7XXc97DMM4e8it9H1GXkjsOxSlkPy+n4zHYyRJolKp0Gg0kCTpKmW86412Ninnd73guRkx32h44ebFfFK77iiUE4reYVQijolnbXlNTus5uoMuYRwS+iFJKU+01cWNPBI/ZsHM83h7Dd/38b1tNURJYjAaIQgFSkqJiC6b/ZDVzYgg8Pnql30JeqJzS6POM5tNlmZnOZXoNIdNbNemqlWJ4ssIboyk1qhZdd78ibfwtR95I+9d/ZMpvp+9+9/wv33pv+Djr/sIrzr1KhzHYS43h+s4KFaecb+PXiwfSPlye33ichk3DPeMy37K13A4PJBuuNGzWW4OWO05+OMBNVPGFOOptsykgHKapvR6PWzbnmI4iPI1GAyOpXy5rnsshdLzvGMplIf1aTeNbVLs+ihq3m48h9HYDsKzn8Y2HA6PpVD2+/0j++R5Hqurq8fSDX3fP5ZuOBqNjqUb7sdzEN3Qtu0j+3TYGO/u00SI7nrp08rKyrEUyoPw7KdQjkajYymUk4LsR1EoJ/fVURTK9fX1m5dCuVs+9CD75K98Hd+Tu3L4cUv38XOv/1VEYUTH61AbQXvc5LaZO9ny2ghRhDTb4Mlhi/NWndC1ODtjseWHlCQRTZZ4anPIwF5DK57C33qSvLXI7acb03N0hh6Doc9SXmN5a5nQSrl9/nZ8f4txJPOZy5/hxz7zY3tw/fjpH+Ob7/hGQjWLa+9WWAQIPY84jtDjERTmDuyb//TTqLfccuT4HDaGQy/EDWLiJGW+ZDAcDhEEAdd1mdnevN3N7RZFkXa7fWQB5OPm6lq3OUnB5i9ifmFtbjS8cPNhvuEplCcpDnGYvWL2FXym/yDxYIj91NPEaYwkq2jjLIEjThO0+QUKagEntIl8j0EvSzFOAU2WWOu7BHFKQa1QNRXmSgXqVsQHL38QP87aFryEagxhElEuh3zs4ke457fv4YHffyNf8QdfscfB/8Wb/5yHv+0hXjXzMiI1RVEUTNNkMBjQXV8l3qZK9ZubdPsDgOnKYr+JheKJxucqbWs/wvYjVElkvmRM2+Tz+amDh0xUrVAoYJompmkey555Pli+kG1Owvb5IuYX1uZGwws3L+aT2nXHrjlene3gN4/7jfN8cvOTAPzN5sd5jTCTsUEEkTO3vow0TkiTCEmQMBWTB2q38Nhwk1uqM9i7NkKGbkg5p9L3Bcb2mK3WGr9w6Re4OFx+Tv34jtPfyY++9p9Pn9izs3NEgY8peYiKhWmatPo95H6XWJCIoghTjun3HMRyjiTJinXkcjl838+KZgt7x8d13QPFjiYXyKW2jaXJ2H6EpoiUc1lMfTgcHlktfpKoMTd38BvFxE6ipHct2xyH91rjuRkx32h44ebFfFK77lbyJymNtd++/szX8Qvnf5A3nXkTAP/f8nuyeFcSZfVRazXkmTq1sUhezdgodbNG4ieUZIl+GNMKsjh3JaeyUDLoKwIpPj/w8A+dyMG/sn4/33D26/jhu/4ZH3vj3/Cjr/mRPa9kl599llJjlvHqMySjLfzxiFwuRywpePYIUdUIPYfC4l0EQYBlWUiSNC1i3XvsMfwgmI7PJA271+tN43STmN3q/9/euce2dd13/PO7fIiURJGmXn7HluMsdd3EdmPHwYKgbZ7OkKYpsiwbluVRIEM7D92wIkuXYUtStEMHtN2GFWudLnHrpk1ab0WSFZ2brgY2DGseTRPHduJYfsq2ZMmSKIrvS96zP+4hTb1IKqYokrlfgODluT/e+zk/HR3ee87v/s6ZMxwZOI9bwK0yYCZZ2mHn4InFYpimWVEmvFOnSte73pZMK8dba55mZG40Xmhe5kpVd1fy5RLqp90zQwYf2fIFQsDfebay7+Q+jsUOs7J7DQYZXGJfsYoInta2wnquAFuWbyAxkcEQN2GPm4GxBF63/bsXjCf5i1cfLth+bNl1PPyhz7K0bQV7j/6YniVLWelvZUVgLS3eTlrVJO7xMK5WH2lDEPfU30+X1wtjxwms6GPs2Ak8AT/BdVdjRQawPAZLjCzm0BAsX194itTn85E9fpxoLkfPmjWYra2YetJ1dHSUQCBQGPrxeDyFSZyOYIh4AjrcWVLJFJct7SEajeJ2u8lkMoU0wOVUbr6m3hZaqGR+yWG+NJtG44XmZa5UdXclXy772rnwthllHrcXae/Ek7n43ZFUllXShb8o53nL5ZdP/Z6ZIZrL0etxM5LJMhbPkNMLdu/839/jdMye4N1/z36+sepB+jxtdBkpHrz6Lu7ovJJNoStZGbqSNtpoa+ujZWUId9i+YiY9CTH7KdjxoXOIlQFXCxZtdCzrIzMWI33g/xDDRUhlME8cJvtb27EmJ8lNTJDIZMlFo3g7Olj+kY/g7uzE7/cjIkQikUIuHaCwILZlWYRCIcaSOfq62wkGg4TDYUSEYDCIx+MphG9VI8tdJceopU0lcpgv3aaeWD6oPp6P6q6Tr2QYoVh/yjUIwrGJ44DitrWfAuCv332SXwz8T8lfRJ/PRy6dI4fgEugJtCACr42PMpq28zk/tPouunydGN2rcI9Pko0NINkMiAsrdhZ0wrK8kjE7GybpGBguSNiPJ/cFs2CZmEPDWLFJurb+Nt71GzEHTmApP571m2htb8fd3Y2Zszh59Cwjo5Mz0ha0t7cTCoW4kHFzZjxBNmcV6pJfZNkyPLToO4nimFyv11tYoakSP1922WUl99fbQgvleGvN04zMjcYLzctcqeouTj6RSJROuVl0rM+NTLJZwpgZk5RpcSYW4dPr7QeN3oi/zSOHHuej/3nTlGMUx8DGJuOEPXA6kaRLFOeSKQwjykMvfgyA7T3b+OMP3UdquB/D6yLt7cbq7CMxfAyvO4gV2ISKnLLHzVMm8ViSTMrEqy5gRs6S9XYweKIfTzbGiOkn5+4k42uhZd064vE44g+QXbuZXDrNqKuDRCLB2dEoAxmDZctCuFJxTk+kSSZThZjcZDLJ4OgEXpfBEq9iJJZmYHi8UCfLslCZRNkUtnPFcBfH9Q4NDZWMKY9EImXj5PORQqViyvN2c/2d8jH55eLkBwYGytapmGeuWOXZeKbHKk9MTJSNk58eMz1b/PXQ0FDZmPJUKlU2pjwft18q/nquGO7imPJYLFayTnP5uLhOxbH29VCnwcHBsnHys/FMj5OPRqNl4+TzOWdKxcnn58VKxckPDw83b5y8ZVklczv8cO/X+Ep8NwA7R6Jc33MPffc8wXlzkrGxQVqCS7n3hZunfOelO37KmvDM5bTSKROU8NTR54jExth3+idE0hdXZNlzw9e5yr8cI7gC4hegyx7uicfeoy0ah+WbIZvGig2TpIdc1kIyEQIBAbcP/CGiF0Zob7EYOD/OMl8A95IQxixjcmfGE+RyFi6XQWdbCz6PYS+xpxRnI0kCPg9BvwfLshiJZejVE6kAyUyOsUSGcKuXSDJDm9dFxxwrQlXqZygfX1zJMWppU0k8tMN8aTaNxgvNx9zwcfL5icVKZChwmX7EXiSpkLzs6VufnmJ3x0u/Q8KcOg6WSWbJpDN86VeP8+3ffJXnjz41pYP/842fZ1PnRswll4M/BJ3riF6wf2nb2q+wO3gAdwsjowbEhujo8qMmR2ByCPwh0vpq0Aj0khu8gAizdvBgJwpbGvDgNgz8XlchMkdEWKkTh50ZT5BOZ2Z81+91sSLkJ2nm7HBJo/wP93z8fCnHqKVNJXKYL92mnlg+qD6ej+quky8Vv21ragdmtq0AkSlrvV7Tew0f776JL1/3lYLdtT+4ls17NturyVxIMj6UQDB44fQLU463tec6dn70r7h//X1TeUQwXAaTo6kZRL5giOiRs6jYMB3LuqHrCpstnaI9HAage/VqxOeb8d28Wr1uXC4XS4Oz2wT9HlaE/AzHTCaSszeScJuXy3sCFfiwEj9Db29vyf3VOk+teGvN04zMjcYLzctcqSrq5EXkNhE5IiL9IvLoLPtvEJE3RCQrIndfClD5MaeLnXwquJq0txPR3bvlbcdIxxARntj6JJ+84g6+tOaRgn3WyrL12a0kJ03aQi0cHngPgJA3xN4bf8aeHXvYtfUfeWj1pyEzCaP99phczCQVNxGB1g4v8YmLT6NaOQtLWXRtXId57jzKyoK3ldH+o+SSCQzDRebUKTKJOO45QheVUihU2bqLCMuCLazrLp28qJJxu0ps8ku9LfR5asVba55mZG40Xmhe5kpVtpMXERfwTWAHsAH4fRHZMM3sNPAA8INLBSqX06FY0bU7ALCU3fUrbxuGaU9WtIfsK+Kbr76HHStvL3wnnUvz7LlnSCdMPvPrPwTgye3/QEgF2NSzCfEaSFYhmND7YUSEXNYi+e4RzFgCt9eF22OQitlX05NjaXIjo4DgWb8Rc8LO8mhYFrnIBFYigfj9TJaYUU+aOVo97orqLiK4jNJ2lR6nnPKTWgt9nlrx1pqnGZkbjReal7lSVXIlvw3oV0odV0plgOeAO4sNlFInlVIHgPlP/U7TfDr5nMpxmjgTKTv/i9flIq1TFOSPY7iFx676W35668/ZddMuAJ7p/w4Hshcnfbctv4qoQE7nuHGHfRheSOXa7eEZgfaVPbjiEZRl0dLqQaFIJ0zcHoNWn0J0J+1ZvZqJdw6jzAzhDR/GisdRZXJVxNJZAr7KO/la2dQTSzV4q3kuh7k+WD6oPp6PKnnidQUwUPT5DHDt+zmZiDwM5B8jjYnIkVnMgsBEmUN1ARcOclB/fGA2m5LH+QQfL9i0P1AyEqUSnoqZq3CcWtqUY2403lrzNCNzo/FC8zHPL4heKVXyBdwNfKfo833AP89huxu4u9wxy5xvVwU2r1fpOLW0aTrmRuN1mC/dptF4m5m50lclwzVngVVFn1fqsoXSSzU8Ti1tKlGjMTcabzXP5TAv/HkcH1dBZR+GEhE38B5wI3bn/hrwB0qpQ7PY7gb+Qym1t/qoU87zuprHwwD1IId54dVovNB4zI3GCw5z2St5pVQW2AnsA94BfqSUOiQiT4rIJzXQVhE5A/wu8G0RmfEDUGXtWuDjL4Qc5oVXo/FC4zE3Gi98wJkXLa2BI0eOHDlaeNXdE6+OHDly5Kh6cjp5R44cOWpiNVwnXy7FwmJJRE6KyNsi8qaIvK7LwiLysogc1e9LdLmIyD/pOhwQkS01YnxaRIZF5GBR2bwZReR+bX9URO5fBObHReSs9vWbInJ70b4vauYjInJrUXlN2o2IrBKR/SJyWEQOicjndXnd+rkEc136WUR8IvKqiLyleZ/Q5WtF5BV97udFxKvLW/Tnfr1/Tbl61JB5t4icKPLxJl1evXZRrVjMWrwAF3AM6AO8wFvAhsXm0mwnga5pZX8PPKq3HwW+qrdvB34GCLAdeKVGjDcAW4CD75cRCAPH9fsSvb2kxsyPA1+YxXaDbhMtwFrdVly1bDfAMmCL3g5gR6ZtqGc/l2CuSz9rX7XrbQ/wivbdj4B7dfm3gM/q7c8B39Lb9wLPl6rHAvl4LubdzPJsUTXbRaNdyZdNsVBnuhP4rt7+LvCpovLvKVu/AkIiUr3l2eeQUuq/gbFLZLwVeFkpNaaUGgdeBm6rMfNcuhN4TimVVkqdAPqx20zN2o1SalAp9YbensSOSFtBHfu5BPNcWlQ/a1/F9EePfingE0A+fHu6j/O+3wvcKCJSoh5VVwnmuVS1dtFonfxsKRZKNcZaSgE/F5Ffi52+AaBXKTWot4eAfP7QeqrHfBnrhX2nvo19Oj/0QZ0x62GBzdhXbQ3h52nMUKd+FhGXiLwJDGN3dMeAiLJDvqefu8Cl908AnbXknY1ZKZX38Ze1j78hIi3TmaexzZu50Tr5etb1Sqkt2Nk6/0REbijeqex7rbqOV20ERq1/AdYBm4BB4GuLizNTItIO/BvwZ0qpaPG+evXzLMx162elVE4ptQn7CfxtwJWLjFRW05lFZCPwRWz2rdhDMH9Z7fM2Widf6xQLFUspdVa/DwM/wW545/PDMPp9WJvXUz3my7jo7Eqp8/ofxgKe4uItdl0wi4gHu7N8Vin177q4rv08G3O9+1kzRoD9wHXYQxr5pIvF5y5w6f1BYHQxeKcx36aHypRSKg08wwL4uNE6+deA9XoW3Ys9ifLiIjMhIm0iEshvA7cAB7HZ8rPf9wP5ZaheBP5Iz6BvByaKbuVrrfky7gNuEZEl+vb9Fl1WM02bv7gLCulIXwTu1dEUa4H1wKvUsN3osd5/Bd5RSn29aFfd+nku5nr1s4h0i0hIb/uBm7HnEfZjJ1SEmT7O+/5u4Jf6bmquelRdczC/W/TDL9hzCMU+rk67eL+zxYv1wp51fg97DO6xxebRTH3Ys/RvAYfyXNjjfv8FHAV+AYTVxZn2b+o6vA1cUyPOH2LfdpvYY3mfeT+MwEPYk1T9wIOLwLxHMx3Q/wzLiuwf08xHgB21bjfA9dhDMQeAN/Xr9nr2cwnmuvQzcBXwG811EPgbXd6H3Un3Az8GWnS5T3/u1/v7ytWjhsy/1D4+CHyfixE4VWsXTloDR44cOWpiNdpwjSNHjhw5moecTt6RI0eOmlhOJ+/IkSNHTSynk3fkyJGjJpbTyTty5MhRE8vp5B05cuSoieV08o4cOXLUxPp/V6UlvODXOgwAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["### CMAB demo on Simulated dataset"],"metadata":{"id":"o_S529BK1qop"}},{"cell_type":"code","source":["# generate 100 000 samples with 4 features and 3 actions\n","dataset = generate_samples(100000, 4, 3, True)\n","dataset.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"8OOtls5G1qlT","executionInfo":{"status":"ok","timestamp":1639149837227,"user_tz":-330,"elapsed":4833,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"597535e2-9ebb-4bce-a443-6c90344656dc"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
X_1X_2X_3X_4ayprobs
013000.00.00.266661
131311.00.00.236514
230011.00.00.236514
303101.00.00.236514
421200.00.00.266661
\n","
"],"text/plain":[" X_1 X_2 X_3 X_4 a y probs\n","0 1 3 0 0 0.0 0.0 0.266661\n","1 3 1 3 1 1.0 0.0 0.236514\n","2 3 0 0 1 1.0 0.0 0.236514\n","3 0 3 1 0 1.0 0.0 0.236514\n","4 2 1 2 0 0.0 0.0 0.266661"]},"metadata":{},"execution_count":71}]},{"cell_type":"markdown","source":["#### LinTS dynamic by steps"],"metadata":{"id":"h9x6zOMx1qdr"}},{"cell_type":"code","source":["num_experiments = 10\n","batch_size1 = 30\n","batch_size2 = 100\n","env_info = {'pickle_file': dataset}\n","\n","agent1_info = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch_size1,\n"," 'replay_buffer_size': 100000}\n","agent2_info = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch_size2,\n"," 'replay_buffer_size': 100000}\n","experiment_parameters = {\"num_runs\": num_experiments}"],"metadata":{"id":"SDmmh_Js1qag"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["agent = LinTSAgent\n","environment = ReplayEnvironment\n","\n","result1 = run_experiment(environment, agent, env_info, agent1_info, experiment_parameters, False)\n","result2 = run_experiment(environment, agent, env_info, agent2_info, experiment_parameters, False)\n","\n","smoothed_leveled_result1 = smooth(result1, 100)\n","smoothed_leveled_result2 = smooth(result2, 100)\n","\n","mean_smoothed_leveled_result1 = np.mean(smoothed_leveled_result1, axis=0)\n","mean_smoothed_leveled_result2 = np.mean(smoothed_leveled_result2, axis=0)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["79e9b193d9284e4b898cb4344db92ef3","8ea51857d7cf48c1beddb0a9bd3b764b","7b0e4960186b4307ae594b269043c16d","aa39361015a84c158e922f8e0b8dca9a","63acbf6a68e6455c91cbea40169d1ee9","e2c2139f96b345768e075ef3593fac06","3dcbffa6176e4824b9a48cccfaf024cc","740c9d9905b542d4a0edef78f1d8ca30","cd2dc9d9c7504cd4b118a33b7c24f683","33707c49e4fd42ad932dd74e59f4a060","60745768aedc4942bad8883d708b0e25","48077097754d4b96b1345fb1c06560b8","760943cea30f4236a0715b61a047fd99","3bf9f4be47424b13b1db937f67894360","28c1b766f56c41179840f2140da35b44","487beca1cfc847c0a41a414968097b6b","c28b5aecd2eb4353b99534e8eef2380c","f167c63d8ac54b07b31b261c77a98391","4a8e550188ac481fa2838df0996d3b0b","50dfc1413ebd4298af9fc979de4ebb63","f36eb7ce107741159cfc86169724580f","5b4edf526246433d8cca04eba8be3229"]},"id":"Sp9wGAPS1-V-","executionInfo":{"status":"ok","timestamp":1639150020202,"user_tz":-330,"elapsed":182996,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b1fc5a37-fb85-44dc-cb62-57e3a9813b63"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"79e9b193d9284e4b898cb4344db92ef3","version_minor":0,"version_major":2},"text/plain":[" 0%| | 0/10 [00:00"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### LinTS dynamic by batches"],"metadata":{"id":"B3W58Zqv2CnH"}},{"cell_type":"code","source":["num_experiments = 20\n","env_info = {'pickle_file': dataset}\n","experiment_parameters = {\"num_runs\": num_experiments}\n","\n","agent = LinTSAgent\n","environment = ReplayEnvironment\n","\n","# run batch agent\n","batch_sizes = np.logspace(1.0, 2.7, num=20).astype(int)\n","actual_regret = []\n","for batch in batch_sizes:\n"," agent_info_batch = {'alpha': 1,\n"," 'num_actions': 3,\n"," 'seed': 1,\n"," 'batch_size': batch,\n"," 'replay_buffer_size': 100000}\n"," batch_result = run_experiment(environment, agent, env_info, agent_info_batch,\n"," experiment_parameters, False)\n"," # smooth and average the result\n"," smoothed_leveled_result1 = smooth(batch_result, 100)\n"," mean_smoothed_leveled_result1 = np.mean(smoothed_leveled_result1, axis=0)\n"," mean_smoothed_leveled_result1 = mean_smoothed_leveled_result1[~np.isnan(mean_smoothed_leveled_result1)]\n","\n"," actual_regret.append(mean_smoothed_leveled_result1[-1])"],"metadata":{"id":"Vs_O6HE92Ckv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plt.plot(batch_sizes, actual_regret, label='actual regret')\n","plt.legend()\n","plt.title(\"Reward as a f-n of batch size (each point is averaged over {} runs)\".format(num_experiments))\n","plt.xlabel('batch size (log scale)')\n","plt.ylabel('reward')\n","plt.grid(b=True, which='major', linestyle='--', alpha=0.5)\n","plt.minorticks_on()\n","plt.grid(b=True, which='minor', linestyle=':', alpha=0.2)\n","plt.show()"],"metadata":{"id":"4iErUrNt2CiB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"id":"ajBoPT1U2CfL"},"execution_count":null,"outputs":[]}]} \ No newline at end of file diff --git a/_notebooks/2022-01-12-sess-word2vec.ipynb b/_notebooks/2022-01-12-sess-word2vec.ipynb new file mode 100644 index 0000000..f7c740c --- /dev/null +++ b/_notebooks/2022-01-12-sess-word2vec.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"markdown","source":["# Training Session-based Product Recommender using Word2vec on Retail data"],"metadata":{"id":"0M2nRjhs2bhj"}},{"cell_type":"markdown","source":["## Executive summary\n","\n","| | |\n","| --- | --- |\n","| Problem | A key trend over the past few years has been session-based recommendation algorithms that provide recommendations solely based on a user’s interactions in an ongoing session, and which do not require the existence of user profiles or their entire historical preferences. This tutorial explores a simple, yet powerful, NLP-based approach (word2vec) to recommend a next item to a user. While NLP-based approaches are generally employed for linguistic tasks, can we exploit them to learn the structure induced by a user’s behavior or an item’s nature. |\n","| Prblm Stmnt. | Given a series of events (e.g. user's browsing history), the task is to predict the next event. |\n","| Solution | We will implement a simple, yet powerful, NLP-based approach (word2vec) to recommend a next item to a user. While NLP-based approaches are generally employed for linguistic tasks, here we exploit them to learn the structure induced by a user’s behavior or an item’s nature. |\n","| Dataset | Retail Session Data |\n","| Preprocessing | There are some rows with missing information, so we'll filter those out. Since we want to define customer sessions, we'll use group by CustomerID field and filter out any customer entries that have fewer than three purchased items. We used withholding the last element of the session for sessionization. |\n","| Metrics | Recall, MRR |\n","| Models | Prod2vec |\n","| Cluster | Python 3.6+, RayTune |\n","| Tags | `SessionRecommender`, `Word2vec`, `HyperParamOptimization`, `RayTune` |\n","| Credits | Cloudera Fast Forward Labs |"],"metadata":{"id":"eZrZz_8u2_yL"}},{"cell_type":"markdown","source":["## Process flow\n","\n","![](https://github.com/RecoHut-Stanzas/S810511/raw/main/images/S810511%20%7C%20Retail%20Session-based%20recommendations.drawio.svg)"],"metadata":{"id":"PVgIuKuy6rYX"}},{"cell_type":"markdown","source":["A user’s choice of items not only depends on long-term historical preference, but also on short-term and more recent preferences. Choices almost always have time-sensitive context; for instance, “recently viewed” or “recently purchased” items may actually be more relevant than others. These short-term preferences are embedded in the user’s most recent interactions, but may account for only a small proportion of historical interactions. In addition, a user’s preference towards certain items can tend to be dynamic rather than static; it often evolves over time.\n","\n","A key trend over the past few years has been session-based recommendation algorithms that provide recommendations solely based on a user’s interactions in an ongoing session, and which do not require the existence of user profiles or their entire historical preferences.\n","\n","We will implement a simple, yet powerful, NLP-based approach (word2vec) to recommend a next item to a user. While NLP-based approaches are generally employed for linguistic tasks, here we exploit them to learn the structure induced by a user’s behavior or an item’s nature."],"metadata":{"id":"DKJ0AqIzuXJ5"}},{"cell_type":"markdown","metadata":{"id":"XIY8wbf-VFyT"},"source":["## Install/import libraries"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"5pz-XFyoj5is"},"outputs":[],"source":["!sudo apt-get install -y ray\n","!pip install ray\n","!pip install ray[default]\n","!pip install ray[tune]"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"zV-QhI_1UkjA"},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","import os\n","import pickle\n","from numpy.random import default_rng\n","import collections\n","import itertools\n","from copy import deepcopy \n","\n","from gensim.models.word2vec import Word2Vec\n","from gensim.models.callbacks import CallbackAny2Vec\n","from ray import tune\n","\n","import os\n","import argparse\n","import ray\n","import time\n","\n","MODEL_DIR = \"/content\""]},{"cell_type":"code","execution_count":null,"metadata":{"id":"XDOxi0wabyQs"},"outputs":[],"source":["plt.style.use(\"seaborn-white\")\n","cldr_colors = ['#00b6b5', '#f7955b','#6c8cc7', '#828282']#\n","cldr_green = '#a4d65d'\n","color_palette = \"viridis\"\n","\n","rng = default_rng(123)\n","\n","ECOMM_PATH = \"/content\"\n","ECOMM_FILENAME = \"OnlineRetail.csv\"\n","\n","%load_ext tensorboard"]},{"cell_type":"markdown","metadata":{"id":"4UPgoIvlU9z2"},"source":["## Load data"]},{"cell_type":"markdown","source":["We chose an open domain e-commerce dataset from a UK-based online boutique selling specialty gifts. This dataset was collected between 12/01/2010 and 12/09/2011 and contains purchase histories for 4,372 customers and 3,684 unique products. These purchase histories record transactions for each customer and detail the items that were purchased in each transaction. This is a bit different from a browsing history, as it does not contain the order of items clicked while perusing the website; it only includes the items that were eventually purchased in each transaction. However, the transactions are ordered in time, so we can treat a customer’s full transaction history as a session. Instead of predicting recommendations for what a customer might click on next, we’ll be predicting recommendations for what that customer might actually buy next. Session definitions are flexible, and care must be taken in order to properly interpret the results.\n","\n","The dataset is composed of the following columns:\n","\n","- **InvoiceNo**: Invoice number. Nominal, a 6-digit integral number uniquely assigned to each transaction. If this code starts with letter 'c', it indicates a cancellation.\n","- **StockCode**: Product (item) code. Nominal, a 5-digit integral number uniquely assigned to each distinct product.\n","- **Description**: Product (item) name. Nominal.\n","- **Quantity**: The quantities of each product (item) per transaction. Numeric.\n","- **InvoiceDate**: Invice Date and time. Numeric, the day and time when each transaction was generated.\n","- **UnitPrice**: Unit price. Numeric, Product price per unit in sterling.\n","- **CustomerID**: Customer number. Nominal, a 5-digit integral number uniquely assigned to each customer.\n","- **Country**: Country name. Nominal, the name of the country where each customer resides."],"metadata":{"id":"6RToYkyYuUsw"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"t5DGOggiTryP","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1639397725768,"user_tz":-330,"elapsed":2418,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"42c4d353-8dc5-49df-d01f-9c08b3c51ded"},"outputs":[{"output_type":"stream","name":"stdout","text":["--2021-12-13 12:15:32-- https://github.com/RecoHut-Datasets/retail_session/raw/v1/onlineretail.zip\n","Resolving github.com (github.com)... 13.114.40.48\n","Connecting to github.com (github.com)|13.114.40.48|:443... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://raw.githubusercontent.com/RecoHut-Datasets/retail_session/v1/onlineretail.zip [following]\n","--2021-12-13 12:15:32-- https://raw.githubusercontent.com/RecoHut-Datasets/retail_session/v1/onlineretail.zip\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 7548702 (7.2M) [application/zip]\n","Saving to: ‘data.zip’\n","\n","data.zip 100%[===================>] 7.20M --.-KB/s in 0.1s \n","\n","2021-12-13 12:15:33 (70.4 MB/s) - ‘data.zip’ saved [7548702/7548702]\n","\n","Archive: data.zip\n"," inflating: OnlineRetail.csv \n"]}],"source":["!wget -O data.zip https://github.com/RecoHut-Datasets/retail_session/raw/v1/onlineretail.zip\n","!unzip data.zip"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Hgc7vOvTV7y9"},"outputs":[],"source":["def load_original_ecomm(pathname=ECOMM_PATH):\n"," df = pd.read_csv(os.path.join(pathname, ECOMM_FILENAME),\n"," encoding=\"ISO-8859-1\",\n"," parse_dates=[\"InvoiceDate\"],\n"," )\n"," return df"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":293},"id":"TmsQtNH4dEwb","outputId":"88389534-aedf-4c66-9d27-cfaff583d2bd","executionInfo":{"status":"ok","timestamp":1639397728832,"user_tz":-330,"elapsed":3070,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
InvoiceNoStockCodeDescriptionQuantityInvoiceDateUnitPriceCustomerIDCountry
053636585123AWHITE HANGING HEART T-LIGHT HOLDER62010-12-01 08:26:002.5517850.0United Kingdom
153636571053WHITE METAL LANTERN62010-12-01 08:26:003.3917850.0United Kingdom
253636584406BCREAM CUPID HEARTS COAT HANGER82010-12-01 08:26:002.7517850.0United Kingdom
353636584029GKNITTED UNION FLAG HOT WATER BOTTLE62010-12-01 08:26:003.3917850.0United Kingdom
453636584029ERED WOOLLY HOTTIE WHITE HEART.62010-12-01 08:26:003.3917850.0United Kingdom
\n","
"],"text/plain":[" InvoiceNo StockCode ... CustomerID Country\n","0 536365 85123A ... 17850.0 United Kingdom\n","1 536365 71053 ... 17850.0 United Kingdom\n","2 536365 84406B ... 17850.0 United Kingdom\n","3 536365 84029G ... 17850.0 United Kingdom\n","4 536365 84029E ... 17850.0 United Kingdom\n","\n","[5 rows x 8 columns]"]},"metadata":{},"execution_count":6}],"source":["df = load_original_ecomm()\n","df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QnlEr3cngRbq","outputId":"fafe3543-51e9-4290-bf18-0f14063879b3","executionInfo":{"status":"ok","timestamp":1639397728833,"user_tz":-330,"elapsed":11,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":["InvoiceNo 0\n","StockCode 0\n","Description 1454\n","Quantity 0\n","InvoiceDate 0\n","UnitPrice 0\n","CustomerID 135080\n","Country 0\n","dtype: int64"]},"metadata":{},"execution_count":7}],"source":["df.isnull().sum()"]},{"cell_type":"markdown","metadata":{"id":"XedYV2JkWigc"},"source":["## Preprocess"]},{"cell_type":"markdown","metadata":{"id":"474PEoLOgFI_"},"source":["There are some rows with missing information, so we'll filter those out. Since we want to define customer sessions, we'll use group by CustomerID field and filter out any customer entries that have fewer than three purchased items."]},{"cell_type":"markdown","source":["- Personally identifying information has already been removed\n","- We removed entries that did not contain a customer ID number (which is how we define a session)\n","- We removed sessions that contain fewer than three purchased items. A session with only two, for instance, is just a [query item, ground truth item] pair and does not give us any examples for training"],"metadata":{"id":"NLatMazxvKAt"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"3KVsMP58gp5h"},"outputs":[],"source":["def preprocess_ecomm(df, min_session_count=3):\n","\n"," df.dropna(inplace=True)\n"," item_counts = df.groupby([\"CustomerID\"]).count()[\"StockCode\"]\n"," df = df[df[\"CustomerID\"].isin(item_counts[item_counts >= min_session_count].index)].reset_index(drop=True)\n"," \n"," return df"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":293},"id":"5138zkW_g1g0","outputId":"0f6eceb6-f226-43d0-c32e-9483841b2ab5","executionInfo":{"status":"ok","timestamp":1639397729609,"user_tz":-330,"elapsed":22,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
InvoiceNoStockCodeDescriptionQuantityInvoiceDateUnitPriceCustomerIDCountry
053636585123AWHITE HANGING HEART T-LIGHT HOLDER62010-12-01 08:26:002.5517850.0United Kingdom
153636571053WHITE METAL LANTERN62010-12-01 08:26:003.3917850.0United Kingdom
253636584406BCREAM CUPID HEARTS COAT HANGER82010-12-01 08:26:002.7517850.0United Kingdom
353636584029GKNITTED UNION FLAG HOT WATER BOTTLE62010-12-01 08:26:003.3917850.0United Kingdom
453636584029ERED WOOLLY HOTTIE WHITE HEART.62010-12-01 08:26:003.3917850.0United Kingdom
\n","
"],"text/plain":[" InvoiceNo StockCode ... CustomerID Country\n","0 536365 85123A ... 17850.0 United Kingdom\n","1 536365 71053 ... 17850.0 United Kingdom\n","2 536365 84406B ... 17850.0 United Kingdom\n","3 536365 84029G ... 17850.0 United Kingdom\n","4 536365 84029E ... 17850.0 United Kingdom\n","\n","[5 rows x 8 columns]"]},"metadata":{},"execution_count":9}],"source":["df = preprocess_ecomm(df)\n","df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0P1BcYYYhDqI","outputId":"f54b9869-1131-4708-dc37-17d30321d43a","executionInfo":{"status":"ok","timestamp":1639397729611,"user_tz":-330,"elapsed":21,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":["4234"]},"metadata":{},"execution_count":10}],"source":["# Number of unique customers after preprocessing\n","df.CustomerID.nunique()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wq63Y-D-hWKR","outputId":"fd780147-969a-4570-e330-e3b678847fb4","executionInfo":{"status":"ok","timestamp":1639397729612,"user_tz":-330,"elapsed":17,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":["3684"]},"metadata":{},"execution_count":11}],"source":["# Number of unique stock codes (products)\n","df.StockCode.nunique()"]},{"cell_type":"markdown","metadata":{"id":"lKpBJdaOiExk"},"source":["## Product popularity\n","Here we plot the frequency by which each product is purchased (occurs in a transaction). Most products are not very popular and are only purchased a handful of times. On the other hand, a few products are wildly popular and purchased thousands of times."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":441},"id":"TZLOGjaPiHPY","outputId":"ae45988a-c017-445d-cfaf-a459dd150fc9","executionInfo":{"status":"ok","timestamp":1639397731128,"user_tz":-330,"elapsed":1528,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAjgAAAGoCAYAAABL+58oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUVeLG8Tc9QAokBAgQuiGQBEJHAUFQUSmKi6KiLj8rKLuugLvoiooNXcSCKCpYsIJ13VWKuFhgFxAFBAKE3kkIgfQyycz5/YEZjSQwQCZ3ZvL9PI/PY86dZN6TZDIv9557r58xxggAAMCH+FsdAAAAoLpRcAAAgM+h4AAAAJ9DwQEAAD4n0OoA1am4uFibNm1STEyMAgICrI4DAADcyG63KzMzU0lJSQoNDa2wzacKzqZNmzR69GirYwAAgBr03nvvqXv37hXGfKrgxMTESDox0SZNmlicBgAAuFN6erpGjx7tfP//LZ8qOOWHpZo0aaLmzZtbnAYAANSEypalsMgYAAD4HAoOAADwORQcAADgcyg4AADA51BwAACAz6HgAAAAn0PBAQAAPoeCAwAAfA4FBwAA+BwKDgAA8DkUHAAA4HMoOAAAwOdQcAAAgM+h4AAAALcwxuinvccseW4KDgAAcIuVO7P0h9krteNIXo0/NwUHAAC4RW5xqSSp1G5q/LkpOAAAwC1svxSboAC/Gn9uCg4AAHCLMrtDkhQUUPN1g4IDAADcopSCAwAAfE35IapADlEBAABfUVp2Yg9OMHtwAACAryhzcIgKAAD4mFLnWVQUHAAA4CNsZeV7cFiDAwAAfESZw6FAfz/5+VFwAACAjyi1G0sOT0kUHAAA4Ca2Moclp4hLFBwAAOAmpXaHJaeISxQcAADgJmUcogIAAL6m1O5QUCCHqAAAgA+x2R0K8mcPDgAA8CGHsosUHhpoyXNTcAAAgFtkF5YqLqquJc9NwQEAAG6RX1KmesHswQEAAD6k0GZXvRAKDgAA8BHGGBXYyhQWEmDJ81NwAABAtSu02WWM2IMDAAB8R0FJmSSpLgUHAAD4igKbXZI4RAUAAHxH+R4czqICAAA+I/+XghPGISoAAOArNh/KlWTdImNrnvUs5Ofna9KkSSooKFBpaakeeOABderUyepYAACgEj/tOy5Jat8k3JLn95o9OF999ZUuvPBCvfPOO5o0aZJefPFFqyMBAIAqHM4u0gVtoxUaZM0iY6/Zg3P11Vc7///QoUNq0qSJhWkAAMCpZOaXqFuLBpY9vyV7cNLS0jR06FANHDiwwvjhw4c1duxY9erVS/3799ejjz4qm83m3J6Xl6eRI0fq5Zdf1l/+8peajg0AAFywMzNf+48V6bzG1hyekiwoOAsXLtRtt92mli1bnrRt/PjxatCggZYuXar3339f69at08yZM53bw8PD9fHHH+tPf/qTnnzyyZqMDQAAXDTvf3skSVd3bWZZhhovOIWFhVqwYIHOP//8CuMbN27U5s2bdd999ykiIkLNmjXTnXfeqQ8//FAOh0Pr169XVlaWJGngwIFav359TUcHAAAu2HAgR8EB/oqNrGNZhhovOCNHjlTTpk1PGk9NTVVsbKyioqKcY4mJicrJydG+ffu0cuVKffjhh5KkTZs2qVWrVjUVGQAAuMjuMNp0MEc39GphaQ6PWWScnZ2tiIiICmORkZGSpOPHj+umm27S3/72N40ePVp2u12PPvqoFTEBAMApHMkrVpnD6LzGYZbm8JiCI524tXpVwsLC9NJLL9VgGgAAcKb2ZhVKkprWt+7wlORB18GJiopSdnZ2hbHyj6Ojo62IBAAAzoAxRrO/3angAH8lWHSBv3IeU3CSkpKUkZGhzMxM59iGDRsUHR2tuLg4C5MBAABXZOaV6LttmbqzfxtLFxhLHlRwOnbsqJSUFE2fPl15eXnav3+/Zs+erdGjR8vPz8/qeAAA4DS+STsiSUpuFmlxEgvW4AwePFiHDh2Sw+FQWVmZkpOTJUmLFy/WCy+8oIcfflj9+vVTaGioRowYobFjx9Z0RAAAcBbeXrlXEaGBujA+xuooNV9wlixZcsrtr7zySg0lAQAA1SUzr0Sph3J13+D2lt1/6rc85hAVAADwXsu3n1hDe+F51u+9kSg4AACgGny/LVPR9YKV2DTi9A+uARQcAABwTnKLS/Xttkz1O6+h/P0948QgCg4AADgn0xZuUXZhqcb0aW11FCcKDgAAOGt2h9H3246qVXRdpcTVtzqOEwUHAACctQ9+2KeD2UW695J4q6NUQMEBAABnxeEwevX7nUpoEq7hnZtaHacCCg4AADgrq3cf0/5jRbqzfxuPu+sABQcAAJyVD3/cr7CQQF2WGGt1lJNQcAAAwBnbf6xQn607qJHdmqtOsPVXLv49Cg4AADhjH/10QJI0ulcLi5NUjoIDAADOSEFJmb7ZekRNI0N1XuNwq+NUioIDAABclltcqmteWamNB3P0l4s969Tw36rxu4kDAADv9fry3dqSnqtnrumskd2aWx2nSuzBAQAALjHG6NN1B9StRQOPLjcSBQcAALho1rId2n+sSNd09+xyI1FwAACAC+wOoy83HpYkXd2VggMAAHzAi8u2a2t6nl64LkVBAZ5fHzw/IQAAsNTn6w/q+a+36+ouzXRlSjOr47iEggMAAKr03x1Hde+C9UpsGqFpf0i2Oo7LKDgAAKBSO47ka/Tc1WoYFqL3buulkEDPuyVDVSg4AACgUm/9b7ck6dlrU1S/brDFac4MF/oDAAAVGGP0xn/36N1V+3RFchP1Pa+h1ZHOGAUHAAA45ZeU6d4F67V0c4b6tmuoZ69NsTrSWaHgAAAASSf23Pzt4w36z5YMPTikg27p01r+/n5WxzorFBwAACBJmrt8t77ceFj3DW6v2/q1sTrOOWGRMQAAkDFGH/ywT80b1NHY/m2tjnPOKDgAAECLNqVr19EC3TWgnQK89LDUb1FwAACo5dLS8zTpo5+V2DRC13rBjTRdQcEBAKAW23EkT9fPWaU6QQF6/Y89FOgF95lyhW/MAgAAnDGHw+jpxWkqszv08bgL1CQy1OpI1YazqAAAqIV2Hy3QbfPWaGdmgcYNaKvWDetZHalaUXAAAKhlcotLNXrOKh3KKdbU4Ym6+fyWVkeqdhQcAABqkZ2Z+brj7R91KKdYT/8hWaN6tLA6kltQcAAAqAWO5pfo7vfWavXuY4qsE6R3b+3llfeYchUFBwAAH3e8wKZrX1mpXUcLdH3PFrprQFvFRdW1OpZbUXAAAPBhGbnFuuaVldp3rFBjLmilR4YnWh2pRlBwAADwUT/vz9aYN3/Q8cJSvTy6q65IjrU6Uo2h4AAA4GM2HczRxz8d0Fv/2yNJuv/yhFpVbiQKDgAAPuNIXrHu/2Sj/rP1iCSpb7uGemR4R7VrFG5xsppHwQEAwMs5HEbv/7BPT3y5RUWldl3bvbkmXtpejSN858rEZ4qCAwCAFzuSW6w/vrlGWw7nKrlZpP4xspM6xEZYHctyFBwAALyQMUbfpB3RpI82qMhm1xMjknRdjxYK8PezOppHoOAAAOBF7A6jz9cf1Cvf7dS2jHzFRobqrf/roU7N61sdzaNQcAAA8ALGGH2+/pCeWLhFmXklala/jh4c0kGjesQpPDTI6ngeh4IDAICH+yo1XQ98tklH80vUNqae/jq4va7q0kxBAf5WR/NYFBwAADxURm6x7vt4g77flqmQQH/d2LuFHhzSUaFBAVZH83gUHAAAPNDerALd/MYP2nesUOMGtNU9g86j2JwBCg4AAB5k/7FCPfrFZi3dnKHQIH8tuON89WwdZXUsr0PBAQDAA5TaHZr/wz49sXCLbGUOje7VQrf2ba02MWFWR/NKFBwAACyUV1yq91bv0/ur92nfsUIlNo3QM9d05mJ954iCAwCABewOo+XbM3X3e2tVYLOrTcN6eurqZI3s1lyBnB11zig4AADUoO0ZeVq1+5je/t8ebT+Sr4jQQN1/eYLuuLCN/Py4CnF1oeAAAOBmDofR/DX79cnaA/pp73FJUlhIoB64IkGjurdQZF0u1FfdKDgAALiJw2H07w2HNPvbndqanqeQQH+NG9BWQ5JjdV7jMIUEctq3u1BwAABwg52Z+br7vbXamp6nyDpBevTKRN3UuyWHoWoIBQcAgGqUXWjTe6v3adayHbIbo8mXJ+jWvq25rUINo+AAAFBNVmw/qlvmrZGtzKG2MfX0yo3ddF7jcKtj1UoUHAAAzpExRruOFuie+etkK3No3i09deF5DTkcZSEKDgAAZym/pExvrtit+Wv262B2kYID/TV7dFf1j4+xOlqtR8EBAOAsfJN2RP/35hpJUkKTcP3l4vM0rHNTteXWCh6BggMAwBnYcCBbt877UZl5JZKkqcMTdfP5nB3laSg4AAC44Of92Xpn1V59/NMBSdINvVrob5clKLIOF+nzRBQcAACqUFxq1+frD2ru8t3afiRfknRhfIweHNJB8Zwd5dEoOAAA/EZBSZm+3pKhb9My9e+fD6nMYRRVL1j3XhyvK5KbcNq3l6DgAAAgaceRPM1atkP/XH/IOTYwoZGu6tJMlyc14UJ9XoaCAwCo1bILbZr67836bN1BSVL9ukEac0Er3davjcJCeJv0VvzkAAC10jdpR/TGit1avv2oJCkk0F/PXNNZwzo3tTgZqgMFBwBQaxTZ7Hp31V59tu6gNh/OlSQ1Cg/RpEvb69oecRanQ3Wi4AAAfFpmXon+/fMhffjjfm1Nz3OOd4iN0LPXdlaH2AgL08FdKDgAAJ819d+pevO/e5wft2sUput6xOmm81sqJDDAumBwOwoOAMDn/LT3mCZ++LP2ZBVKkp4YkaQRXZqpbjBve7UFP2kAgE/Yf6xQ/1x3UG/8d7eOF5ZKOnGPqHdv66WGYSEWp0NNo+AAALyWMUaLNqXryYVbdOB4kXN8QPsY/WngeerWsoGF6WAlCg4AwKvkl5Rpy+FcLd6UrtdX7HaOd2oeqTsvbKtBHRopNIj1NbUdBQcA4PHsDqP/bMnQ5+sP6cuNhytsG9U9Tndf1E4toutalA6eiIIDAPBYP+w+pk/XHtD8NfudY/WCA3T7hW10QduG6hwXydlQqNRZF5zc3FwdOHBA7dq1U3BwcHVmAgDUYnnFpfrnuoOa8nmqcyw40F8D4mN0/xUd1LphPQvTwVu4VHB27dqlcePGacaMGUpKStLq1as1duxYFRcXq0GDBpo7d646duzo7qwAAB+WmVeiu99bqx/2HHOODekUq7EXtlVy80gLk8EbuVRwpk2bprZt26pVq1aSpMcff1ydO3fW3/72N82bN0/PP/+8XnvtNXfmBAD4GLvD6HBOkZ5enKb1+49r/7ETZ0EFB/prypAOGpzYRI0iQi1OCW/lUsH5+eef9e677yosLEy7du3S9u3b9dhjj6lDhw66/fbbdeONN7o7JwDAR+w/Vqh3V+/VnO93yWFOjAUH+qtHqwa6vmcLXZnSTAH+ftaGhNdzqeCUlpYqLCxMkrRy5UpFRUUpJSVFkhQaGqrCwkL3JQQA+ARjjJ75Kk0vfbPTOXZ1l2a6KKGRhnaKlZ8fpQbVx6WC06pVKy1ZskRXXXWV5s+fr4EDBzq3rV27Vk2aNHFbQACAd9tztEAzl23Xoo3pKiq1S5KevbazLktqwq0T4DYu/Wbdfvvtuu+++/SPf/xDERERuu222yRJq1ev1iOPPKJx48a5NSQAwLvYyhx65budmrN8l/KKyyRJAf5+ujypiR67KolbJ8DtXCo4V1xxhRISEpSWlqauXbuqcePGkqTIyEj99a9/1ahRo9waEgDgHbLyS/Tuqn167uttzrFBCY10ddfmuiypCWtrUGNcKjj333+//v73v6tNmzYVxhMSEhQSEqI///nPmjlzplsCAgA82+GcIj30ear2HC3Q9iP5zvErkpvomWs6cxgKlnDpt+6f//yn7rvvvkq37dy5U9988021hgIAeLZCW5m2pufpox/364MfTlxluE5QgLq0qK9R3eP0h27NFRTgb3FK1GanLDgJCQnOVe19+vSp8nFc5A8Aage7w+i173fp6cVbK4xPuCRe4y9qJ38OQcFDnLLgrFixQuvXr9f48eM1duxY1a178o3MIiMjdckll7gtYLnS0lJNnjxZ6enpcjgcevzxx9W2bVu3Py8AQPps3QF99OMB/W9nlnPs+p5xGta5qc5vE80p3vA4pyw4DRs21MUXX6xp06ZpyJAhlt5z6vPPP1dMTIxmzJihb7/9Vi+99JKeffZZy/IAQG3wbdoRvbtqr77eckSSFBMeotG9WuiWvq0VERpkcTqgai6twRkxYoQyMzO1ZcsW5ebmyhhz0mOGDRtW7eF+a/jw4c7njY6OVk5OjlufDwBqqxOlZp++3pLhHPPzk/49vq+SmnFPKHgHlwrO559/rilTpshms1W63c/P74wKTlpamiZOnKjCwkItW7bMOX748GFNnTpV69atU2hoqAYNGqTJkycrODi4wt6jd999V5dffrnLzwcAOL09Rwv05MIt+mrziWITHOiv0b1aaFSPOLWLCVMgi4bhRVwqOC+99JIGDhyoW265RVFRUed0rHXhwoWaNm2aOnXqpC1btlTYNn78eMXHx2vp0qXKy8vT+PHjNXPmTE2aNMn5mFmzZslut2vkyJFnnQEAcILDYfTe6r16e+XeCqd4z7+jt3q3ibYwGXBuXCo4R44c0dy5c9WiRYtzfsLCwkItWLBAy5Ytq1BwNm7cqM2bN2vOnDmKiIhQRESE7rzzTj300EOaMGGC/P399fbbb2vbtm167rnnzjkHANRGxhg5jLR+/3H9c90hvbNqr3Nbz9ZRGtmtuUZ0acYp3vB6LhWc9u3b68iRI9VScKra85KamqrY2FhFRUU5xxITE5WTk6N9+/bJGKOFCxfq7bffVkBAwDnnAIDaoszu0LECm17/7269+t2uk7aP7NZcf72svRqFh1qQDnAPlwrOlClT9I9//EMTJkxQYmKigoKqf+V8dna2IiIiKoxFRp5YzHb8+HEtW7ZMWVlZuvXWWyWdOMOLPTkAcDJbmUOHsov09ZYMbcvI08c/HZDjN+eGXJ7URB1iIzQwoZESmoSztgY+yaWCc/fdd6uoqEjXX3+9JFW6B2XTpk3nHKays7PKTZw4URMnTjzn5wAAX3XgeKHmLt+td1btlf03jSY2MlQJTcI1tFNTXdWlGfeDQq3gUsEZOXKk2y/iFBUVpezs7Apj5R9HR7PQDQAqU2gr01epGZq3co/W7fv1b+ighEYa1rmp+rRrqJhw7tyN2selgvOnP/3J3TmUlJSkjIwMZWZmKiYmRpK0YcMGRUdHKy4uzu3PDwDeZP+xQr27em+FNTVtY+rpjgvbaFjnptzgErWeS6+ANWvWnPYxPXr0OKcgHTt2VEpKiqZPn64pU6YoOztbs2fP1ujRo7kEOABIKimz69u0TL363U6t/c3emuGdm+qui9oqoUnEKT4bqF1cKjg33XST/Pz8KqyR+X3p+P01baoyePBgHTp0SA6HQ2VlZUpOTpYkLV68WC+88IIefvhh9evXT6GhoRoxYoTGjh3r6lwAwCfZHUbz/rdHj36x2TnWMrqu7h7QToOTmiiyDrdMAH7PpYKzYMGCk8YKCwu1fv16LVu2TFOmTHH5CZcsWXLK7a+88orLXwsAfJnDYTRn+S49//V2FZXaJUkXtY/RvZfEq1Pz+hanAzybSwWnc+fOlY6ff/75atWqlV5++WWKCQBUg//tPKoth/P0ny0ZFe7c3bVFfb1yUzeuVQO46JxXoSUlJemBBx6ojiwAUCut3pWlnw9k640Ve5SeW1xh298uS9AtfVspJJALnAJn4pwKTkFBgRYsWKB69epVVx4AqBVsZQ4t+HG/3vrvbu3MLHCOX9Kxse68sI3OaxyuesEBXIQPOEsuFZzExMSTFhUbY+RwOCRJd911V/UnAwAfZMyJdTXPLNkmm/3E39DuLRtowiXxatmwnprVr2NxQsA3uFRwxo4dW+mp2mFhYUpKSlL37t2rPRgA+ApjjJZuztDRfJse+Gyjc7xnqyg9MSJJ5zUOtzAd4Js85kJ/AOBrjuaX6L1V+/TBD/sqrK3pHx+jF2/ooohQTu8G3MXlNThbt27V/PnztXnzZuXn5ys8PFzJyckaPXq0Wrdu7c6MAOBV0nOK9Y/FW/XpuoPOsQvaRuvvQzooul6ImkRyJhTgbi4VnBUrVmjs2LFq2LChEhMT1apVK+Xl5Wnx4sX6+OOP9dZbbyklJcXdWQHAYxWX2jV3+S59svagdh/9ddHw5MsTNLRTrJo3qGthOqD2cangvPjii7rqqqv02GOPVViLY7fbNXnyZM2YMUPvvPOO20ICgCdbuPGwJn74s/NifMM7N1XnuPq6pU8rbjUDWMSlgrN161ZNmzbtpBdqQECAbr/9do0aNcot4QDAE+UWl2p3ZoEmffSzCm12HcwukiR1jquv567trDYxYRYnBOBSwQkICFBpaWml2357fyoA8GVZ+SV6b/U+Pbt0m3OscUSIru3eXLf0bc3NLgEP4lLB6dKli1566SU99dRTqlv31+PI+fn5evbZZ9W1a1e3BQQAKxWX2vXPdQf1zqq9Sj2U6xy/MD5Gt/ZtrQvaRiuIi/EBHselgjNp0iT98Y9/1AUXXKB27dopLCxMeXl52rFjh+rUqaO3337b3TkBoEat3Xdcu345DFXu4g6NFd84TH+9LMHCZABc4VLB6dChg7788kt98sknSk1NVX5+vpo1a6bBgwdr5MiRioqKcndOAHC7o/klemZJmopK7fp8/SHneM9WUXpwaAfu4A14EZevgxMTE6M77rhD/v6/7ootKSlRSEiIW4IBQE0pLrVrxfajuu3tHyVJEaGBahldV3cPaKe+5zVUbGQoZ0MBXsalgpOfn68HH3xQYWFhevzxx53jt956q6KiovTkk08qLIyzBgB4l/3HCvXT3uO67+OfVWo/ccLE//VppYeGdqTQAF7OpZVxM2bMUGpqqi699NIK47fffrt27Nih6dOnuyUcALjLf3ccVb9/fKO/LFivUrtRx9gIvXdbLz087OSbCwPwPi7twVm2bJmef/55denSpcJ4//79FR4ernvuuUdTp051S0AAqE7HC2y6Z8F6fb8tU5J0fc8WGtu/jeIa1JW/P8UG8BUuFZzc3FxFRFR+fYeoqCjl5+dXaygAqE5H8oplK3PoxrmrtSerUJLUrH4dPXBFBw3pFGtxOgDu4FLBSUlJ0dy5czV16lQFBwc7x/Pz8/XMM88oKSnJbQEB4GxtOpijj37cr3kr9zrHYiNDdWPvlhrXvy17bAAf5lLBuf/++zVmzBj17t1bbdu2VZ06dZSfn++8Ds6bb77p7pwA4JJ9WYX6fnum9h8v1Kvf7XKOPzEiSXWDAzQ4sYnqBrt8AikAL+XSqzw+Pl5ffPGFPv30U6Wmpio3N1dt2rTRsGHDNHLkSIWHh7s7JwCc0oYD2Vq8KV0vf7uzwviTI5LV77yGiovibt5AbeLyP2OioqJ02223uTMLAJwRY4ye/3q7Dhwv0idrD0iSAvz9NKxTrB4Y0kGhQQGKCA2yOCUAK7CfFoBXevarNH27LVMbDuRIOrFo+IZeLXT3Re0sTgbAE1BwAHiN3OJS/fmDdcouLNX6/dmSpB6tGmj6yM5q1bCexekAeBIKDgCP9/Tirfo2LVNb03NljNQwLFj942P018vaK7FppNXxAHigaik4xhiu/AmgWh3NL9Gt835UQUmZdhw5ca2tSzo2VmSdID1+VZJCgwIsTgjAk7lUcAYNGqSPP/5YDRo0OGnbli1bdPvtt2vFihXVHg5A7fPIv1K1YsdRFdnsOphdpH7nNVRCk3Dd2re1urQ4+W8QAFTmlAVnzZo1kqSDBw/qp59+UmRkxV3BxhitWLFCeXl57ksIwKdl5BbrrvfWqtBmlyRtz8hTi+i6SomrrwHtY/TwsEQFB7p02zwAcDplwXnooYe0Z88e+fn5afz48VU+bujQodUeDIBvKy61608frNOeowXafiRffdpFq25woFpE1dFdA9qpc1x9qyMC8GKnLDiLFi1STk6OevXqpVdffVX165/8ByciIkKtW7d2W0AAvmXBmn1aujlD+SVlWrXrmBKbRmh456Z65prO7KkBUG1OuwYnMjJS//nPf9S0aVMZY+Tv/+sfoJKSEoWEhLg1IADfMPvbndp/vFCLNh6Ww0jNG9RRz1ZRevnGrmoYxt8RANXLpUXGkZGRuvfeexUWFqbHH3/cOX7rrbcqKipKTz75pMLCwtwWEoB3+nz9QR3MLlJJqUMv/Ge7wkMCFRocoImXxOu6ni2sjgfAh7lUcGbMmKHU1FRNmTKlwvjtt9+up59+WtOnT9fUqVPdEhCAd9mbVaCt6Xkqstn1lwXrneOB/n5665Ye6tYyysJ0AGoLlwrOsmXL9Pzzz6tLly4Vxvv376/w8HDdc889FBygljuSW6ySMofuePsnpWX8emblm2N66Py20Qrw91NQAGtsANQMlwpObm6uIiIiKt0WFRWl/Pz8ag0FwLus2H5UN76+2vnxH7o21y19Wyk0KEBtGtbjQqAAapxLBSclJUVz587V1KlTFRwc7BzPz8/XM888o6SkJLcFBOC5nl26Te+v3qfi0hPXsHnsykTVCQ7URe1jFM3CYQAWcqng3H///RozZox69+6ttm3bqk6dOsrPz9eOHTtUp04dvfnmm+7OCcBDHM4p0jNLtslmd+i/O46qTlCALk1srLgGdXXT+a2sjgcAklwsOPHx8friiy/06aefKjU1Vbm5uWrTpo2GDRumkSNHKjw83N05AVjs27QjOlZg06pdWfpk7QG1jK6r+nWCdFu/NrqhF2dEAfAsLt9sMyoqSrfddps7swDwMGV2h8ocRgezizTmzTXO8cg6QfrPhP4KZNEwAA/lUsGZNWvWaR9zqls5APA+xwps6v+Pb5RXUuYce35Uirq0qK/6dYMpNwA8mksF57XXXjtprKysTA6HQ/Xr11e9evUoOIAPMMbo6cVp2n+sUDlFpcorKdOo7seutP0AACAASURBVHFq1bCe6oUEaEinWE71BuAVXCo4GzZsOGmstLRUGzZs0PPPP6+JEydWezAANWf/sULtP16oghK7Xvlup2LCQxRZJ0id4+pr0uD2ignnjCgA3sXlNTi/FxQUpG7duumee+7Ro48+qk8//bQ6cwGoQde8slLpucXOj58ckaxLOja2MBEAnJuzLjjloqOjtXPnzurIAqCGlNkduvv9tTqcc6LUpOcWa1T3OI3o2kwhgf7q3Ly+xQkB4Ny4VHDWrl170pgxRrm5uXr33XfVpEmTag8GwD0OHC/UD7uPaUlqhjrERqhJRIgu7tBYY/q0UofYyq9YDgDexqWCc8MNN1R6qXVjjMLCwvTUU09VezAA7jH23Z+06WCuJOnBIR3Up11DixMBQPVzqeC8/fbbJ435+fkpLCxMLVq0UL169ao9GIDq8+naA/ps3UFJ0raMfF2e1ER/HnSeEppwkU4AvsmlgtOzZ0935wBQzUrK7DpeUCpJenvlXu3MzFe7RmFKbhap63q24HAUAJ9WZcG55ZZbzugLvfHGG+ccBkD1uf61VVq7L9v58dBOsZp1Q1cLEwFAzamy4JSWllb4eOfOncrPz1fbtm1Vt25d5ebmateuXYqOjlanTp3cHhTAqeWXlOnz9QdVWuaQJKWl5+mCttEa1rmpJKkva20A1CJVFpx33nnH+f+ffvqpli5dqunTpyssLMw5npGRoQceeEADBw50b0oAp7Vw42H9/bNNFcYuT47V9T25ESaA2selNTizZ8/WrFmzKpQbSWrcuLEmTpyoP//5z7r66qvdEhBA5fYcLdAzX6Wp1O745eNCSdLK+wcqNDBA/n5+iqwbZGVEALCMSwUnIyNDxphKt/n5+SkjI6NaQwE4vWVbj+iLDYcV3zhM/n5+8vOThnSKVWxkHaujAYDlXCo47du315QpUzRlyhQlJCQoODhYNptNGzdu1PTp0xUfH+/unECttvtogca+85OKSu3OsZyiUvn5SYvuuVAB/idfpwoAajOXCs5jjz2mcePGadSoUZJO7LUxxsgYo4YNG+rll192a0igtttwIFtpGXm6pGNjhYX8+rJt3ySccgMAlXCp4CQkJOirr77S6tWrtXPnThUUFKhu3bpq3bq1evfurZAQ7jQMVLdlWzM0/v11KnMY2R0nDhE/OSKZO3sDgAtcvtlmUFCQzj//fMXGxqqgoEBhYWFq1aqV/P393ZkPqLU2HshVoc2uO/u3kZ/81LR+KOUGAFzkUsGx2+2aMWOGPvzwQxUUFDjHw8PD9cc//lF333232wICtcW6fcd1/ZxVKvnlOjbGSOEhgbr/8g4WJwMA7+NSwXnxxRc1f/583XTTTUpOTla9evWUn5+vtWvXas6cOQoNDdWtt97q7qyAT0tLz1NxqUO39GmtsJAASeJ2CgBwllwqOP/617/0yCOPaPjw4RXGL7nkErVp00Zz5syh4ABnaOnmDN3/6Qbn+pri0hN7bu695DyFh3L9GgA4Fy4VnCNHjqhr18rvYdO7d29NnTq1WkMBtcFPe4/reGGpRvf69UrDLaLqUm4AoBq4VHCioqK0a9cuNW/e/KRt27dvV4MGDao9GOCLnl26Tat3ZUmS9mQVKLJOkB69MsniVADge1wqOIMHD9bf//533XPPPUpJSVFYWJjy8vK0du1avfjiixoyZIi7cwI+4f3VexXg76dW0fXUKrqeerWOsjoSAPgklwrOpEmTdPToUU2ZMqXCuJ+fn4YOHaqJEye6JRzg7UrK7Ppg9T4V/nIF4pyiUt3StzVnRgGAm7lUcEJCQvTcc89p8uTJSk1NVX5+vsLDw9WxY0c1btzY3RkBr7Vq1zE98u/NFcbiG4VblAYAag+XCs5NN92k5557To0bN6bQAKeQW1yq396XNiO3WJL0xZ/6ql2jMPn5SSGBARalA4Daw+W7ie/du1cNGzZ0dx7Aa72+Yrce+2JzpdsahYcoNIhiAwA1xaWCc++99+rpp5/WxRdfrA4dOqhevXonPaaq08iB2mJnZr7CQwL1l0viK4w3Cg9Ro4hQi1IBQO3kcsGRpA0bNkg6sbi4nDFGfn5+2rJlixviAZ5t/f5sLd2cLkn6ac9xRYcF69a+rS1OBQBwqeDMmzevQqkBcMKL/9mu/2w9okD/E6+PK5JjLU4EAJBcLDi9evVydw7AK+WVlKl3myjNv+N8q6MAAH7jlAVn8+bN+uCDD5Senq64uDhdddVV6tSpU01lAzzOU4u26tu0I86P92QV6IK2LL4HAE/jX9WGNWvW6Nprr9U333yj4uJiffvtt7r++uv19ddf12Q+wKN8seGQ8orL1DK6rlpG11X/+Bhd37PF6T8RAFCjqtyDM2vWLA0YMEDPPvusgoODZYzR9OnT9dRTT+niiy+uyYyAxyiy2XVZUhM9MSLZ6igAgFOosuBs3LhR8+bNU3BwsKQTZ06NGzdOb7zxhrKyshQdHV1jIQGrXPXSf7U9I8/5cYHNrnohLi1dAwBYqMq/1IWFhYqNrXhGSHh4uOrUqaPCwkIKDnyercyh9fuz1b1lA6XE1Zck+fv76boecRYnAwCczin/Kcqp4ajNin65QeZlSU10W782FqcBAJwJCg7wO3/9+Gf9d0eW7I4TN5WqE8wtFgDA25yy4Nxzzz0KCgqqMGaz2fS3v/1NoaG/Xnrez89Pr7/+unsSAjVs2dZMRdQJVNcWDRQU4K+BCY2sjgQAOENVFpwePXpIkkpLSyuMl99z6vfjgK8oLrVreOememhYR6ujAADOUpUF55133qnJHECNszuMXvh6m44XVizrhbYy1Qmu8hJRAAAvwPmuqLV2ZuZr5rIdCgsJVHDgr4Umql6IOjevb2EyAMC5ouCg1iqynThL6oXrUjSoQ2OL0wAAqhMFB7VGqd1R4eOCkjJJUp0gzpICAF9DwUGt8OTCLXrt+12VbqvLlYkBwOfwlx21Qlp6nmIjQzW6V8UbY4aFBCqpaYRFqQAA7kLBQa1QUmZXXFRdjR94ntVRAAA1wKvOhV27dq0uuOACff/991ZHgQeyO4x2Hy3Qrsz8k/7LKSpTSKBX/boDAM6B1+zBOXr0qF599VV16dLF6ijwUM98labZ3+6scvuQ5NgqtwEAfIvXFJyIiAjNmjVLU6ZMsToKPFRGbrGi6gXr4SquQNyjVVQNJwIAWMWSgpOWlqaJEyeqsLBQy5Ytc44fPnxYU6dO1bp16xQaGqpBgwZp8uTJCg4OVnBwsBVR4UVsZQ7VrxukK1OaWR0FAGCxGl+UsHDhQt12221q2bLlSdvGjx+vBg0aaOnSpXr//fe1bt06zZw5s6YjwkuVlDkUEsg1bQAAFuzBKSws1IIFC7Rs2TJt2bLFOb5x40Zt3rxZc+bMUUREhCIiInTnnXfqoYce0oQJE+TvzwJRnHDgeKFGz12tghJ7hfHcolJ15JRvAIAsKDgjR46sdDw1NVWxsbGKivp1nURiYqJycnK0b98+tWrVqoYSwtPtzCzQ3qxCXdqxsRqGh1TYdlH7RhalAgB4Eo9ZZJydna2IiIr/+o6MjJQkHT9+XEePHtULL7ygXbt2KTU1VR9++KFmzZplRVRYrLTsxC0Xxg9sp07cFBMAUAmPKTiSZIypclv37t31zjvv1GAaeCrbL/eUYr0NAKAqHlNwoqKilJ2dXWGs/OPo6GgrIsED2B1GWw7nOkuNJG3PyJckBXPhPgBAFTym4CQlJSkjI0OZmZmKiYmRJG3YsEHR0dGKi4uzOB2s8u+fD+kvC9ZXui081GN+fQEAHsZj3iE6duyolJQUTZ8+XVOmTFF2drZmz56t0aNHy8/Pz+p4sMixApsk6eXRXVU3+NdDUtH1QtQwLKSqTwMA1HI1XnAGDx6sQ4cOyeFwqKysTMnJyZKkxYsX64UXXtDDDz+sfv36KTQ0VCNGjNDYsWNrOiI8SOkvh6YGtI9R3WCP6eMAAA9X4+8YS5YsOeX2V155pYaSwBuUF5ygANbbAABcx7sGPJrNfuLMukB/DlMCAFzHPn9YJiO3WJe/sFx5xaVVPqbMYRQS6M86LADAGaHgwDIHjhfpWIFNQzvFqkVU3SofF984vAZTAQB8AQUHlin7ZX3NDT1b6IJ2DS1OAwDwJazBgWVKf1lfE8QF+wAA1Yx3Flim/AwpFhADAKobh6jgdqV2h+yOk+8zVmizS+IUcABA9aPgwK12Hy3Q4Oe/l63MUeVjQoMoOACA6kXBgVsdyi6SrcyhG3q1UFyDk8+Uql83SG1jwixIBgDwZRQcuFXZL4em/tC1ubq1bGBxGgBAbcGxAbhVmfNWCywkBgDUHAoO3Kr8VPAAzpQCANQgCg7cqszBzTIBADWPNTg4J3uzClRUaq9y+/5jRZLYgwMAqFkUHJy1n/Ye0x9mr3TpsWEh/KoBAGoO7zo4a1n5NknSA1ckVHoKeLmoesFqHBFaU7EAAKDg4OyVX534wvgYJTSJsDgNAAC/YuUnzlr5NW64lxQAwNNQcHDWyvfgBPjzawQA8Cy8M+GssQcHAOCpKDg4a/ZfrnHDKeAAAE/DImNUqbjUrpJT3AU8r7hMEntwAACeh4KDSu0/VqhBM76TzV51wSkXHMiOQACAZ6HgoFIZucWy2R26oVcLtY0Jq/JxTSJCVb9ucA0mAwDg9Cg4qFT5GVJDkmPVp11Di9MAAHBmOLaAStnNiYLj78f6GgCA96HgoFK/nCDFGVIAAK9EwUGlypyngFscBACAs8DbFyrlMFylGADgvXj3QqXKzw4PYA0OAMALUXBQqfKzqNiBAwDwRpwmXksdL7A5r0RcmSN5xZJYZAwA8E4UnFroWIFNvZ78WqV2c9rH1g3iVwQA4H1496qFsgttKrUbje7VQl1bNKjycQ3qBalFdN0aTAYAQPWg4NRC5WdI9WoTreGdm1qcBgCA6scS0lrol/XDnCEFAPBZFJxayHmGFP0GAOCjKDi1UPkhKn8aDgDAR1FwaqHy+0xxI00AgK+i4NRCv96GweIgAAC4CW9xtZC9/BAVe3AAAD6KglMLORwUHACAb6Pg1ELO08RZZAwA8FEUnFqo/DRxduAAAHwVVzL2cpsO5mhvVuEZfU5aRp4kLvQHAPBdFBwvN3ruauUUlZ7V5zaoF1zNaQAA8AwUHC9XZLNrVPc43dqv9Rl9Xr2QQDWrX8dNqQAAsBYFx8sZGTUMD1Z843CrowAA4DFYZOzlHIbTvQEA+D0KjpdzGCPqDQAAFVFwvJgxRsZIfuzBAQCgAgqOF/vljgscogIA4HcoOF7sl34jLkgMAEBFFBwvVn5XcH8aDgAAFVBwvFh5wQEAABVRcLwYa3AAAKgcBceL/VpwrM0BAICnoeB4MecaHPbgAABQAQXHi5UXHPoNAAAVUXC8mOOXQ1Rc6A8AgIooOF7MOA9RWRwEAAAPQ8HxYpxFBQBA5QKtDlCb7crM196swrP+/NziUknswQEA4PcoOBa6Yc5qpecWn/PXiagTVA1pAADwHRQcCxXYyjSkU6xu79fmrL9GUICfOjSJqMZUAAB4PwqOlYzUKDxEKXH1rU4CAIBPYZGxhYwkP7GABgCA6kbBsZAxhov0AQDgBhQcC53YgwMAAKobBcdCxnCbBQAA3IGCYyEjw20WAABwAwqOhdiDAwCAe1BwLMRZVAAAuAcFx0rswQEAwC0oOBYyMuy/AQDADSg4FmINDgAA7kHBsRBrcAAAcA8KjoW4kjEAAO5BwbEQVzIGAMA9KDgWMkYswgEAwA0oOBYxxkhiDw4AAO5AwbHIL/2GHTgAALgBBcciv/QbzqICAMANKDgWcR6iot8AAFDtKDgW+XUPDgAAqG4UHIuwBgcAAPeh4FjEqPwQFQ0HAIDqRsGxSPkeHAAAUP0oOBZjBw4AANUv0OoAZ2Lq1KnasmWLAgMDNW3aNMXFxVkd6aw51+CwzBgAgGrnNXtwVq5cqaysLM2fP1/jxo3Ts88+a3Wkc/LrGhyLgwAA4IO8puCsWrVKAwYMkCSdf/75+vnnn60NdI5+3YMDAACqmyUFJy0tTUOHDtXAgQMrjB8+fFhjx45Vr1691L9/fz366KOy2WySpKysLEVFRUmS/P395XA45HA4ajx7dXFeB4eGAwBAtavxNTgLFy7UtGnT1KlTJ23ZsqXCtvHjxys+Pl5Lly5VXl6exo8fr5kzZ2rSpEknfR1Tw6chORxG/95wSJl5JdXy9UrKTpQz1uAAAFD9arzgFBYWasGCBVq2bFmFgrNx40Zt3rxZc+bMUUREhCIiInTnnXfqoYce0oQJExQTE6OjR49KkkpLS+Xv7y9//5rbAVVUateUf25SbnFZtX7dZg3qVOvXAwAAFhSckSNHVjqempqq2NhY52EoSUpMTFROTo727dunCy64QG+88YZGjhyp7777Tj179qypyJKkeiGBWvPgxbKVVd9hsQB/P9UN9qoT2QAA8Aoe8+6anZ2tiIiICmORkZGSpOPHj6tHjx76+uuvdd111yk4OFhPP/10jWcMCQxQSGBAjT8vAAA4Mx5TcKTTr6u5//77aygJAADwZh5zmnhUVJSys7MrjJV/HB0dbUUkAADgpTym4CQlJSkjI0OZmZnOsQ0bNig6Otqrr1gMAABqnscUnI4dOyolJUXTp09XXl6e9u/fr9mzZ2v06NHccRsAAJyRGl+DM3jwYB06dEgOh0NlZWVKTk6WJC1evFgvvPCCHn74YfXr10+hoaEaMWKExo4dW9MRAQCAl6vxgrNkyZJTbn/llVdqKAkAAPBVHnOICgAAoLpQcAAAgM+h4AAAAJ9DwQEAAD6HggMAAHwOBQcAAPgcCg4AAPA5HnWzzXNlt9slSenp6RYnAQAA7lb+fl/+/v9bPlVwyu9jNXr0aIuTAACAmpKZmamWLVtWGPMzxhiL8lS74uJibdq0STExMQoICLA6DgAAcCO73a7MzEwlJSUpNDS0wjafKjgAAAASi4wBAIAPouAAAACfQ8EBAAA+h4LjgsOHD2vs2LHq1auX+vfvr0cffVQ2m83qWNWiffv2SkpKUnJysvO/hx9+WJL0ww8/6Nprr1XXrl112WWX6YMPPqjwue+9954uv/xyde3aVddee61+/PFHK6ZwRtLS0jR06FANHDiwwvi5zNVms2nq1KkaMGCAevXqpbFjx3rkpQoqm/vq1avVvn37Cj//5ORkffHFF87HePvcDx48qD/96U/q3bu3evfurXvuuUcZGRmSTnxPbr75ZnXv3l2DBg3Siy++qN8uS1y8eLGuvPJKdenSRcOHD9dXX33l3GaM0cyZM3XxxRere/fuuvnmm7V9+/Yan9+pVDX3AwcOVPraf+2115yf6+1zX79+vW688UZ17dpVffr00YQJE5xn2vry672qedeG1/pJDE7r6quvNpMnTzY5OTnmwIED5qqrrjLTp0+3Ola1iI+PN6tWrTpp/MiRI6ZLly7mvffeM0VFReann34yXbt2Nd99950xxphvvvnGdO3a1axZs8YUFxebDz74wHTt2tVkZmbW9BRc9uWXX5q+ffuau+66y1x00UXO8XOd61NPPWWuvPJKs2/fPpObm2smT55srrnmGkvmWJWq5r5q1SoTHx9f5ef5wtyHDh1qJk6caPLy8szRo0fNzTffbO644w5TVFRk+vfvb5599lmTn59vtm3bZvr372/ef/99Y4wxW7ZsMUlJSWbp0qWmuLjYfP311yY5OdmkpaUZY4x59913Tf/+/c3WrVtNQUGBee6558xFF11kiouLrZxuBVXNff/+/SY+Pt7s37+/0s/z9rlnZ2ebLl26mLfeesvYbDZz9OhRc+ONN5px48b59Ov9VPOuDa/136PgnMaGDRtMQkKCycrKco4tWrTI9OjRw9jtdguTVY+qCs7cuXPN0KFDK4xNnTrVjBs3zhhjzB133GEee+yxCtuHDBli3nzzTbdlPVcfffSROXjwoHnnnXcqvMmfy1xLS0tNt27dzJIlS5zbsrKyTPv27c3mzZvdOJszU9XcT/dHz9vnnpOTYyZPnmzS09OdY1988YXp0qWLWbRokenZs6cpLS11bps7d64ZPny4MebE78Cdd95Z4evdcccd5vHHHzfGnPg+vPHGG85tNpvNdO/e3SxdutSdU3LZqeZ+uoLj7XM/cuSI+fjjjyuMzZs3z1x00UU+/Xo/1bx9/bVeGQ5RnUZqaqpiY2MVFRXlHEtMTFROTo727dtnYbLqM2/ePA0aNEjdunXTX//6V+Xm5io1NVWJiYkVHtexY0dt3LhR0onvS8eOHavc7olGjhyppk2bnjR+LnPdt2+f8vLyKmyPiopSkyZNPOp7UdXcy02aNEkXXHCB+vTpo9mzZ8vhcEjy/rlHRERo2rRpaty4sXPs8OHDaty4sVJTUxUfH6/AwF+vd9qxY0dt27ZNJSUlp/y9KC4u1o4dOyrMPSgoSPHx8V4x93LTp0/XhRdeqJ49e+rJJ590Hnr39rnHxMToD3/4g6QTh9N27typzz77TEOGDPHp1/up5l3OV1/rlaHgnEZ2drYiIiIqjEVGRkqSjh8/bkWkatW5c2d1795dX375pT799FOlpaXpoYceqnTe9evXd865qu9LdnZ2jWWvLucy1/L5lv9O/Ha7N/x+hIWFqUuXLho6dKi+++47zZgxQ6+//rrmz58vyffmvmvXLs2ePVt33XVXlT93h8OhnJycKud+/Phx5eTkyBjjtXMPDg5WSkqKBgwYoK+//lpvvfWWli5dqhdeeEFS1T93b5v71q1blZSUpKFDhyo5OVl/+ctfasXrvbJ517bXukTBcYnx4Wshfvjhh7rlllsUGhqqli1basKECVq8eLHMicOXVserMec6V2/9XiUmJmr+/PkaMGCAgoKC1Lt3b40aNUqff/65y1/DW+a+ceNG3Xjjjfq///s/DRs2TNLps5/rdk/x+7k3atRICxYs0IgRIxQcHKyOHTvq9ttvr/Bz94W5JyQkaNOmTfriiy+0e/duTZgwQZLvv94rm3dteq2Xo+CcRlRU1El7Jco/jo6OtiKSWzVv3lzGmErnffz4ceecGzRocFJzz87OrnAoz1s0aNDgrOdaPt/Kfke88XshSc2aNdORI0ck+c7cly9frjFjxmj8+PEaP368pKpf2wEBAapfv36lvxfZ2dmKjo5W/fr15e/v77Vzr0yzZs2UlZUlu93uM3OXJD8/P7Vt29b5jze73V4rXu+/n3f5GWS/5Yuv9d+i4JxGUlKSMjIyKvxybNiwQdHR0YqLi7Mw2bnbvHmznnrqqQpjO3fuVFBQkDp06KBNmzZV2LZx40Z17txZ0onvy++3b9iwQSkpKe4N7QbJyclnPde4uDhFRkZW2J6RkaH09HSv+F4sWrRI77//foWxXbt2qXnz5pJ8Y+4///yz7r33Xj399NO64YYbnONJSUlKS0urcMmHDRs2qEOHDgoODq507uW/FyEhITrvvPMqrD+w2WzaunWrV8x95cqVmj17doXH7tq1S7GxsQoICPD6uS9atEhXX311hTF//xNvd/379/fZ1/up5r1mzRqff62fpIYXNXulUaNGmfvuu8/k5uaaffv2mSuuuMLMmjXL6ljnLD093aSkpJhXX33VlJSUmF27dpkrrrjCTJ061WRlZZlu3bqZd9991xQXF5tVq1aZlJQU88MPPxhjjFm+fLlJSUlxnlL45ptvml69epns7GyLZ3V6vz+T6FznOmPGDDN06FCzf/9+k5OTYyZMmGBuvvlmS+Z2Or+f+9KlS02nTp3M8uXLjc1mMytWrDApKSlm0aJFxhjvn3tpaam54oorzFtvvXXStpKSEjNw4EDzzDPPmIKCArNlyxbTp08f89lnnxljjNm+fbtJSkoyX331lSkpKTELFy40nTp1Mnv27DHGGDN//nzTt29fk5aWZgoKCsxTTz1lBg8ebGw2W43OsSqnmvvGjRtNYmKi+ec//2lsNpvZsGGD6dOnj5k7d64xxvvnnp6ebrp27WpmzZplioqKzNGjR82tt95qrrvuOp9+vZ9q3r7+Wq8MBccF6enp5s477zSdO3c2vXr1Mk899ZQpKyuzOla1+OGHH8yoUaNMSkqK6dmzp5k2bZrzWhY//vijGTFihElKSjKDBg1y/uEvt2DBAnPRRReZpKQkM3LkSPPzzz9bMQWXXXrppSYpKcl07NjRxMfHm6SkJJOUlGQOHDhwTnO12WzmscceMz179jQpKSnm7rvvrnBZAU9wqrnPnz/fXHrppSY5OdlcdNFF5sMPP6zwud489zVr1lSY72//O3DggNmxY4cZPXq0SU5ONn379jVz5syp8PlLly41l112mUlMTDRDhgxxXiul3KxZs0yfPn1McnKy+eMf/+gsAJ7gdHP/6quvzPDhw02nTp1Mnz59zCuvvFLh0hfePHdjjFm/fr0ZNWqUSU5ONueff7659957nafM+/Lr/VTz9uXXemW4mzgAAPA5rMEBAAA+h4IDAAB8DgUHAAD4HAoOAADwORQcAADgcyg4AADA51BwAEiSbrrpJrVv377Cf126dNHNN9+sH374wa3PO2bMGLd9/cqsXr1a7du3148//nhOX+fTTz9V+/btlZ6eXk3JAFQXCg4Ap+7du2vFihVasWKFli9frnnz5ik8PFy33HLLSZdx92SXXXaZVq9eXeX2Ll26aMWKFc7L8wPwPRQcAE5BQUGKiYlRTEyMGjVqpE6dOum5555TZGSkPvjgA6vjuSQnJ0d79uw55WOCg4MVExOjoKCgmgkFoMZRcACcUnBwsFq3bu08DFN+eGfhwoW65JJLNHr0aElScXGxnnjiCfXr109JSUkaOHCgnnvuOZWVlTm/1tatW3XNNdcoOTlZgwYN0ieffFLhuao6dJScnKwXX3zR+XFaWprGjBmjlJQU9evXT4888ojy8/N14MABcSVcFAAABqhJREFU9ezZU8YY3XzzzRo4cGClc/r980yePFnXX3+9vv/+ew0bNkydO3fW0KFDtXz5cufn2Gw2Pfjgg+rWrZu6deumyZMnq6ioqMLXdTgceu211zRkyBB16tRJAwcO1GuvvabyC8Z/+eWX6tixo7Zs2eL8nLVr1yohIUFLlixx7QcCwCUUHACn5HA4dPDgQcXFxVUYf+ONN/Tkk0/queeekyTdf//9WrRokR577DEtWrRIf/7zn/X2229rxowZkk4UhHHjxsnhcGj+/Pl66aWXtHTpUu3YseOM8mRlZWnMmDFq3LixPvroIz3//PNasWKFHnjgAcXGxuq1116TJL344ov6+OOPXf66hw8f1ltvvaUnnnhCn3zyierXr6/77rtPJSUlkqSZM2fqX//6lx5++GF98sknSkhI0Kuvvlrha7z88suaOXOmbrjhBv373//W3XffrZdeeklz586VJA0ZMkT9+/fX1KlTZYyR3W7Xo48+qsGDB2vw4MFn9H0AcGqBVgcA4Lny8vI0e/Zspaen68orr6ywbdCgQerRo4ck/X/79hfSVB/HcfytJbm1zVnGjKiRRVDCKpCSLrKLCsNRQhFUUEFBgTKYXVhKzaLQ/kDSQChpRUhdBOVNRhdddFEXUgYtd2OuYhVTr7ZcWKv2XMgOngqeZ+gDD3s+Lxic32+/c/bdufrwPb9DPB7n4cOHnDlzhk2bNgGwePFiotEoPT09NDU10d/fz6dPn7h8+TKVlZUAXLx4kQ0bNuRU0/3795mYmOD06dMUFxcDcPLkSR49ekQmk6GkpASAkpIS5s2b94+vG4/HuXPnDgsXLgRg7969+P1+YrEYy5cvp7e3l/r6erZv3w7AwYMHefXqFQ8ePAAgnU4TCoXYs2eP0dVyu928efOGUCjEoUOHKCwspK2tjbq6Ou7du8fExATxeJzr16/ndA9E5O8p4IiIob+/n7Vr1xrjL1++sGjRIjo7O03zAKtWrTKOBwcHyWQyrFmzxrTG4/GQSqV4//690alZuXKl8b3dbmfZsmU51fj69WsqKiqMcANQU1NDTU1NTtf5VVlZmRFuACMcJRIJkskkY2NjptoBVq9ebQSc4eFhUqkU1dXVpjXr1q0jFAoxOjpKeXk5LpeL5uZmLl26xI8fPwgEAsyfP39atYvI7xRwRMTg8Xg4f/68MbZarSxYsOCPa+fOnWscj4+PA2Cz2f64Znx8nFQqRUFBAXPmzDGtsVqtOdWYTCZzPuefsFgspnFBQQEAmUyGVCr1xzVT68jeA7/fz6xZs4z5nz9/AjA2NkZ5eTkAXq+X9vZ2ioqK2LJlywz/ExEBBRwRmaK4uBi3253zeXa7HZh8pDVVdmy327FarWQyGb5+/WoKOZ8/fzY6GNlQMdW3b99MG5VLS0uJRqM51zgd2WDz66biqf83ew8CgQBVVVW/XcPlchnHV65cweVykU6nCQaDHDt27N8oW+R/TZuMRWTaKisrKSwsZGBgwDT/8uVL7HY7brebpUuXAhAOh43vR0ZGGB4eNsbZDlAikTDmwuGw0QXJ/tbQ0BDJZNKYe/LkCfv27TMFkOybSzPB6XRSWlpqqh3g2bNnxnFFRQU2m43R0VHcbrfxcTgcWK1W45FaOBzm1q1btLW1cerUKUKhEIODgzNWq4hMUsARkWlzuVx4vV6CwSCPHz8mFotx9+5dbt++zYEDB5g9ezbV1dWUlZVx4cIFIpEIkUiElpYW00bgJUuWYLPZ6OnpIRqN8vz5czo7O3E6ncaaXbt2YbFYOH78OG/fvmVgYICOjg6cTicWiwWHwwHA06dPiUQiMxZ0vF4vfX199PX18e7dO7q7u01vgBUVFbF//366u7vp7e0lFovx4sULjhw5gs/nAyY3Ire2tuL1elm/fj0bN25k8+bNnDhxgnQ6PSN1isgkBRwRmRFnz56lrq6OQCBAbW0tV69epaGhgcbGRmDy8VdXVxffv39n9+7dNDY2sm3bNjwej3ENm81GR0cHHz58YMeOHZw7dw6/32/a7+NwOLhx4waJRIL6+np8Ph9VVVW0t7cDk50Ur9fLzZs3OXz4sKn7Mx1NTU1s3bqV1tZWdu7cydDQEH6/37TG5/Nx9OhRgsEgtbW1NDQ0sGLFCrq6ugC4du0aIyMjNDc3G+e0tLTw8ePH3145F5HpKcjMZB9XRERE5D9AHRwRERHJOwo4IiIikncUcERERCTvKOCIiIhI3lHAERERkbyjgCMiIiJ5RwFHRERE8o4CjoiIiOSdvwDTlwrJl4QMlQAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{}}],"source":["plt.style.use(\"seaborn-white\")\n","\n","# Number of unique customer IDs\n","product_counts = df.groupby(['StockCode']).count()['InvoiceNo'].values\n","\n","fig = plt.figure(figsize=(8,6))\n","plt.yticks(fontsize=14)\n","plt.xticks(fontsize=14)\n","\n","plt.semilogy(sorted(product_counts))\n","plt.ylabel(\"Product counts\", fontsize=16);\n","plt.xlabel(\"Product index\", fontsize=16);\n","\n","plt.tight_layout()"]},{"cell_type":"markdown","metadata":{"id":"I4evk-MwiT-f"},"source":["The left side of the figure corresponds to products that are not very popular (because they aren't purchased very often), while the far right side indicates that some products are extremely popular and have been purchased hundreds of times.\n","\n","## Customer session lengths\n","We define a customer's \"session\" as all the products they purchased in each transaction, in the order in which they were purchased (ordered InvoiceDate). We can then examine statistics regarding the length of these sessions. Below is a boxplot of all customer session lengths."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":441},"id":"0nlaj3ieiILl","outputId":"c235e944-39e9-4eba-9ba4-242765988731","executionInfo":{"status":"ok","timestamp":1639397731129,"user_tz":-330,"elapsed":15,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAjgAAAGoCAYAAABL+58oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxVdeL/8fcVBLfUVFxynXG6mAIBOmaDSe4tOl8tTcs1ncklHdNyHdtM06yZrMxmtMZ9adVv6dRUU6l9xxb3cEMrQUwQTUBQQODz+8PfPXMvXAQVtD69no+HD7nnc85nO+fe8+ace7kuY4wRAACARSpc7Q4AAACUNQIOAACwDgEHAABYh4ADAACsE1jaFbOzsxUXF6eQkBAFBASUZ58AAMAvUH5+vlJTUxUWFqZKlSpdVl2lDjhxcXEaMGDAZTUGAABQkpUrV6pNmzaXVUepA05ISIjTaP369S+rUQAAgMKSk5M1YMAAJ3NcjlIHHM9tqfr166tRo0aX3TAAAIA/ZfFWGN5kDAAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWCbySja1YsUIJCQnlUnd6erokqUaNGuVS/9XStGlTDRw48Gp3AwCAn5UrGnASEhJ06NvvVa1GSJnXfTrtpCTp7LmAMq/7aslMT73aXQAA4GfpigYcSapWI0Stb+1f5vVu+2yNJJVL3VeLZ0wAAODi8B4cAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOpcUcD7//HN9/vnnZd0XwFo8ZwDgygq8lI02btwoSWrfvn2ZdgawFc8ZALiyuEUFAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOoFXuwPAL0FmZqaSkpI0adIkBQYGKiUlRfn5+crPz3fWqVmzptLS0lS3bl1VqFBBKSkpql+/voKDg5Wdna2UlBSFhIQoLS1NkmSMkcvlUt26dZWfn6/k5GQZYyRJAQEBcrlcysvLU6NGjTRy5EgtWrRIx44dc9qtU6eOfvzxRxUUFKhChQoKCAiQMUb5+fm65pprlJGRoVq1aqlq1ao6fvy4qlWrppMnT6pixYqaMGGCXn/9dSUnJ2vcuHF66623lJubq9TUVHXt2lXvvfeeM65+/fpp586dGjNmjCRp7ty5OnLkiCQpMDBQAQEBuvfee7V06VLVrl1bJ06cUEhIiFJTUxUUFKTu3bvrvffeU+3atZWZman69evrj3/8oxYvXqzs7GydPHnSafP+++9X8+bNNWvWLE2fPl1NmjSRJKWlpWnevHnKycnR8ePHnXmqUKGCxo8frzfffFPZ2dlKTU2VJF133XXq16+fXnjhBYWEhCgoKEgPPfSQNm/erDfeeEP9+vVT+/btnTpTU1MVEhKigIAA5eXl6eTJk3r00UdVvXp1zZ8/X2PGjFHNmjWdvsyZM0c//PCDGjZsqJEjR2r58uUaNGiQFi1apJSUFJ9tBw8erIULFyolJUX9+/fXkiVL1LhxY02aNEnp6emaOXOm6tWrpz/+8Y9avnx5kbbmzZvnzNPw4cP1j3/8Q8OGDdNrr72mevXqqUePHlqwYIEaNGigChUqKDU11ZljT32SNG/ePEnSQw895ByrnrF5yj3tPProoz5zP3/+fGd8ycnJatCggfr16+dT57p16zR48GAtW7ZMgwYNcuZk+fLl6tWrl1588UVnn3rq9KzvKe/Xr5+WLl2qyZMnq1WrVkXW856bhIQEzZo1S3/605+0bt06nzLvfnuWe/bb0aNHdd1116ly5crq06eP06/C+9p73N7z6G8df/3zt713//xJS0vTX/7yFx07dkwNGjTQww8/XKTNwuMbNGiQFi9eLEkaNmyY/vGPfyg/P18BAQHOvi6urQvV6z0mTx/8jbM8BTzxxBNPlGbFjIwMLVu2TEOGDNGuXbskSR06dLioxjZv3qzMM7m6rlnYRXe0JMcOx0lSudR9tRw7HKdqVYIuep7x07N8+XIZY5SZmamMjAzl5+c7J1mP7OxsSVJWVpYyMzMlnQ9GaWlpzuOsrCwnoBQUFCg/P18ZGRlOuYcxRgUFBZLOP3cPHDigI0eO+LR75swZ52fP+p5tcnJyJElnz551+nv27FlJUkFBgXbu3KnU1FTl5eVp586dSklJUUZGhvLy8hQfH+/Tlz179ujHH39UTk6O9u3b57x+eOrKz8/Xrl27ZIzRmTNnnL5JUn5+vlPf2bNnlZ+fr/T0dB04cEAJCQk6ffq0T5u7du3S/v37derUKR04cEBdunSRJK1evVrbt293xuIZa35+vnbs2KGUlBSdPn3amdu0tDTt2LHDGf+pU6eUm5ur9evXO2PKycnxqTMjI0Pp6elOnw4cOKDU1FRt27ZNOTk5ioyMdPqye/dun31z+PBhZx8V3nb//v3O8p07dzrb5ebm6v3339epU6ecOTl8+HCRtrZv3+70aceOHcrNzXX+T09P144dO1RQUKDTp087Yylc3759+7R9+3ZnHiIjI7V69WpnbJ5y77F7z/22bdt8jkHP/GZnZys/P187d+5UcnKy9u/f78yF9/87duzQmTNnnHo9dXrW95R75mfHjh3q0aNHkfW852b27Nk6deqU07Z3mXe/Pcu999vp06d16tQp5xjxt6+9x+09j/7W8dc/f9t798+f1atXa8eOHc4c5+bmFmmz8Pg8zyXPcyYhIUFpaWk++7q4ti5Ur/eYPH3wN87CvLNG9erVLzjeknCLCihncXFxTnC4Wo4ePVqm9WVlZfn9uTjGGG3cuFEbN24stvxiFDceY4x++OEHZ53ExESlpaVp06ZNxdZVXP89Icvj3//+t8/jTz75pMQ+bty4UcYYbdq0SWlpaUpLS9Nnn31WZD1jjM+Yjh49qk2bNhVZ7u2zzz5zxupdj3dbmzdv9tkmLy/P5//CP/vr16ZNm3z226ZNm5SQkKDNmzc75YXn13vuPesVHof3/GZlZTnr+Pvfs4+OHj2quLg4nzq9y73r++KLL4qs55mbhIQEZ+48bXvKJPn0e9OmTUpMTCyy37zH4L2/POv7a9t7He85LG7f+Ssrjr/jfOPGjT5t+htf4eOu8Pb+2iw8P8XV63neX+xYysol3aJKT09XWlqaZs2adVHbJSQkyBVQ6VKa/EXKzc5SQsLJi55n/LQUvqLxS+XvqlV5W7BggUJDQ31uBV5JnnaNMVq3bp3PlbWSFBc8CtddmHdbJdVR2n5477e8vDy98sorzrLC5R6euS/rfT5//vxS1fn3v/+9yDLP3Ozbt6/YsqFDh2rt2rU+VzcXLFhQ4n7zzLVn/cJ99N4XxhifOSzcB2NMsWVDhw712/7atWuL7O+8vDy5XK4Lju9C8vPz/bZZeH4uVK+/47SksZQVruAA5exqX735qbjS4UY6/xvpli1brkrb3vLy8vSf//xHW7ZsuaJtlcW4/Z1ojx496nNC98cz92URsrxlZWWVqs68vDy/J/z//Oc/Ple+CpdJ8ul3Xl7eRV0B9axfuG3v0FLcOt77rriy4hR3bHm36W98F2KM8dtm4fm5UL3+wlpJYykrl3QFp0aNGqpRo4b+/Oc/X9R2s2bNUvKJzJJXhCQpqFJV1a9T76LnGT8tI0aMKHK745fI5XJd8aDRsGFDhYaG6tNPP72qIScwMFC/+93vZIwp8dZWWbZVFuMuvN9cLpeuu+46paSkOFcI/LXhmftNmzaVacipWrWqcnJySqwzMPD86c17Pc/c7Nu3r0jI8ZRJ0s033+z0OzAwUPXq1St1yPGs75kfD+8rKcWt473vCs+bd//8ufnmm/0eW579U9z4LsTlcvlts/D8XKhe73GXdixlhSs4QDkbO3bs1e7CT0JAQIACAgKuaJujR49W7969r3i7Hp52XS6XevXqpd69e6tChdK97HpO0CXVXZh3WyXVUdp+eLcVGBioUaNGOSeuwMBAv+145t6zXlkZM2ZMqeocMWJEkfU8czN69Ogi63vKJPn02+VyafTo0SXuN88ceNYv3Lb3PLpcLp85LNwHf/Pm3T9//O3vwm36G9+FBAQE+G2z8PxcqN6AgIAi/SppLGWFgAOUs7CwsFKf1MpLw4YNy7S+qlWr+v25OC6XS7GxsYqNjS22/GIUNx7P1QXPOk2aNFHNmjUv+EnE4vpfpUoVn8edO3f2edypU6cS+xgbGyuXy6UOHTqoZs2aqlmzpm699dYi67lcLp8xNWzYUB06dCiy3Nutt97qjNW7Hu+2brnlFp9tPCca7xNOcSHIuz7v/dahQwc1bdpUt9xyi1NeeH69596zXuFxeM9v1apVnXX8/e/ZRw0bNlRYWJhPnd7l3vW1a9euyHqeuWnatKkzd562PWWSfPrdoUMHNWnSpMh+8x6D9/7yrO+vbe91vOewuH3nr6w4/o7z2NhYnzb9ja/wcVd4e39tFp6f4ur1PO8vdixlhYADXAGeF9MGDRqocePGCgoKKvIbuOcJX7duXdWvX18ul0sNGjRQs2bNnMd169ZVUFCQgoKCVLFiRQUFBalRo0Zq0KCBT0jw/q2pUaNGGjVqlJo2berTbp06dZzgVaFCBVWsWFGBgYFyuVzOxzNr1aqlxo0bKzg4WLVr15YkVaxYUWPGjFGzZs1UqVIljRkzRs2bN1fjxo1VqVIl9ezZ02dc/fr1k9vtdn4zbdy4sVMWGBio4OBgDRkyRC6XS3Xq1JEkhYSESJKCgoKc+mrXrq3g4GA1bdpUo0aNUvPmzdWwYUOfNocOHarRo0ercuXKPr+l9+7dW82bN1ejRo2cuatYsaKCg4M1duxYpy7P3DZr1kxjx45VpUqV1LhxYzVv3ly9evXSPffc44zJu87g4GA1atRITZs2dfrkuYLhGbt3Xzwv9J5943a7nX1UeNvRo0c7yz1vymzcuLFzJaJSpUrOnPhry3ueRowYocqVK2vEiBHOdp4rHdddd50zlsL1eerxzIOn7sLl3mP37oP3+IKDg535DQ4OVnBwsMaMGeOM1bOu9/9jxozx2afec+NdPnToULlcLueqaeH1vOfGc5x42i58RaHwvvPsN+n887l58+YaO3as0y9/63uPwTNP/tbx1z9/25ekd+/eatasmTPH/tr0V79n344ePVrNmzdXs2bNfPZ1cW1dqF7vMV1onOXJZUp5gzYpKUmdO3fWv//9by1dulSSLvk9OK1v7X/xPS3Bts/WSFK51H21bPtsjerXqcZ7cCzg+SQc+xIAiuedNRo1anRZdXEFBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUCL2Wj2NjYsu4HYDWeMwBwZV1SwGnfvn1Z9wOwGs8ZALiyuEUFAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYJ/BKN5iZnqptn60p83pPpx2XpHKp+2rJTE+V6lS72t0AAOBn54oGnKZNm5Zb3ZUr5kuSatSwKBDUqVaucwYAgK2uaMAZOHDglWwOAAD8QvEeHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8ABAADWIeAAAADrEHAAAIB1CDgAAMA6BBwAAGAdAg4AALAOAQcAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHUIOAAAwDoEHAAAYB0CDgAAsA4BBwAAWIeAAwAArEPAAQAA1iHgAAAA6xBwAACAdQg4AADAOgQcAABgHQIOAACwTmBpV8zPz5ckJScnl1tnAADAL5cnY3gyx+UodcBJTU2VJA0YMOCyGwUAAChOamqqmjZtell1uIwxpjQrZmdnKy4uTiEhIQoICLisRgEAAArLz89XamqqwsLCVKlSpcuqq9QBBwAA4OeCNxkDAADrEHAAAIB1CDgAAMA6pQo4x44d08iRI3XTTTcpNjZWM2bMUG5ubnn3zToHDhxQjx491KlTJ5/lX331le655x5FR0frtttu0+rVq33KV65cqdtvv13R0dG65557tHXr1ivZ7Z+do0ePauzYsWrXrp3atWuncePGKSUlRdL5fTB48GC1adNGnTt31ksvvSTvt6F98MEH+p//+R9FRUXp97//vT788MOrNYyftJ07d2rgwIGKjo5WTEyMJkyY4HzSkuO5fDz99NMKDQ11HjPPZSc0NFRhYWEKDw93/j3++OOSmOey9tprr6lDhw6KjIzUfffdp0OHDkkqp9dmUwp33XWXmTJliklPTzdJSUmmV69e5tlnny3Npvj/NmzYYNq3b29Gjx5tOnbs6Cw/fvy4iYqKMitXrjRnz54127ZtM9HR0Wbjxo3GGGM+/fRTEx0dbb7++muTnZ1tVq9ebaKjo01qaurVGspPXo8ePczDDz9sTp8+bU6cOGEGDx5sHnjgAXP27FkTGxtr/vrXv5rMzEwTHx9vYmNjzapVq4wxxuzbt8+EhYWZjz76yGRnZ5uPP/7YhIeHmwMHDlzlEf20pKWlmaioKLNkyRKTm5trTpw4YQYOHGhGjRrF8VxO9u7da9q2bWvcbrcxhteNsuZ2u80XX3xRZDnzXLZWr15tunbtag4cOGAyMzPNX/7yF/Pwww+X22tziQFn9+7dpkWLFubkyZPOsvfff9/89re/Nfn5+Zc53F+ON9980xw9etQsX77cJ+C8+uqrpkePHj7rPvnkk2bUqFHGGGMeeOAB89RTT/mU33nnnWbx4sXl3uefo/T0dDNlyhSTnJzsLFu/fr2Jiooy77//vmnbtq05d+6cU/bqq6+a3//+98aY8/M+YsQIn/oeeOABM3PmzCvT+Z+J48ePm7feestn2dKlS03Hjh05nstBfn6+6du3r3nllVecgMM8l63iAg7zXLY6depk1q9fX2R5eb02l3iLas+ePWrQoIFq1arlLGvVqpXS09OVmJh4CReofpn69Omj6667rsjyPXv2qFWrVj7LWrZsqW+++cYpb9myZbHl8FW9enXNnj1b9erVc5YdO3ZM9erV0549e+R2uxUY+N+/b9myZUvFx8crJyenxH2B80JCQnT33XdLkowx+vbbb7V27VrdeeedHM/lYM2aNapUqZJ69OjhLGOey97SpUvVuXNntW7dWpMmTVJGRgbzXIZSUlKUlJSkM2fOqGfPnvrtb3+rESNGKDk5udxem0sMOGlpaapevbrPsho1akiSTp06VerBwT9/81uzZk1nboub/7S0tCvWx5+z7777Tq+88opGjx5d7FwXFBQoPT292LnmOPdv//79CgsLU48ePRQeHq6HHnqI47mMnThxQi+//LKeeOIJn+XMc9m68cYb1aZNG23YsEHvvPOODhw4oMcee4x5LkOer2BYv369Fi5cqPfff1/nzp3ThAkTyu21uVRvMjb8LcByxfyWj2+++UYDBw7U/fffr549e0oqea7ZF6XXokULxcXFaf369fr+++81YcIEScxhWZo9e7b69u2rX//610XKmOey88Ybb2jYsGGqVKmSmjZtqgkTJuiDDz6QOf82jqvdPSt45nH48OFq0KCB6tSpowkTJmjbtm3Ky8srl9fmEgNOrVq1iqRRz+PatWtfdIPwde211xaZ31OnTjlze+211xZJqWlpaT63DFHU5s2bNXToUI0ZM0ZjxoyRVPyxHBAQoJo1a/rdF2lpaRznF+ByudS8eXPnhJCfn8/xXEa2bNmib775RqNGjSpSxutG+WrUqJGMMX5fM5jnS1OnTh1J56/MeDRs2FDS+e+dKo/X5hIDTlhYmFJSUpyPgErS7t27Vbt2bTVu3LikzVGC8PBwxcXF+Sz75ptvdOONN0o6P/+Fy3fv3q3IyMgr1sefm127dmn8+PF65plndN999znLw8LCdODAAZ8/cbB7927dcMMNCgoK8jvX3vsC573//vu66667fJZVqHD+pSQ2NpbjuYy8++67SklJUYcOHXTTTTc5c37TTTfJ7XYzz2Vk7969mjNnjs+yb7/9VhUrVtQNN9zAPJeR+vXr65prrtHevXudZUlJSZKku+66q3xem0vzzud+/fqZiRMnmoyMDJOYmGjuuOMOM3/+/NJsikIKf4rq5MmTpnXr1mbFihUmOzvbfPHFFyYyMtJ89dVXxhhjNm/ebCIjI52PIS5evNjcdNNNJi0t7WoN4Sft3Llz5o477jBLliwpUpaTk2M6depknnvuOZOVlWX27dtnYmJizNq1a40xxhw8eNCEhYWZDz/80OTk5Jh//vOfJiIiwhw+fPhKD+MnLTk52URHR5v58+ebs2fPmhMnTpjhw4eb/v37czyXobS0NHPs2DHn344dO4zb7TbHjh0zSUlJzHMZSU5ONpGRkebvf/+7ycnJMd9995254447zJNPPsnxXMaee+45Exsbaw4dOmTS0tLMsGHDzAMPPFBur82lCjjJyclmxIgR5sYbbzQ33XSTmTNnjsnLy7v80f6CdOvWzYSFhZmWLVsat9ttwsLCTFhYmElKSjJbt241vXv3NmFhYaZz587OTvV4/fXXTceOHU1YWJjp06eP2bVr11UaxU/f119/7TO/3v+SkpLMoUOHzIABA0x4eLhp3769WbRokc/2H330kbnttttMq1atzJ133un8vQv42rlzp+nXr58JDw83N998sxk/frzz0XyO5/Jx5MgR52PixjDPZemrr74y/fr1M5GRkaZt27Zm9uzZJjs72xjDPJel3Nxc89RTT5m2bduaG2+80YwbN86cOnXKGGPK5bWZbxMHAADW4buoAACAdQg4AADAOgQcAABgHQIOAACwDgEHAABYh4ADAACsQ8DBBW3fvl0PPvigOnTooLCwMLVt21bB52UAABUuSURBVFYjR47U1q1br2g/vvzyS4WGhpZ7u1eqndJ45513FBoaqh9//PGStp85c6b69u3r89dB//rXv+qf//yn87hnz57Kycm5rH4uXLhQ0dHRP5m/+JyUlKTQ0FD97//+79Xuyk/GlClTfL6N/Kdm0KBBGjp0qHJzc9W7d28988wzV7tLsAABB8XaunWrhgwZonr16mnhwoX66KOP9PLLLysvL0/3339/iV9VX5aioqL0+eef/2ROouVh4cKFmjJlSpnUtWHDBq1du1bPP/+8goKCnOXx8fFyu92SpNzcXBUUFCg4OPiS28nOzta8efN02223acOGDZfd75+qnTt3qlOnTle7G9YLCgrSvHnztGbNGn388cdXuzv4mSPgoFjLly9Xs2bN9Nhjj6lFixZq0KCBfvvb32rBggVq1arVFQ04QUFBCgkJUcWKFa9Ym1fazp07y6SenJwczZkzRwMHDlSjRo18yr7//ns1bdpUknT48GE1a9bsstrKzMxUfn6+2rRpU6Qtm5TVvkHJmjZtqv79++vpp5/WuXPnrnZ38DNGwEGxcnJynBOYt6CgIK1Zs8bniyxTUlI0fvx4dejQQTfeeKP69++vHTt2OOUFBQV68cUX1blzZ4WHh6t9+/b685//rMzMzFKV+7t1tHLlSt12220KCwtTu3btNHHiRJ04ccIpHzRokB555BGtW7dO3bp104033qg+ffpcdDB766231KtXL0VGRqp9+/aaO3euz22f0rSTlpamcePGKSoqSu3atdMLL7ygxYsXq2XLlk4d//73v7V27VqFhobqyy+/dLY9duyYhgwZooiICMXExOj111+/YH/ffvttnTp1SkOHDvVZnpWVpaCgICckHjx4UL/5zW8uWNe+ffs0fPhwRUVFKSIiQvfcc482b94s6fw+iYmJkSRNnTpVoaGhfut46aWX1K5dO3399dfq0aOHwsLC1L17d5/f0KdMmaJ7771Xf/vb3xQVFaU333yzxPY9lixZoltuuUUREREaNGiQEhISfMqnTJmirl27+izbsGGDQkNDnS/7k6RVq1ape/fuioiIUM+ePZ1bXC+99JJmz56to0ePKjQ0VC+99JIkacWKFbr99tsVERGhdu3aady4cTp+/Hixczlo0CD96U9/0sqVK3XLLbcoPDxc/fv317fffuusExoaqgULFvhsN3z4cA0aNMhnncWLF+u+++5TeHi4cywW139vW7ZscfbB7bff7vMczcjI0PTp03XzzTcrLCxMnTt31vz58+X9x+63bNmi/v37Kzo6WtHR0RowYIC2b9/ulOfm5mru3Lnq3r27wsPDddttt+mtt97y6cP+/fvVt29fhYeHq3Pnznr77beL9HP48OFKTk7Wu+++W+x8AiUq+2+bgC2WL19u3G63GTRokPnss8/M2bNn/a6Xk5Njunfvbnr06GG+/PJLEx8fbyZNmmQiIyNNYmKiMcaYNWvWmMjISPPJJ5+Yo0ePmq+++sp069bNTJs2rVTlX3zxhXG73ebrr782xhizatUqc8MNN5gVK1aYw4cPmy1btphu3bqZu+66yxQUFBhjjBk4cKDp2rWrGTt2rNm/f7+Ji4szt99+u7nzzjuLHXPhdt555x3jdrvNSy+9ZL7//nvz4Ycfmptuusk8/vjjzjalaWfcuHGmTZs25qOPPjIHDx40jzzyiOnatau54YYbjDHGnDp1ynTt2tWMGzfOHD9+3OTk5Ji3337buN1uM3ToULNx40bz3XffmYcffti0aNHCJCUlFTuGP/zhD+a+++5zHn/55ZemY8eOJiYmxkRGRpqOHTuajh07mtatW5t27dqZjh07+v1ywJSUFNOmTRszevRos3fvXnPo0CEzbdo007JlS7N3716Tk5Nj9u/fb9xut1myZIk5fvy43/68+OKLplWrVmbw4MFm69at5sCBA2bUqFEmPDzcHDt2zBhjzOTJk03Hjh3N6NGjTWJiojl9+nSJ7RtjzMaNG43b7TZ//etfzffff28+/vhj07NnT+N2u826deucurt06eLTp/Xr1xu3222OHDlijDHmrbfeMuHh4eadd94xCQkJZvny5SY0NNR8+umnJjMz00yaNMl06NDBHD9+3GRmZprNmzebFi1amLVr15qkpCSza9cu069fPzNkyJBi98vAgQNN+/btzUMPPWTi4+PNtm3bTLdu3UyPHj2cY9btdpuXX37ZZ7thw4aZgQMHOo/dbrfp0qWLef31101SUpIpKCi4YP89cxATE2NGjhxp4uLizJ49e0yPHj185uWRRx4xsbGxZtu2bebo0aPmgw8+MOHh4WbVqlXGmPNf/hkZGWmefvppc/jwYXPo0CEzffp007p1a5OVlWWMMWbKlCmmTZs25t133zXff/+9ee2110yLFi3Mhg0bjDHnXytuvfVWc9ddd5m4uDizb98+M2LECNOuXbsic3f33Xeb0aNHFzufQEkIOChWfn6+ee6550xYWJhxu92mVatW5t577zWvvvqqzwlxw4YNxu12OycdY86/kMXExJg5c+YYY4x5/PHHiwSLxMRE8+2335aqvHDw6Natm5kwYYLP+ps3bzZut9vs2LHDGHP+hNKmTRvnxdcYYxYuXGjcbrc5c+aM3zEXbue2224zo0aN8lln+fLlpmXLliY9Pb1U7WRlZZlWrVqZBQsWOOV5eXk+AccYY7p3724mT57sPPYEnPfee89Ztm/fPuN2u82//vUvv/03xpioqCjzl7/8pcjyZcuWmYULFzqPhw0bZhISEoqt55VXXjHh4eHm9OnTzrL8/HzToUMH8+ijjxpjjDl+/Lhxu93m7bffLraeF1980bjdbvP55587y5KTk01oaKhZsWKFMeb8CTg0NNQJPKVtf8KECaZ79+4+7a1evfqiA84dd9xhpk6d6rPO3LlzzRtvvGGMMebRRx81HTt2dMoWLlxooqOjfb50OCUlxezbt6/YeRg4cGCR8bz77rvG7Xab+Ph4Y0zpA87QoUN91imp/5MnTzYtWrRwvhDVGGOWLl1q3G6382WHycnJRYLzwIEDzYMPPmiMMWbXrl3G7Xb7fJlkTk6O2bZtm8nJyTHJycmmRYsWZvHixT51jB071vTu3dsYU/Q5aowxGRkZJiwsrEjAefrpp03btm0NcKm4RYViVahQQQ8//LA2bdqkOXPm6M4771RiYqJzCTouLk6StGvXLtWoUUM33HCDs21QUJCio6O1b98+SVJsbKwOHTqkP/zhD3rvvfd04sQJNW7cWL/+9a9LVe4tMzNThw8fVlRUlM/yiIgISdKePXucZc2bN1eVKlWcx7Vq1ZJ0/nJ8STIzM/Xdd9+pXbt2Psvbtm2rvLw8xcfHl6qdY8eO6dy5c86beyUpICBAv/vd70rsgySFhYUVqTcrK8vvumfOnFFWVpbq1KlTpGzv3r0+dSUlJalx48bFthsXF6ff/OY3qlatmrOsQoUKatWqlfbu3VuqvnvzfoN4vXr1VLt2bR09etRZVqtWLdWvX/+i2j906JDPcSdJkZGRF9Wv7OxsHTp0SK1atfJZPnHiRPXt29fvNr/73e+Um5urAQMG6M0339TRo0dVt25dtWjR4oJtFR6P5xblDz/8cFF99mx3Mf2vU6eO6tWr5zwufCy5XC699tpr6tatm1q3bq2oqCht27ZN6enpTt8bNmyohx56SIsWLdK+fftUsWJFRUdHKygoSHFxcSooKPD7fImPj5cxRocOHZIkn312zTXXqHnz5kXGGBISorS0NJ/bwcDFCLzaHcBP37XXXqvevXurd+/eKigo0CeffKIpU6Zo1qxZWr16tTIzM5WRkVEkcOTm5upXv/qVJKljx4567bXXtGzZMk2fPl05OTmKiYnRjBkz1LBhwxLLvXnel+N9opCkqlWrSvI9+VeqVMlnHZfLJUk+7ysojqedZ599Vs8//7yz3LOt9/t9LtROWlqapPMv5N5q1KhRYh8K111S/0+fPi2p6NxI5wPO1KlTJZ1/X0/dunWd+vzJzMz0W0/VqlWduSktl8tVpK4qVao4/fXUe7HtZ2VlFZl776BZGp4TeOXKlUu9TatWrbRy5Ur94x//0Jw5c5SZmamoqCg98cQTFww5xR2zpQnc/raTSt//Cx2jxhgNHz5caWlpmjp1qtxutypWrKhp06Y561epUkWrV6/WokWLtGLFCj333HNq2LChJk6cqNtvv93ZJ/379/c5rvLy8nTu3DmdOnVKWVlZcrlcRT6552+fVa9eXdL5Y7p27dolzglQGAEHxcrJyZHL5fL5mHGFChXUpUsX3X333c4bQa+55hrVrFnT75tfAwP/e4jFxMQoJiZGOTk52rRpk2bPnq3x48frjTfeKFW5h+ck4X1y9H7s76R4KTz1jBw50u/fECnti67nxbzw35vxBJ+y5AlR3gFk4sSJ+vjjj3X27FnFxsZKOv+m7ry8PEVFRalLly569tln/dblfYXF4/Tp00XCWkmMMcrOzvY5yWZlZTknseLGUlL7lStXVnZ2tk954bDgcrmKBMIzZ844P1977bVyuVwXHdoiIiI0b948nTt3Tl999ZXmzp2rP/7xj9q4caMqVPB/cfzs2bM+jz1h3Hse/PXV+3lU2KX231t8fLzi4+P13HPP6Y477nCWnz592ieI16tXT9OnT9f06dO1f/9+vfLKK5owYYKuv/56Z5/Mnz/f75XB6tWrq0qVKjLGKCcnxyfk+Asxnv14scca4MEtKvh14sQJtWnTRkuWLPFbnpiY6FzujoiIUHp6uipWrKimTZs6/6Tzl5kl6fPPP3cuTwcHB6tr164aPHiwDh48WKpyb9WqVVOzZs18Pr0h/fejvOHh4Zc5+v+28+tf/1rHjh3zGVdISIgCAgJKHaSaNGkil8vlc+ssNzdXn332WZF1S3Nl6UKqVKmiqlWr6uTJk86ySZMmae7cubr11lu1bt06rVu3Tn369NGECRO0bt06TZo0yW9dYWFhOnjwoE9gyMvLU1xc3CXNsfcn4I4dO6aTJ086V/gutf1f/epXzq1Sj//85z8+j6tWrVok9Ozatcv5OSgoSNdff32R42nmzJmaN2+e89h732zfvl27d++WJFWsWFExMTEaM2aMjh8/7lxR8Sc+Pt6nL56+e27FVqtWzaf8zJkzzvOiOKXt/4V4Po597bXXOsv279/v3FqSpISEBH366adOeYsWLTRjxgwVFBTo22+/VVhYmCpUqKAff/zR5/lSqVIl1axZU4GBgc7+9v6EYUpKis8nyTxSU1NVo0YNn1+wgItBwIFfderU0b333qsXXnhBzz//vL755hv98MMP2r17t2bMmKFPPvlEo0ePliR17txZTZo00YQJE7R9+3YlJSXp7bffVq9evZyPqr7zzjsaN26cvvjiCx07dky7d+/We++9p7Zt25aqvLA//OEP+te//qUlS5YoMTFRn3/+uWbOnKm2bduWWcCRzn9cdd26dVq6dKkSEhL0zTffaPz48RoyZEip3xtQvXp1xcTEaNmyZdq8ebMOHTqkqVOnFrklU6NGDe3du1f79u3zuf11sVq3bq1t27Y5j0NCQpSSkqKbb77ZOel8//336tKlixPY/OnTp4+qVKmihx9+2DnZTZ06VRkZGRowYMBF9SkgIEB///vftXXrVh08eFAzZsxQ5cqV1a1bt2K3KU37PXr0UGJiol588UUdPnxYH374odavX+9TT1hYmNLT07Vs2TIdOXJEa9as8Qk4knT//ffrX//6l9asWaOkpCStWbNGq1atct6zVKNGDaWmpmrr1q06cuSIPv30Uz344IP69NNP9cMPP2j//v164403dP311/uEhMKqVaum6dOnKz4+Xtu3b9fLL7+sqKgo54pHq1at9MEHH2jnzp06ePCgpk6d6rxX5kJK6n9JfvWrX+maa67RqlWrlJiYqM2bN2vatGnq1KmTEhMTlZCQoMTERI0ZM0YrVqzQkSNHlJiYqEWLFik4OFgRERGqW7euevbsqblz5+rjjz9WUlKS/u///k+DBw/WU089JUlq166d6tSpo7lz52rv3r3au3evpk2b5neM27dvV5s2bUrVf8AfblGhWNOmTdMNN9ygt99+W2+++abS09NVrVo1RURE6LXXXlP79u0lnb/ismTJEj3zzDMaMWKEzpw5oyZNmmjy5MnOmxxnzJihOXPm6JFHHlFaWppq1aqlmJgYPfLII6UqL6xv3746d+6cli5dqmeffVY1atRQ586dNXHixDKdgz59+sgYo8WLF+vZZ59VpUqVFBMToyVLllzUb5azZs3StGnT9OCDD+raa6/V0KFD1bhxYy1dutRZZ9iwYXrsscd07733avbs2Zfc51tvvVWzZ89WWlqaatasKen8ycLzd3Hy8/P1ww8/OFfZilO7dm0tXbpUzzzzjPr37y9jjMLDw7V48WK/bwotybhx4zRz5kwdOnRIDRs21AsvvHDBW1Slab9bt26aMGGCli1bpldffVURERHOV1R49OjRwwkT8+bNU6dOnTRhwgSNHDnSWeeuu+5SRkaGFi1apFmzZqlJkyaaOXOmunTpIknq3bu3PvzwQw0dOlT33nuvJk2apPz8fD355JM6ceKEatSoodatWxf5GzaFXX/99br55ps1YsQInThxQhERET77+rHHHtOf//xnDRkyRLVr19aoUaNUuXJlv7fqvJXU/5JUrVpVc+fO1Zw5c9SzZ0+1aNFCTz31lM6ePasHH3xQ/fv315YtW/Tkk086z7mKFSsqNDRUf/vb39SgQQNJ568aPf/885oxY4ZOnDihWrVqqWfPnho3bpyk8+8DWrBggZ588kndc889qlu3rkaPHq3g4GCf24Y//vij9uzZ4wQj4FK4zOVeEwdQopycHJ09e9YJHJI0fvx4HTp0SO+9916ZtpWdna0uXbqob9++zonlanrppZf0yiuvXNInr2wyaNAgBQQEFHvbF//17LPP6v3339cHH3zALSpcMm5RAVfApEmT1KtXL23ZskVJSUl655139OGHH+ruu+8u87YqVaqkyZMna/ny5Rf98WPgajty5IhWrVqlyZMnE25wWbhFBVwBM2fO1DPPPKOJEycqIyND1113ncaPH+/zJ/jLUs+ePbVr1y499NBDWrlypdXf4QV75Obmaty4cerfv7+6d+9+tbuDnzluUQEAAOtwiwoAAFiHgAMAAKxDwAEAANYh4AAAAOsQcAAAgHX+H4PoEHJndZDKAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{}}],"source":["session_lengths = df.groupby(\"CustomerID\").count()['InvoiceNo'].values\n","\n","fig = plt.figure(figsize=(8,6))\n","plt.xticks(fontsize=14)\n","\n","ax = sns.boxplot(x=session_lengths, color=cldr_colors[2])\n","\n","for patch in ax.artists:\n"," r, g, b, a = patch.get_facecolor()\n"," patch.set_facecolor((r, g, b, .7))\n"," \n","plt.xlim(0,600)\n","plt.xlabel(\"Session length (# of products purchased)\", fontsize=16);\n","\n","plt.tight_layout()\n","plt.savefig(\"session_lengths.png\", transparent=True, dpi=150)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"a6aVEx4uiIIi","outputId":"6d7db7d5-7bcb-4d8a-93ef-eab20c357406","executionInfo":{"status":"ok","timestamp":1639397731131,"user_tz":-330,"elapsed":13,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Minimum session length: \t 3\n","Maximum session length: \t 7983\n","Mean session length: \t \t 96.03967879074162\n","Median session length: \t \t 44.0\n","Total number of purchases: \t 406632\n"]}],"source":["print(\"Minimum session length: \\t\", min(session_lengths))\n","print(\"Maximum session length: \\t\", max(session_lengths))\n","print(\"Mean session length: \\t \\t\", np.mean(session_lengths))\n","print(\"Median session length: \\t \\t\", np.median(session_lengths))\n","print(\"Total number of purchases: \\t\", np.sum(session_lengths))"]},{"cell_type":"markdown","source":["The median customer purchased 44 products over the course of the dataset, while the average customer purchased 96 products."],"metadata":{"id":"0-wyj-SCtGNr"}},{"cell_type":"markdown","metadata":{"id":"pJ7qK-MCXBwx"},"source":["## Sessionization"]},{"cell_type":"markdown","source":["The effectiveness of an algorithm is measured by its ability to predict items withheld from the session. There are a variety of withholding strategies:\n","\n","- withholding the last element of each session\n","- iteratively revealing each interaction in a session\n","- in cases where each user has multiple sessions, withholding the entire final session\n","\n","In this tutorial, we have employed withholding the last element of the session.\n","\n"],"metadata":{"id":"YYeVyqfsvRtt"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"V_tNkOldV7t6"},"outputs":[],"source":["def construct_session_sequences(df, sessionID, itemID, save_filename):\n"," \"\"\"\n"," Given a dataset in pandas df format, construct a list of lists where each sublist\n"," represents the interactions relevant to a specific session, for each sessionID. \n"," These sublists are composed of a series of itemIDs (str) and are the core training \n"," data used in the Word2Vec algorithm. \n"," This is performed by first grouping over the SessionID column, then casting to list\n"," each group's series of values in the ItemID column. \n"," INPUTS\n"," ------------\n"," df: pandas dataframe\n"," sessionID: str column name in the df that represents invididual sessions\n"," itemID: str column name in the df that represents the items within a session\n"," save_filename: str output filename \n"," \n"," Example:\n"," Given a df that looks like \n"," SessionID | ItemID \n"," ----------------------\n"," 1 | 111\n"," 1 | 123\n"," 1 | 345\n"," 2 | 045 \n"," 2 | 334\n"," 2 | 342\n"," 2 | 8970\n"," 2 | 345\n"," \n"," Retrun a list of lists like this: \n"," sessions = [\n"," ['111', '123', '345'],\n"," ['045', '334', '342', '8970', '345'],\n"," ]\n"," \"\"\"\n"," grp_by_session = df.groupby([sessionID])\n","\n"," session_sequences = []\n"," for name, group in grp_by_session:\n"," session_sequences.append(list(group[itemID].values))\n","\n"," pickle.dump(session_sequences, open(save_filename, \"wb\"))\n"," return session_sequences"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":171},"id":"jiswHnavV7wt","outputId":"a4078ea5-9b1c-4253-aa38-97ab6628ced3"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'85116 --> 22375 --> 71477 --> 22492 --> 22771 --> 22772 --> 22773 --> 22774 --> 22775 --> 22805 --> 22725 --> 22726 --> 22727 --> 22728 --> 22729 --> 22212 --> 85167B --> 21171 --> 22195 --> 84969 --> 84997C --> 84997B --> 84997D --> 22494 --> 22497 --> 85232D --> 21064 --> 21731 --> 84558A --> 20780 --> 20782 --> 84625A --> 84625C --> 85116 --> 20719 --> 22375 --> 22376 --> 20966 --> 22725 --> 22726 --> 22727 --> 22728 --> 22729 --> 22196 --> 84992 --> 84991 --> 21976 --> 22417 --> 47559B --> 21154 --> 21041 --> 21035 --> 22423 --> 84969 --> 22134 --> 21832 --> 22422 --> 22497 --> 21731 --> 84558A --> 22376 --> 22374 --> 22371 --> 22375 --> 20665 --> 23076 --> 21791 --> 22550 --> 23177 --> 22432 --> 22774 --> 22195 --> 22196 --> 21975 --> 21041 --> 22423 --> 22699 --> 21731 --> 22492 --> 84559A --> 84559B --> 16008 --> 22821 --> 22497 --> 23084 --> 23162 --> 23171 --> 23172 --> 23170 --> 23173 --> 23174 --> 23175 --> 22371 --> 22375 --> 85178 --> 17021 --> 23146 --> 22196 --> 84558A --> 51014C --> 22727 --> 22725 --> 23308 --> 23297 --> 22375 --> 22374 --> 22376 --> 22371 --> 22372 --> 21578 --> 20719 --> 22727 --> 23146 --> 23147 --> 47559B --> 84992 --> 84991 --> 21975 --> 22423 --> 23175 --> 84558A --> 22992 --> 21791 --> 23316 --> 23480 --> 21265 --> 21636 --> 22372 --> 22375 --> 22371 --> 22374 --> 22252 --> 22945 --> 22423 --> 23173 --> 47580 --> 47567B --> 47559B --> 22698 --> 22697 --> 84558A --> 23084 --> 21731 --> 23177 --> 21791 --> 23508 --> 23506 --> 23503 --> 22992 --> 22561 --> 22492 --> 22621 --> 23146 --> 23421 --> 23422 --> 23420 --> 22699 --> 22725 --> 22728 --> 22726 --> 22727 --> 21976 --> 22417 --> 23308 --> 84991 --> 84992 --> 22196 --> 22195 --> 20719 --> 23162 --> 22131 --> 23497 --> 23552 --> 21064 --> 84625A --> 21731 --> 23084 --> 20719 --> 21265 --> 23271 --> 23506 --> 23508'"]},"execution_count":63,"metadata":{"tags":[]},"output_type":"execute_result"}],"source":["filename = os.path.join(ECOMM_PATH, ECOMM_FILENAME.replace(\".csv\", \"_sessions.pkl\"))\n","sessions = construct_session_sequences(df, \"CustomerID\", \"StockCode\", save_filename=filename)\n","' --> '.join(sessions[0])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7nvizM2Tl4_n"},"outputs":[],"source":["def load_ecomm(filename=None):\n"," \"\"\"\n"," Checks to see if the processed Online Retail ecommerce session sequence file exists\n"," If True: loads and returns the session sequences\n"," If False: creates and returns the session sequences constructed from the original data file\n"," \"\"\"\n"," original_filename = os.path.join(ECOMM_PATH, ECOMM_FILENAME)\n"," if filename is None:\n"," processed_filename = original_filename.replace(\".csv\", \"_sessions.pkl\")\n"," if os.path.exists(processed_filename):\n"," return pickle.load(open(processed_filename,'rb'))\n","\n"," df = load_original_ecomm(original_filename)\n"," df = preprocess_ecomm(df)\n"," session_sequences = construct_session_sequences(df, \"CustomerID\", \"StockCode\",\n"," save_filename=original_filename)\n"," return session_sequences"]},{"cell_type":"markdown","metadata":{"id":"4IgJEMr7XHB2"},"source":["## Splitting"]},{"cell_type":"markdown","source":["Wherein the first n-1 items highlighted in a green box act as part of the training set, while the item outside is used as ground truth for the recommendations generated.\n","\n","For each customer in the Online Retail Data Set, we construct the training set from the first n-1 purchased items. We construct test and validation sets as a series of [query item, ground truth item] pairs. The test and validation sets must be disjoint—that is, each set is composed of pairs with no pairs shared between the two sets (or else we would leak information from our validation into the final test set!)."],"metadata":{"id":"s1vAtXtVvYhp"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"GpIVmG6QV7rG"},"outputs":[],"source":["def train_test_split(session_sequences, test_size: int = 10000, rng=rng):\n"," \"\"\"\n"," Next Event Prediction (NEP) does not necessarily follow the traditional train/test split. \n"," Instead training is perform on the first n-1 items in a session sequence of n items. \n"," The test set is constructed of (n-1, n) \"query\" pairs where the n-1 item is used to generate \n"," recommendation predictions and it is checked whether the nth item is included in those recommendations. \n"," Example:\n"," Given a session sequence ['045', '334', '342', '8970', '128']\n"," Training is done on ['045', '334', '342', '8970']\n"," Testing (and validation) is done on ['8970', '128']\n"," \n"," Test and Validation sets are constructed to be disjoint. \n"," \"\"\"\n","\n"," ## Construct training set\n"," # use (1 st, ..., n-1 th) items from each session sequence to form the train set (drop last item)\n"," train = [sess[:-1] for sess in session_sequences]\n","\n"," if test_size > len(train):\n"," print(\n"," f\"Test set cannot be larger than train set. Train set contains {len(train)} sessions.\"\n"," )\n"," return\n","\n"," ## Construct test and validation sets\n"," # sub-sample 10k sessions, and use (n-1 th, n th) pairs of items from session_squences to form the\n"," # disjoint validaton and test sets\n"," test_validation = [sess[-2:] for sess in session_sequences]\n"," index = np.random.choice(range(len(test_validation)), test_size * 2, replace=False)\n"," test = np.array(test_validation)[index[:test_size]].tolist()\n"," validation = np.array(test_validation)[index[test_size:]].tolist()\n","\n"," return train, test, validation"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dR2iVWuscfvt","outputId":"218a61ca-53c1-4381-8af0-b00651e97fe5"},"outputs":[{"name":"stdout","output_type":"stream","text":["4234\n","4234 1000 1000\n"]}],"source":["print(len(sessions))\n","train, test, valid = train_test_split(sessions, test_size=1000)\n","print(len(train), len(valid), len(test))"]},{"cell_type":"markdown","metadata":{"id":"nrfW72QYXL3Z"},"source":["## Metrics"]},{"cell_type":"markdown","source":["For a sequence containing n interactions, we use the first (n-1) items in that sequence as part of the model training set. We randomly sample (n-1th, nth) pairs from these sequences for the validation and test sets. For prediction, we use the last item in the training sequence (the n-1th item) as the query item, and predict the K closest items to the query item using cosine similarity between the vector representations. We can then evaluate with the following metrics:\n","\n","- Recall at K (Recall@K) defined as the proportion of cases in which the ground truth item is among the top K recommendations for all test cases (that is, a test example is assigned a score of 1 if the nth item appears in the list, and 0 otherwise).\n","- Mean Reciprocal Rank at K (MRR@K), takes the average of the reciprocal ranks of the ground truth items within the top K recommendations for all test cases (that is, if the nth item was second in the list of recommendations, its reciprocal rank would be 1/2). This metric measures and favors higher ranks in the ordered list of recommendation results."],"metadata":{"id":"b-5ri_xSwfok"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"9gZKiSz4fALx"},"outputs":[],"source":["def recall_at_k(test, embeddings, k: int = 10) -> float:\n"," \"\"\"\n"," test must be a list of (query, ground truth) pairs\n"," embeddings must be a gensim.word2vec.wv thingy\n"," \"\"\"\n"," ratk_score = 0\n"," for query_item, ground_truth in test:\n"," # get the k most similar items to the query item (computes cosine similarity)\n"," neighbors = embeddings.similar_by_vector(query_item, topn=k)\n"," # clean up the list\n"," recommendations = [item for item, score in neighbors]\n"," # check if ground truth is in the recommedations\n"," if ground_truth in recommendations:\n"," ratk_score += 1\n"," ratk_score /= len(test)\n"," return ratk_score\n","\n","\n","def recall_at_k_baseline(test, comatrix, k: int = 10) -> float:\n"," \"\"\"\n"," test must be a list of (query, ground truth) pairs\n"," embeddings must be a gensim.word2vec.wv thingy\n"," \"\"\"\n"," ratk_score = 0\n"," for query_item, ground_truth in test:\n"," # get the k most similar items to the query item (computes cosine similarity)\n"," try:\n"," co_occ = collections.Counter(comatrix[query_item])\n"," items_and_counts = co_occ.most_common(k)\n"," recommendations = [item for (item, counts) in items_and_counts]\n"," if ground_truth in recommendations: \n"," ratk_score +=1\n"," except:\n"," pass\n"," ratk_score /= len(test)\n"," return ratk_score\n","\n","\n","def hitratio_at_k(test, embeddings, k: int = 10) -> float:\n"," \"\"\"\n"," Implemented EXACTLY as was done in the Hyperparameters Matter paper. \n"," In the paper this metric is described as \n"," • Hit ratio at K (HR@K). It is equal to 1 if the test item appears\n"," in the list of k predicted items and 0 otherwise [13]. \n"," \n"," But this is not what they implement, where they instead divide by k. \n"," What they have actually implemented is more like Precision@k.\n"," However, Precision@k doesn't make a lot of sense in this context because\n"," there is only ONE possible correct answer in the list of generated \n"," recommendations. I don't think this is the best metric to use but \n"," I'll keep it here for posterity. \n"," test must be a list of (query, ground truth) pairs\n"," embeddings must be a gensim.word2vec.wv thingy\n"," \"\"\"\n"," hratk_score = 0\n"," for query_item, ground_truth in test:\n"," # If the query item and next item are the same, prediction is automatically correct\n"," if query_item == ground_truth:\n"," hratk_score += 1 / k\n"," else:\n"," # get the k most similar items to the query item (computes cosine similarity)\n"," neighbors = embeddings.similar_by_vector(query_item, topn=k)\n"," # clean up the list\n"," recommendations = [item for item, score in neighbors]\n"," # check if ground truth is in the recommedations\n"," if ground_truth in recommendations:\n"," hratk_score += 1 / k\n"," hratk_score /= len(test)\n"," return hratk_score*1000\n","\n","\n","def mrr_at_k(test, embeddings, k: int) -> float:\n"," \"\"\"\n"," Mean Reciprocal Rank. \n"," test must be a list of (query, ground truth) pairs\n"," embeddings must be a gensim.word2vec.wv thingy\n"," \"\"\"\n"," mrratk_score = 0\n"," for query_item, ground_truth in test:\n"," # get the k most similar items to the query item (computes cosine similarity)\n"," neighbors = embeddings.similar_by_vector(query_item, topn=k)\n"," # clean up the list\n"," recommendations = [item for item, score in neighbors]\n"," # check if ground truth is in the recommedations\n"," if ground_truth in recommendations:\n"," # identify where the item is in the list\n"," rank_idx = (\n"," np.argwhere(np.array(recommendations) == ground_truth)[0][0] + 1\n"," )\n"," # score higher-ranked ground truth higher than lower-ranked ground truth\n"," mrratk_score += 1 / rank_idx\n"," mrratk_score /= len(test)\n"," return mrratk_score\n","\n","\n","def mrr_at_k_baseline(test, comatrix, k: int = 10) -> float:\n"," \"\"\"\n"," Mean Reciprocal Rank. \n"," test must be a list of (query, ground truth) pairs\n"," embeddings must be a gensim.word2vec.wv thingy\n"," \"\"\"\n"," mrratk_score = 0\n"," for query_item, ground_truth in test:\n"," # get the k most similar items to the query item (computes cosine similarity)\n"," try:\n"," co_occ = collections.Counter(comatrix[query_item])\n"," items_and_counts = co_occ.most_common(k)\n"," recommendations = [item for (item, counts) in items_and_counts]\n"," if ground_truth in recommendations: \n"," rank_idx = (\n"," np.argwhere(np.array(recommendations) == ground_truth)[0][0] + 1\n"," )\n"," mrratk_score += 1 / rank_idx\n"," except:\n"," pass\n"," mrratk_score /= len(test)\n"," return mrratk_score"]},{"cell_type":"markdown","metadata":{"id":"rxY8Dbfzshsv"},"source":["## Baseline analysis"]},{"cell_type":"markdown","source":["There are many baselines for the next event prediction (NEP) task. The simplest and most common are designed to recommend the item that most frequently co-occurs with the last item in the session. Known as “Association Rules,” this heuristic is straightforward, but doesn’t capture the complexity of the user’s session history."],"metadata":{"id":"XOkNZ1yevgeB"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"ZPmpUqm_smzm"},"outputs":[],"source":["def association_rules_baseline(train_sessions):\n"," \"\"\"\n"," Constructs a co-occurence matrix that counts how frequently each item \n"," co-occurs with any other item in a given session. This matrix can \n"," then be used to generate a list of recommendations according to the most\n"," frequently co-occurring items for the item in question. \n","\n"," These recommendations must be evaluated using the \"_baseline\" recall/mrr functions in metrics.py\n"," \"\"\"\n"," comatrix = collections.defaultdict(list)\n"," for session in train_sessions:\n"," for (x, y) in itertools.permutations(session, 2):\n"," comatrix[x].append(y)\n"," return comatrix"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dCUFeOw4smwI","outputId":"bb1aaf8c-b440-4e45-f69c-a6f473e193b6"},"outputs":[{"name":"stdout","output_type":"stream","text":["Recall@10: 0.143\n","MRR@10: 0.06450158730158734\n"]}],"source":["# Construct a co-occurrence matrix containing how frequently \n","# each item is found in the same session as any other item\n","comatrix = association_rules_baseline(train)\n","\n","# Recommendations are generated as the top K most frequently co-occurring items\n","# Compute metrics on these recommendations for each (query item, ground truth item)\n","# pair in the test set\n","recall_at_10 = recall_at_k_baseline(test, comatrix, k=10)\n","mrr_at_10 = mrr_at_k_baseline(test, comatrix, k=10)\n","\n","print(\"Recall@10:\", recall_at_10)\n","print(\"MRR@10:\", mrr_at_10)"]},{"cell_type":"markdown","metadata":{"id":"sfCnGNBUXT-V"},"source":["## Initializing Ray"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9uT3Woact705","outputId":"36390597-5e7d-482b-ba7d-702bff389666"},"outputs":[{"name":"stderr","output_type":"stream","text":["2021-06-11 06:21:17,485\tINFO services.py:1274 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n"]},{"data":{"text/plain":["{'metrics_export_port': 42187,\n"," 'node_id': '36950d392a77ab525dbcfe5ffdb2a3629ba8446f2aed261213be1f1f',\n"," 'node_ip_address': '172.28.0.2',\n"," 'object_store_address': '/tmp/ray/session_2021-06-11_06-21-14_124261_62/sockets/plasma_store',\n"," 'raylet_ip_address': '172.28.0.2',\n"," 'raylet_socket_name': '/tmp/ray/session_2021-06-11_06-21-14_124261_62/sockets/raylet',\n"," 'redis_address': '172.28.0.2:6379',\n"," 'session_dir': '/tmp/ray/session_2021-06-11_06-21-14_124261_62',\n"," 'webui_url': '127.0.0.1:8265'}"]},"execution_count":82,"metadata":{"tags":[]},"output_type":"execute_result"}],"source":["ray.init(num_cpus=4, ignore_reinit_error=True)"]},{"cell_type":"markdown","metadata":{"id":"nEn7YBp94I3K"},"source":["## Train word2vec with logging"]},{"cell_type":"markdown","source":["More recently, deep learning approaches have begun to make waves. Variations of graph neural networks and recurrent neural networks have been applied to the problem with promising results, and currently represent the state of the art in NEP for several use cases. However, while these algorithms capture complexity, they can also be difficult to understand, unintuitive in their recommendations, and not always better than comparably simple algorithms (in terms of prediction accuracy).\n","\n","There is still another option, though, that sits between simple heuristics and deep learning algorithms. It’s a model that can capture semantic complexity with only a single layer: word2vec.\n","\n","We can treat each session as a sentence, with each item or product in the session representing a “word.” A website’s collection of user browser histories will act as the corpus. Word2vec will crunch over the entire corpus, learning relationships between products in the context of user browsing behavior. The result will be a collection of embeddings: one for each product. The idea is that these learned product embeddings will contain more information than a simple heuristic, and training the word2vec algorithm is typically faster and easier than training more complex, data-hungry deep learning algorithms.\n","\n"],"metadata":{"id":"k24DXm6fvmfs"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"C_6624rRjSdW"},"outputs":[],"source":["def train_w2v(train_data, params:dict, callbacks=None, model_name=None):\n"," if model_name: \n"," # Load a model for additional training. \n"," model = Word2Vec.load(model_name)\n"," else: \n"," # train model\n"," if callbacks:\n"," model = Word2Vec(callbacks=callbacks, **params)\n"," else:\n"," model = Word2Vec(**params)\n"," model.build_vocab(train_data)\n","\n"," model.train(train_data, total_examples=model.corpus_count, epochs=model.epochs, compute_loss=True)\n"," vectors = model.wv\n"," return vectors\n"," \n","\n","def tune_w2v(config):\n"," ratk_logger = RecallAtKLogger(valid, k=config['k'], ray_tune=True)\n","\n"," # remove keys from config that aren't hyperparameters of word2vec\n"," config.pop('dataset')\n"," config.pop('k')\n"," train_w2v(train, params=config, callbacks=[ratk_logger])\n","\n","\n","class RecallAtKLogger(CallbackAny2Vec):\n"," '''Report Recall@K at each epoch'''\n"," def __init__(self, validation_set, k, ray_tune=False, save_model=False):\n"," self.epoch = 0\n"," self.recall_scores = []\n"," self.validation = validation_set\n"," self.k = k\n"," self.tune = ray_tune\n"," self.save = save_model\n","\n"," def on_epoch_begin(self, model):\n"," if not self.tune:\n"," print(f'Epoch: {self.epoch}', end='\\t')\n","\n"," def on_epoch_end(self, model):\n"," # method 1: deepcopy the model and set the model copy's wv to None\n"," mod = deepcopy(model)\n"," mod.wv.norms = None # will cause it recalculate norms? \n"," \n"," # Every 10 epochs, save the model \n"," if self.epoch%10 == 0 and self.save: \n"," # method 2: save and reload the model\n"," model.save(f\"{MODEL_DIR}w2v_{self.epoch}.model\")\n"," #mod = Word2Vec.load(f\"w2v_{self.epoch}.model\")\n"," \n"," ratk_score = recall_at_k(self.validation, mod.wv, self.k) \n","\n"," if self.tune: \n"," tune.report(recall_at_k = ratk_score) \n"," else:\n"," self.recall_scores.append(ratk_score)\n"," print(f' Recall@10: {ratk_score}')\n"," self.epoch += 1\n","\n","\n","class LossLogger(CallbackAny2Vec):\n"," '''Report training loss at each epoch'''\n"," def __init__(self):\n"," self.epoch = 0\n"," self.previous_loss = 0\n"," self.training_loss = []\n","\n"," def on_epoch_end(self, model):\n"," # the loss output by Word2Vec is more akin to a cumulative loss and increases each epoch\n"," # to get a value closer to loss per epoch, we subtract\n"," cumulative_loss = model.get_latest_training_loss()\n"," loss = cumulative_loss - self.previous_loss\n"," self.previous_loss = cumulative_loss\n"," self.training_loss.append(loss)\n"," print(f' Loss: {loss}')\n"," self.epoch += 1"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"1TeCHqd30rcy"},"outputs":[],"source":["expt_dir = '/content/big_HPO_no_distributed'"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":466},"id":"xEJnbuOgsmLQ","outputId":"29669c60-14ec-444c-831e-3c05cc101d3c"},"outputs":[{"name":"stdout","output_type":"stream","text":["Epoch: 0\t Recall@10: 0.181\n"," Loss: 731531.3125\n","Epoch: 1\t Recall@10: 0.209\n"," Loss: 619873.5625\n","Epoch: 2\t Recall@10: 0.213\n"," Loss: 511931.75\n","Epoch: 3\t Recall@10: 0.218\n"," Loss: 568087.625\n","Epoch: 4\t Recall@10: 0.216\n"," Loss: 587301.25\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD1CAYAAABA+A6aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxU9f7H8dewCwPIKIMKuGGl4ZK4S+6oSZuZC14XTNzSbt2bdjVa7F6v3uxqt6630rxalpZoLqn1U4u0W0miZSpoGVQILsAIosMi2/z+ODBKKoMywxmGz/Px8GHDbB9O8uY733M+36/GZDKZEEIIUe85qV2AEEII65BAF0IIByGBLoQQDkICXQghHIQEuhBCOAgXNd60qKiIpKQk/P39cXZ2VqMEIYSod8rKysjOzqZjx454eHhcd78qgZ6UlMSECRPUeGshhKj3NmzYQPfu3a/7uiqB7u/vDyhFNWvWTI0ShBCi3jl//jwTJkwwZ+jvqRLoldMszZo1IygoSI0ShBCi3rrZVLWcFBVCCAchgS6EEA5CAl0IIRxEjQL91KlTREREsH79+uvuO3DgAKNHj2bcuHG88cYb5q8vWbKEcePGERUVxbFjx6xXsRBCiBuyeFK0oKCARYsW0adPnxve//e//501a9YQEBDAxIkTGT58ODk5OaSlpREXF0dqaiqxsbHExcVZvXghhBBXWRyhu7m5sXr1avR6/XX3paen4+vrS/PmzXFycmLAgAEkJCSQkJBAREQEACEhIeTl5WE0Gq1fvRBCdbICt/2wOEJ3cXHBxeXGD8vOzkan05lv63Q60tPTyc3NJTQ0tMrXs7Oz0Wq1ta+4IAdWD4L7lsJd99X+9YQQNVZWbiI128jxjDyOn8kj6UweJ85dwmQCnZcbOi83/Lzc0Hm6ovNyR+flip+XG0283PDzdDM/prGnG85OGrW/HYdTJ9ehW/U3uLsPePjCthkw8yvwa2W91xZCmJWWlZNSEd5JZ5QAP3nuMoUlZQA0cnXm7hY+jOkWhKuzEzkFxeTmF5OTX8yvBiO5+SUYr5Te8LU1GvBt5KoEvKfb1dCvuH3tL4fKr3u5OaPRyC+B6tQq0PV6PQaDwXw7MzMTvV6Pq6trla9nZWXdtLPpljm7wJh1sGoAbJ4CU3eDi7t1XluIBqqkrJyfM43m4FbC+xJXSssB8HRzJrSFD1E9g+kU6EvHQF9C/LUWR9lFJWVcLCghpyLoK0P/Qn5F+FfcTs8p4Gj6RXILiikpu/EA0M3FyRz+Oq+KTwCertf/MtAqfzf2dMPNpWFdyFerQA8KCsJoNJKRkUGzZs3Yt28fy5YtIzc3lxUrVhAVFUVycjJ6vd460y2VdG1g5BsQNxH2vgCRr1jvtYVwcMWl5ZzKvGwO76Szlzh57hLFFeGtdXfh7hY+TOzdqiK8fWjT1HJ434iHqzPNfJ1p5nv9QlI3YjKZuHyl1DzSr/yTW3DNL4H8EnLyr5B0MY+c/GLyCktu+nreHi7KSN/zmtCv/ON5/W2fRi71+lOAxUBPSkpi6dKlnDlzBhcXF/bs2cPgwYMJCgpi6NChvPTSS8ydOxeAyMhI2rRpQ5s2bQgNDSUqKgqNRsPChQutX3mHB6H3HPj2DWjVB0Ifsf57CFHPXSkt49R5Y0VwK1MnP567THGZEt7e7i6EBvoQ3acVHStG3m2aeOGk0vy2RqPBx8MVHw9XWjXxqtFzSsrKuVhQooS+UQn/3/8yyMkv5vylIk6eu8SF/GLzJ4/fc3bSVMz1u16d9rnBL4Nrzwd4uNrPirEaNTaJzsjIYMiQIcTHx9duLZfSYng3ErJ+hJlfQpMQ6xUpRD1TVFLGT+cvm4P7+Jk8fjp/2TyF4ePhQsdAXzoF+hJa8Xcrnadq4a2mguJSJezzS7iQf6Ui9JWRf05+ydVPCBVTQrkFxZTfJCk93ZyV0NdWDfrfB3/lNJFvI9fbPiFsKTtVWZzLalzcYPQ7sKofbI6GmM/AtZHaVQlhc0UlZZw8d4mks5dIqrji5FTmZUorUse3kSudAn2JubctHQN96BToS0udZ72eTrAmTzcXPN1cCPKr2ePLyk1cKiwh55rRf+U5gBxjcZUTwr/U4ITwSw+GEt23tfW+oQr1O9ABGgfDI6vgg7GwewE8+LraFQlhVYXFZZw4d4nks3nmywV/zjJSVhHefp6udAz0ZcZdbc0j8CC/RhLeVuTspMGvYtolpIbXd9zshHBuQTE92+gsv8BtqP+BDnDncLj3z/D1v6BlX+gyTu2KhLgtBcWlnDx3qSK4L5F0Jo+U7Kvh3cTLjY6BvkR0CKBjoA8dA30JbCzhbY9u9YSwNThGoAMMeh5OH4Rdf4LmXUDfXu2KhKhW/pVSTlSEd+Wcd2q20TxX21TrTqdAH4aFBphH3s19PSS8xU05TqA7u8DotbDyXmU+ffoX4Fazs+RC2JrxSinJZ652Vx4/k8cvhnwqL0nQe7vTKdCXEZ2a06kivAN83CW8xS1xnEAH8GkOj/4X3n8EPpkLI99SzkAIUYcuFZWQXDFdUhngv164Gt7NfDzoGOjDg11amMNb71N3H8uF43KsQAcIGQQD5sOXL0OrcAibpHZFwoHlFZaYR96V4f3bhQLz/S18PQgN9GVk10Bzh6W/t3Q2C9twvEAHGPAXOJ0An86DFl2hWUe1KxIO4GJBMUlnLpmDO+lsHmnXhHdg40Z0DPRhdLcgc5NOU62Et6g7jhnoTs7K1MvKfrBpMszYDx4+alcl6pHc/OIqo+6ks3mk5xSa7w/ya0SnQF/Gdr+6tonOy03FioVw1EAH0OqVk6TrHoCdTyn/LfPp4homk4kL+cWkZBlJzTZW/J1PSuZlzuYVmR/XUudJ58DG/KHn1bVNGntKeAv747iBDtA6HAY/D/F/U/67xzS1KxIqKCs3cSa3kJTsy6Rm5ZOSZSQlWwnxiwVXF3Zq5OpMiN6Lnm10dGiudFeGtvDF19NVxeqFqDnHDnSA8D9DWgLsfhYCuylz6sIhFZWU8ashv8qIOyXLyK+G/CqLMTXVutHWX0tkp+a089cSotfSTq+luY9Hg1zXRDgOxw90JycY9XbFfHo0zPwfNGqsdlWiFi4WFFcJ7NRsJcTTcwvMlwZqNBDs50mIvxf97mhKO72WEH/lj5/MdQsH5fiBDuCpgzHvwDsj4OM5MG69zKfbOZPJxNm8IlIrQjsl20hqxcjbYCw2P87NxYm2Tb3oFOTLI10DzcHd1t/LrpY1FaIuNIxABwjuCUP/Bnti4ds3oc8ctSsSKJstpF343TRJtpFfsvMpKC4zP863kSvt9FqGtA8gRO9lDu4gP0/Zm1KICg0n0AF6z4a0A/DZixDUE4J7qF1Rg3G5qMQ8NWK+oiTLSFpOgXnhKVAacUL0Wsb10BHir8xtt9NraeLlJm3wQljQsAJdo4GH/3N1P9JZXynTMcIqTCYTWZevKNMk5ssAlb8zL10xP87FSUPrpl7cGeBNZKfmyojb35u2/l54uTesf5JCWFPD++lp5Adj3oW1w2HbTBgfp5w4FTVWWlbO6ZwC84i7MrhTs41cLrq6qL/W3YUQfy/C2109KdlOr6WlzhNXZznmQlhbwwt0gMAwGL5EWRrgm9eg39NqV2SXCopL+SX7+ssAf7uQX2Vndr23O+30WkbeE2ieIgnx18pqgULUsYYZ6KA0GaUdgC8WQXAvpfGoATKZTORUdEsqV5Lkm68oOXPxaqu7kwZaNfEixF/L4A562lWMttv6a/FtJI03QtiDhhvoGo2yXd25o/DRVJj1NWhruLdUPVRebiIjt/B3128rIX5tt6SHqxMh/lq6t/ZjnH+wecTdqokn7i5yGaAQ9qzhBjooC3aNXQf/jYCt02DiVmVhLwdyLq+Qpz78gaMZF6t0SzbxciPEX8uIjs0rpkiUSwFb+DaSbkkh6qmGHegAzTpB5D9hxx/hf/+EgQvUrshq8gpKiF6byNmLRUzq3UoJbr2WdtItKYRDkkAH6DpJmU/f/7Iynx4ySO2Kaq2opIzp7x3mV0M+6x7rSd92TdUuSQhhYzW6dmzJkiWMGzeOqKgojh07VuW+zz//nEcffZTx48ezfv16AA4ePEjv3r2ZNGkSkyZNYtGiRdav3Jo0Grh/OfjfBVumwaVzaldUK2XlJp7aeIRDaTm8OvYeCXMhGgiLI/TExETS0tKIi4sjNTWV2NhY4uLiACgvL2fRokVs27aNxo0bM336dCIiIgDo2bMn//73v21bvTW5ecGYdbB6EGyJgck7lI2n6xmTycQLHyexJzmThQ/ezYNdWqhdkhCijlgcoSckJJhDOiQkhLy8PIxGIwC5ubn4+Pig0+lwcnKid+/eHDhwwLYV25K+PTzwGqR9A/sWq13NbVnxRQofHDzNrAEhPBbeRu1yhBB1yGKgGwwG/Pz8zLd1Oh3Z2dnm/87Pz+e3336jpKSEgwcPYjAYAEhJSWHWrFmMHz+eb775xkbl20CXcRAWDV+/Cqf2ql3NLfkw8TSvfnaKUWGBzL/vLrXLEULUsVueUzCZrnYIajQaXn75ZWJjY/H29iYoKAiA1q1b88QTTzBixAjS09OZPHkye/fuxc2tnlxZMWIpnPkets2AmV9B42C1K7LosxOZPLftOAPu9Gfpo52lQ1OIBsjiCF2v15tH3QBZWVn4+19twOnZsycffPABq1atwtvbm8DAQAICAoiMjESj0dCyZUuaNm1KZmambb4DW3BtpFyfXlYKHz0GpcWWn6Oi79JyeOKD7+kU6MubE8JknRQhGiiLP/nh4eHs2bMHgOTkZPR6PVqt1nz/tGnTuHDhAgUFBezbt48+ffqwY8cO1qxZA0B2djYXLlwgICDARt+CjTQJgYdXQMYhiP+r2tXc1M+Zl5n67mFaNG7E2ik9ZLVCIRowiz/9YWFhhIaGEhUVhUajYeHChWzduhVvb2+GDh3K2LFjmTp1KhqNhhkzZqDT6Rg8eDDz5s0jPj6ekpISXnrppfoz3XKt0EeU69MT/gMte0OHB9WuqIpzeYVEr03EzcWJ96b2pInWXe2ShBAq0piunRSvIxkZGQwZMoT4+HjzvLvdKr2iLLV74ReY+SXo7OPKkbyCEsauSuDMxUI2zuhNx0BftUsSQtiYpeyUyVZLXNyV9dM1wOZoKClSuyJzF+gvBiOrJnWTMBdCABLoNePXGkauVFZm3PucqqVUdoEm/pbD8rH3EC5doEKIChLoNdU+Evo8AYf+C0lbVCnBZDLxYkUX6IsP3M1D0gUqhLiGBPqtiHhJ2Vx6x5NgSKnzt1/xRQobDp5m5oC2TL3XPubyhRD2QwL9Vji7wph3wNkNNk2GkkLLz7GSjZVdoF0DmT+8fZ29rxCi/pBAv1W+QTBqNWQlw6fP1MlbfnYik9htx+l/pz9LR3eWDSiEEDckgX477oiAfvPgyPvww4c2favKLtCOgb68JV2gQohqSDrcroHPQqt74ZOnIeukTd4iJUvpAm3u6yFdoEIIiyTQb5ezC4xeo6yjvikarhit+vLn84qYvCYRV2cn3pvai6bSBSqEsEACvTa8m8Gja8BwCnb9GazUdJtXqOwFeqmolHcf60HLJp5WeV0hhGOTQK+ttgNgUCwc3wTfr6v1y13bBbpyonSBCiFqTgLdGvrNg5DB8Olf4Nwxy4+/ibJyE3/a+AOJvypdoPfeIV2gQoiak0C3Bicn5VJGT52y3kvRpVt+CZPJxEs7ktmdfJ4XpAtUCHEbJNCtxaspjF4LuWmw44+3PJ/+ny9SeP/bNGb2b0uMdIEKIW6DBLo1teoLQ16EE9shcXWNn7Yx8TTLK7tA75MuUCHE7ZFAt7a+T8Kd98GeWDjzncWHfy5doEIIK5FAtzYnJxj5lnJJ46YpUJh704d+l5bLEx9KF6gQwjokQWzBU6dsinH5HGyffcP59JSsy8SsO0QzH+kCFUJYhwS6rQR1h2GL4KdPlT1Jr3E+r4jotYdwcdJIF6gQwmok0G2p1yxlY+nPFsLpb4GrXaAXC4p597Ge0gUqhLAaCXRb0mjg4TegcTBsfoyii5nX7AXaXbpAhRBWJYFuax6+MGYdpoILpL49gUO/Glg2pot0gQohrE4CvQ6YmndhZ/M/ElpwiE2h3/LwPYFqlySEcEAS6HXgjX0pPJnSlWTdMHr88ib8+pXaJQkhHFCNAn3JkiWMGzeOqKgojh2ruvjU559/zqOPPsr48eNZv359jZ7TkGw6lM6yvad4pGsQHaavAV0IbImBy5lqlyaEcDAWAz0xMZG0tDTi4uJYvHgxixcvNt9XXl7OokWLWL16NRs2bGDfvn2cP3++2uc0JPEnM3l223H63dGUpY92xqmRD4xdpyzetSUGysvULlEI4UAsBnpCQgIREREAhISEkJeXh9Go7M6Tm5uLj48POp0OJycnevfuzYEDB6p9TkPxXVoucz74nrub+/DWxG64uVQc6oBQuH85/PYV7H9Z3SKFEA7FYqAbDAb8/PzMt3U6HdnZ2eb/zs/P57fffqOkpISDBw9iMBiqfU5DkJJlJGbdIQJ8PHjnsR5of98F2nUC3DMB/vdPSIlXp0ghhMO55X5z0zVt7BqNhpdffpnY2Fi8vb0JCgqy+BxHp3SBJlZ0gfa8eRdo5DI4ewS2TodZX4OPrH8uhKgdiyN0vV6PwWAw387KysLf3998u2fPnnzwwQesWrUKb29vAgMDLT7HUeUVljDlnatdoK2aeN38wW6eMGYdlBTBR1OhrKTuChVCOCSLgR4eHs6ePXsASE5ORq/Xo9VqzfdPmzaNCxcuUFBQwL59++jTp4/F5ziiopIyZrx3mNRsIysn1XAvUP874cHX4XQCfLHI9kUKIRyaxSmXsLAwQkNDiYqKQqPRsHDhQrZu3Yq3tzdDhw5l7NixTJ06FY1Gw4wZM9DpdOh0uuue48jKyk08vekHDv6aw+tR99Dvjlv4NNJ5DJw+AN+8Di37wF0jbFeoEMKhaUwqTHBnZGQwZMgQ4uPjbzrvXl+YTCYW7kjmvYQ0nr+/A9P6tb31FykpgjVD4eJpmPUVNG5p/UKFEPWepeyUTtFaenN/Ku8lpDGjf9vbC3MAVw9l/XRTOWyeAqXF1ixRCNFASKDXwqbD6fxzz0+MvKcFC2q7F2iTEHj4P8q2dZ+9aJ0ChRANigT6bYo/mcmzW5Uu0FdGd7HOXqB3Pwy9HoeDb8GJj2v/ekKIBkUC/TZ8f/omXaDWMPRvENgNPn4CLqRa73WFEA5PAv0WpWQZmfqu0gW6dsoNukBry8VNmU/XOMHmaOWEqRBC1IAE+i3IvFS1C9Tf20Z7gTZuCY+sgvPHYc+ztnkPIYTDkUCvoWv3An1nioUuUGu46z4IfwoOr4Vjm237XkIIhyCBXgOVXaApWUoXaKegOtoLdPALSrPRzqcg+1TdvKcQot6SQLfg2i7QZWO63FoXaG05u8Lotcp16psmQ3FB3b23EKLekUCvhslk4q87k/n0+Hmev78DI7uqsBeoTwsYtRqyf4RP59X9+wsh6g0J9GpUdoFO79fm9rtAraHdEOj/DPywAY6st/x4IUSDJIF+E5VdoA/f04JnR3RQuxwYuABa94NP5kFmstrVCCHskAT6DXzx49Uu0H9aqwu0tpyc4dE14OEDm6LhymW1KxJC2BkJ9N85cjqX2Ru+p0Nzb+t3gdaWd4AS6jmpypUvDWgnKCGEZXaUVupLzVa6QPXeHrwzpaf1u0CtoU0/GPQcJG1RrlEXQogKEugVMi8VMXlNIs627gK1hnufhnYRsHsBnP1B7WqEEHZCAh24VKR0geZWdIG2bmrjLtDacnKCR94Gz6bKei9FeWpXJISwAw0+0Kt0gU6swy7Q2vJqoizilZcBH8+R+XQhRMMO9Mou0G9/UbpA+99Zh12g1tCyF0S8BCd3wsGValcjhFBZgw10k8nE3yq6QJ+LVKkL1Br6PAF33Q97n4eMw2pXI4RQUYMN9Df3p7IuIY1p97Zhen8Vu0BrS6OBkW8oSwRsngIFOWpXJIRQSYMM9M3XdIHGRtpBF2htNfJT5tMvn4ftj0N5udoVCSFU0OACfd+PWSzYepx729lRF6g1BHaD4Uvg1G448G+1qxFCqKBBBfq1XaArJ9lZF6g19JwOd4+E+L9B2gG1qxFC1LEatUIuWbKEo0ePotFoiI2NpXPnzub7NmzYwI4dO3BycqJjx44899xzbN26lddff52WLVsC0LdvXx5//HHbfAc1VNkF6u/tbr9doLWl0cBDK+D8MfhoKsz8CrT17ModIcRts5hqiYmJpKWlERcXR2pqKrGxscTFxQFgNBpZs2YNe/fuxcXFhalTp/LDD0rnYmRkJPPnz7dt9TVU2QXqpKkHXaC15eEDY9bBfyNg2wyY8JGysJcQwuFZnHNISEggIiICgJCQEPLy8jAajQC4urri6upKQUEBpaWlFBYW4utrX405l4pKmPLOIaUL9LEe9t8Fag3NO0PkK5D6BXy1XO1qhBB1xGKgGwwG/Pz8zLd1Oh3Z2dkAuLu7M2fOHCIiIhg0aBBdunShTZs2gDKyj4mJITo6mhMnTtio/OpdKVW6QH/OvMzKid3oHNRYlTpUERYNncfBviXwy361qxFC1IFbnkg2XdNibjQaWbVqFbt370ar1RIdHc2PP/5Ily5d0Ol0DBw4kCNHjjB//nx27txp1cItKS838XTcUb79JYd/jauHXaC1pdHA/a8qi3dtmQazvgbvZmpXJYSwIYsjdL1ej8FgMN/OysrC318Jx9TUVIKDg9HpdLi5udG9e3eSkpIICQlh4MCBAHTt2pWcnBzKysps8x3cgMlk4m+7TvDJ8XPERrbnka5BdfbedsVdC2Pfg+J8pemorETtioQQNmQx0MPDw9mzZw8AycnJ6PV6tFotAIGBgaSmplJUVARAUlISrVu3ZvXq1ezatQuAU6dOodPpcHauuxNzb32ZyrsHfiPm3jZMV3MvUHugb69c+XI6Afa+oHY1QggbsjjlEhYWRmhoKFFRUWg0GhYuXMjWrVvx9vZm6NChxMTEMHnyZJydnenatSvdu3cnKCiIZ555ho0bN1JaWsrixYvr4nsBlC7QV3b/xENdWvBcZAc0GgdpHKqNTqOVdV4OvgVB3ZXbQgiHozGZ6n7d1YyMDIYMGUJ8fDxBQdabDtn3UxbT1h2mT9smrJ3Sw/Eah2qjrATWPQjnjsK0zyEgVO2KhBC3yFJ2OkziHTmdy+z139O+mTdvTQyTMP89Z1dlvRd3b4ibKJtiCOGAHCL1fqnoAm3q7cY7j/XA28NV7ZLsk3czpeno4mnYNksW8RLCwdT7QM+6VMTktZVdoL3Qe3uoXZJ9a9UHhi2Gnz6Fr19VuxohhBXV60C/VFRC9DuHyMkvZu2UHrRpCF2g1tBrJnQaA1/8HVLi1a5GCGEl9TbQr5SWMfO97/g58zJvTexGl+AG1AVaWxoNPPg66DvAlhjITVO7IiGEFdTLQC8vN/H0pqMk/HKBV0Z3ZkBD6wK1BjcvGLceystg02QoKVK7IiFELdW7QDd3gR47x7Mj2jMqrIF2gVpDkxB4ZBWc+wE+nad2NUKIWqp3gZ6TX8y7B35jangbZtTnvUDtRftI6DcPjrwP372rdjVCiFqod7s8NNG6882CwbTw9ZAuUGsZFAtnv4dPn4FmnZTt7IQQ9U69G6EDBDZuJGFuTU7O8Oga0DaDuMmQb7D8HCGE3amXgS5swFMH496D/Gxl+7ryulsdUwhhHRLo4qoWXeH+5fDrl/DFIrWrEULcIgl0UVXYJOg2Bb7+F5ys201JhBC1I4EurjfiFWgRBtseB8PPalcjhKghCXRxPRd3ZacjFzdlZcYrRrUrEkLUgAS6uLHGwTB6LRhOwY4noO6XzRdC3CIJdHFzbQfCkBcheRt8+6ba1QghLJBAF9UL/xO0f0DZj/S3r9WuRghRDQl0UT2NBka+Bbo2sHkKXDqrdkVCiJuQQBeWefgoKzMWF8CmaCgtVrsiIcQNSKCLmtF3gIf/AxmJsPc5tasRQtyABLqouY6joM8TkPg2HI1TuxohxO9IoItbE/EStAqHnU/B+eNqVyOEuIYEurg1zq4w5l1o1FhpOirMVbsiIUSFGgX6kiVLGDduHFFRURw7dqzKfRs2bGDcuHGMHz+exYsXA1BSUsLcuXMZP348EydOJD093fqVC/Vo9TBmHeSdgW2zoLxc7YqEqD+MWTZbzdRioCcmJpKWlkZcXByLFy82hzaA0WhkzZo1bNiwgQ8//JDU1FR++OEHdu3ahY+PDx9++CGzZs1i+fLlNileqKhlL7jvH3BqN3y1TO1qhLBPZaVw9ggcXKUsS/2vjrDsDvji7zZ5O4s7FiUkJBAREQFASEgIeXl5GI1GtFotrq6uuLq6UlBQgKenJ4WFhfj6+pKQkMDIkSMB6Nu3L7GxsTYpXqisxzTIOAT7lihL794xVO2KhFBXQQ5kHIb0g8qfM99BSYFyn3cLZSDUZw50HmeTt7cY6AaDgdDQUPNtnU5HdnY2Wq0Wd3d35syZQ0REBO7u7tx///20adMGg8GATqcDwMnJCY1GQ3FxMW5ubjb5JoRKNBp44DXITIYt02Dml+DXWu2qhKgbJhNcSLka3qcPguEn5T6Ns7KdY9dJENwTWvYGX9tvaH/Le4qarlmkyWg0smrVKnbv3o1WqyU6Opoff/yx2ucIB+PmCePeh7cHKidJYz4D10ZqVyWE9RUXKNMn6d9CeqLypzBHuc+jsRLcncdAcG8IDAM3rzov0WKg6/V6DIare0xmZWXh7+8PQGpqKsHBwebRePfu3UlKSkKv15OdnU379u0pKSnBZDLJ6NyR6drCqNXwwVjY9TSMfFMZvQtRn106C6crw/sgnD8G5aXKfU3ugLsilSmU4F7KbSf1Lxq0GOjh4eGsWLGCqKgokpOT0ev1aLVaAAIDA0lNTaWoqAgPDw+SkpIYMGAA7u7u7N69m379+rFv3z569epl829EqOzO4TBgPny5FIK6Q48YtSsSoubKSiHz+NXwTk+EvIqr81w8ILAb9H1SCe+gHuDVRN16b8JioIeFhREaGkpUVBQajWV6odMAABIxSURBVIaFCxeydetWvL29GTp0KDExMUyePBlnZ2e6du1K9+7dKSsr48CBA4wfPx43NzdefvnluvhehNoGLIAz38P/zYfmXZRgF8Ie1fTkZXBPCOikbPZSD2hMKkxwZ2RkMGTIEOLj4wkKsv2JAlGHCnKU+fSyEpj5P9D6q12RaOh+f/IyPRGyK871VZ68DO6lhHdwL2VzFztlKTtv+aSoENXy1CknSdcMg48eg0nbwVn+mYk6ZD55eU2Am09e+iqh3Wm08ndgN1VOXtqK/KQJ62veBR74F2x/HOL/CsMWqV2RcGSXzl69bPBmJy8rLx20k5OXtiKBLmzjnj8oc5QH/q3Mpd/9sNoVCUdQVgqZSVVH39edvPyjcumgHZ+8tBUJdGE79/0Dzh2F7bPBvz3436V2RaK+KcyF9EMOd/LSViTQhe24uMPY92BVf6XpaPoX4O6tdlXCXtXk5GVl52VwL6XzUvodqpBAF7blGwhj3oH3HlZG6mPfkx9Coahy8rLi+u+bnbxsEQbuWnXrrQck0IXttekPEX+Fz16AAysg/Em1KxJqqDx5mZ6odGDe7ORlcC9oeqdDn7y0FQl0UTf6/lFZmfHzhdDiHiXkheMyn7xMvLr2iZy8tDkJdFE3NBpljZfVP8Lmx5SmI99AtasS1lKYq1zVdPrbG5y8bK6MuuXkpc1JoIu64+4N49bD6sGwaTI89qly4lTUX+ePwydzlRCHipOXHeXkpUok0EXd8r8LHn4DNkfD7mfhgVfVrkjcjtJi+Gq5sltVIx0Mek5p3JGTl6qSQBd1L3QknHnyatPRPX9QuyJxK84dU65YyjwOncbAiFeUJR+E6iTQhTqGLFQuWdv1ZwgIVZYLEPattFgZkX+1HDybQNQH0P5+tasS15DrgoQ6nF1g9DvKx/W4ScoqjcJ+nf1BWUXzy6XQ8VGY/a2EuR2SQBfq0forjUaXzsLWGVBernZF4vdKryg71K8eDAUXYPxGGPW2TLHYKQl0oa7gHjBiKaR8poz+hP04870yKv/fP6HzWJjzLdw1Qu2qRDVkDl2or/tU5RrmL19WNte9c7jaFTVspVeUX65fvwZaPYyPg7vuU7sqUQMyQhfq02iUyxebdYKt0yHnF7UrarjOfAerBignPrtEwewECfN6RAJd2AfXRkrTERqIm6ws3CTqTkkRfP4S/DcCivLgD5uVzt5GfmpXJm6BBLqwH36t4dH/KmuA7PqTspyqsL2M7+DtAfD1v5SegNkJcOcwtasSt0ECXdiXO4bCwGfhWBwc+q/a1Ti2kiL47EVYEwFXLsOELUoXb6PGalcmbpOcFBX2p/8zylzu7gXQrLOyK42wrvRD8PFsMJyCsMkw7O/KGuSiXpMRurA/Tk4wapWyqNPmaLicqXZFjqOkEPa+AGuHKecpJm6Bh1ZImDsICXRhnxr5KSdJCy/CR49BWYnaFdV/6Ymwsp+yhk7YZGWuvF2E2lUJK6rRlMuSJUs4evQoGo2G2NhYOnfuDEBmZibz5s0zPy49PZ25c+dSUlLC66+/TsuWLQHo27cvjz/+uA3KFw6tWSd48HXYNkO5AmP4YrUrqp9KCpVuz4Q3lE89k7ZByGC1qxI2YDHQExMTSUtLIy4ujtTUVGJjY4mLiwMgICCA999/H4DS0lImTZrE4MGD2bNnD5GRkcyfP9+21QvH12UcnDkMCf9RdrnpOErtiuqX09/Cx3OUzZe7PQZD/wYePmpXJWzEYqAnJCQQEaF8LAsJCSEvLw+j0YhWW3XN423btjF8+HC8vLxsU6louIYthnNH4eMnQN9B+SOqV1ygjMq/fRN8g2HSdggZpHZVwsYszqEbDAb8/K42F+h0OrKzs6973ObNmxk9erT5dmJiIjExMURHR3PixAkrlSsaJBc3GLMO3LwgbiIUXVK7IvuWlgArw+HbN5RlFWYfkDBvIG75skXTDZo9jhw5Qtu2bc2j9i5duqDT6Rg4cCBHjhxh/vz57Ny5s/bViobLpzmMeQfWPQTbH1dOmMq2ZlUV50P8Iji4EhoHw+Qd0HaA2lWJOmRxhK7X6zEYDObbWVlZ+Pv7V3nM/v376dOnj/l2SEgIAwcOBKBr167k5ORQVlZmpZJFg9X6XmUO+Mdd8M1raldjX377Bt4Kh4NvQY9p8HiChHkDZDHQw8PD2bNnDwDJycno9frr5s+PHz9O+/btzbdXr17Nrl27ADh16hQ6nQ5nZ2dr1i0aqj5zIPQRiP8b/LJf7WrUV5wPn/4F3o0EUzlE74L7l8m+ng2UxSmXsLAwQkNDiYqKQqPRsHDhQrZu3Yq3tzdDhw4FIDs7myZNmpif8+CDD/LMM8+wceNGSktLWbxYLjcTVqLRwEP/gayT8NFUmPGlMr3QEP32tXIFS+5v0HMmDHlRgryB05huNCluYxkZGQwZMoT4+HiCgoLq+u2FIzD8DG8PgqZ3wGP/B64ealdUd64YlevyD61WFjR7+A1lOko4PEvZKZ2ion5qegc88hac/R52N6B+h1//B2/1VcK81yx4/ICEuTCTQBf1V4cH4d4/w3fvwvfvq12NbV0xwq6nYd2D4OSsfCoZsVS5lFOICrLaoqjfBj2v7H35yVxo1hFadFW7Iuv75UvY8QRcTIfes2HwC+DmqXZVwg7JCF3Ub84uMHotePkrOx0V5KhdkfVcuQy7/gzvPQROrsqo/L5/SJiLm5JAF/WfV1MY9x4Yz8OWGCh3gJ6H1H3wZh84/A70eQJmfQ2t+lh+nmjQJNCFYwjsBpH/hNQvYP8/1K7m9hVdgp1PwfsjwcUdpu5RVpmUUbmoAZlDF44jLBoyDsH//gktwqB9pNoV3ZqUeNjxJFw6A33/CIOeUzbPFqKGZIQuHIdGA5HLofk9sG0mXEhVu6KaKcqDHX+E9aOUAI/Zq2wJJ2EubpEEunAsrh4w9j3l0r64iUprvD37+XNlrvzIegh/CmZ9BcE91a5K1FMS6MLx+LWCR9coywPsfArqvhnassKLStv+hkfBTQsxnykLj8moXNSCBLpwTO2GwODn4PhmOLhK7Wqq+vkzZVT+wwdKY9TM/0FQd7WrEg5ATooKx3XvXKXpaO9z0LyL+pf9FV6EPc/BD+vBv72ypntQN3VrEg5FRujCcTk5wci3oHFL2BwNl8+rV8upPfBmbzj6Idz7tLJKpIS5sDIJdOHYGjVWRsJXLsPmKVBWUrfvX5gL2x6HD8aCR2OY9jlELGxYq0OKOiOBLhxfQCg8tAJOJ8DeF+rufX/6P3ijNxyLg37zYOaXEBhWd+8vGhyZQxcNQ6fRkHFY2aItqLty21YKcmD3s3BsI+jvhj9sdMxFw4TdkUAXDcewRXDuB6WJR99BGblb24+fwq4/Qb4B+v8F+j8DLm7Wfx8hbkCmXETD4ewKY94Fd2+l6agoz3qvXZADW6bDxvHKyo/Tv1Aum5QwF3VIAl00LN7NYMw6uHgats2C8vLav+bJXfBGL0jeCgMWwPR90OKe2r+uELdIAl00PK36KGul/PQpfP3q7b9O/gX4KAbiJoA2QAnyQc/KqFyoRubQRcPUa5ZykvSLvysnLNsNubXnn9gBnzytXJY48Fnl2nIJcqEyGaGLhkmjgYf+rZwc3RIDuWk1e17+Bdj8GGyapEzfzNgPAxdImAu7IIEuGi43L6XpqLwMNk2GkqLqH5+8Hd7oCSd3KmuVT98HzTrVTa1C1IAEumjYmoTAI6uUyxk/nXfjx+QbYFO0snyAb6AyKh/wF+WqGSHsSI3m0JcsWcLRo0fRaDTExsbSuXNnADIzM5k37+oPQXp6OnPnzuW+++5jwYIFnD17FmdnZ/7xj38QHBxsm+9AiNpqH6l0cn61TGk66jbl6n3J2+CTucrWcIOfh/A/SZALu2Ux0BMTE0lLSyMuLo7U1FRiY2OJi4sDICAggPfffx+A0tJSJk2axODBg9m1axc+Pj4sX76cr7/+muXLl/Paa6/Z9jsRojYGxcLZ7+HTZ5RpFN+W8OlcOPGxsgNS9FsQcLfaVQpRLYtTLgkJCURERAAQEhJCXl4eRqPxusdt27aN4cOH4+XlRUJCAkOHDgWgb9++fP/991YuWwgrc3JWNsXQNoONE5S58p/+D4a8CNPiJcxFvWAx0A0GA35+fubbOp2O7Ozs6x63efNmRo8ebX6OTqdT3sDJCY1GQ3FxsbVqFsI2PHUw7j1l3XK/1srGE/3mgrNc3Svqh1v+l2q6wXZeR44coW3btmi12ho/Rwi71KIrzD0J7j7KqF2IesTiCF2v12MwGMy3s7Ky8Pf3r/KY/fv306dPnyrPqRzFl5SUYDKZcHOT63RFPdHIT8Jc1EsWAz08PJw9e/YAkJycjF6vv24kfvz4cdq3b1/lObt37wZg37599OrVy5o1CyGEuAGLUy5hYWGEhoYSFRWFRqNh4cKFbN26FW9vb/OJz+zsbJo0aWJ+TmRkJAcOHGD8+PG4ubnx8ssv2+47EEIIAdRwDv3aa82BKqNxgJ07d1a5XXntuRBCiLojnaJCCOEgJNCFEMJBSKALIYSDUKVjoqysDIDz58+r8fZCCFEvVWZmZYb+niqBXnmN+oQJE9R4eyGEqNeys7Np1arVdV/XmFRo4ywqKiIpKQl/f3+cnaWBQwghaqKsrIzs7Gw6duyIh4fHdferEuhCCCGsT06KCiGEg7D7ZeRutrkGwIEDB3j11Vdxdnamf//+zJkzxy7qGjx4MM2aNTNPJy1btoyAgIA6qevUqVPMnj2bKVOmMHHixCr3qXm8qqtLzeP1yiuv8N1331FaWsrMmTMZNmyY+T41j1d1dal1vAoLC1mwYAEXLlzgypUrzJ49m0GDBpnvV+t4WapLzX9foEwxP/DAA8yePZtRo0aZv26T42WyYwcPHjTNmDHDZDKZTCkpKaaxY8dWuX/EiBGms2fPmsrKykzjx483/fzzz3ZR16BBg0xGo7FOarlWfn6+aeLEiabnn3/e9P777193v1rHy1Jdah2vhIQE07Rp00wmk8mUk5NjGjBgQJX71TpelupS63h98sknprfffttkMplMGRkZpmHDhlW5X63jZakutY5XpVdffdU0atQo05YtW6p83RbHy66nXKrbXCM9PR1fX1+aN2+Ok5MTAwYMICEhQfW61OTm5sbq1avR6/XX3afm8aquLjX16NGD119/HQAfHx8KCwvNl4Opebyqq0tNkZGRTJ8+HYBz585VGeWqebyqq0ttqamppKSkMHDgwCpft9XxsuspF4PBQGhoqPl25eYaWq2W7Oxs8yYalfelp6erXlelhQsXcubMGbp168bcuXPRaDQ2r8vFxQUXlxv/L1XzeFVXVyU1jpezszOenp4AfPTRR/Tv39/8sVzN41VdXZXUOF6VoqKiOH/+PCtXrjR/Tc3jVV1dldQ6XkuXLuWFF15g+/btVb5uq+Nl14H+eyY7vSDn93U9+eST9OvXD19fX+bMmcOePXu47777VKrO/ql9vD7//HM++ugj1q5dW2fvWRM3q0vt47Vx40ZOnjzJM888w44dO+r0l0l1blaXWsdr+/bt3HPPPQQHB9v8vSrZ9ZRLdZtr/P6+zMzMOvtIb2nTj5EjR9KkSRNcXFzo378/p06dqpO6qqPm8bJEzeP11VdfsXLlSlavXo23t7f562ofr5vVBeodr6SkJM6dOwdAhw4dKCsrIycnB1D3eFVXF6h3vPbv3098fDxjx45l8+bNvPnmmxw4cACw3fGy60CvbnONoKAgjEYjGRkZlJaWsm/fPsLDw1Wv6/Lly8TExJj3UD106BB33HFHndRVHTWPV3XUPF6XL1/mlVdeYdWqVTRu3LjKfWoer+rqUvN4HT582PxpwWAwUFBQYN5vWM3jVV1dah6v1157jS1btrBp0ybGjBnD7Nmz6du3L2C742X3jUXLli3j8OHD5s01Tpw4Yd5c49ChQyxbtgyAYcOGERMTYxd1rVu3ju3bt+Pu7s7dd9/NCy+8UCcfS5OSkli6dClnzpzBxcWFgIAABg8eTFBQkKrHy1Jdah2vuLg4VqxYQZs2bcxf69WrF3fddZeqx8tSXWodr6KiIp577jnOnTtHUVERTzzxBBcvXlT959FSXWodr2utWLGCwMBAAJseL7sPdCGEEDVj11MuQgghak4CXQghHIQEuhBCOAgJdCGEcBAS6EII4SAk0IUQwkFIoAshhIOQQBdCCAfx/8I2Xz21+MObAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"tags":[]},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["0.235\n","0.13457301587301593\n"]}],"source":["use_saved_expt = False\n","if use_saved_expt:\n"," analysis = Analysis(expt_dir, default_metric=\"recall_at_k\", default_mode=\"max\")\n"," w2v_params = analysis.get_best_config()\n","else:\n"," w2v_params = {\n"," \"min_count\": 1,\n"," \"iter\": 5,\n"," \"workers\": 10,\n"," \"sg\": 1,\n"," }\n","\n","# Instantiate callback to measurs Recall@K on the validation set after each epoch of training\n","ratk_logger = RecallAtKLogger(valid, k=10, save_model=True)\n","# Instantiate callback to compute Word2Vec's training loss on the training set after each epoch of training\n","loss_logger = LossLogger()\n","# Train Word2Vec model and retrieve trained embeddings\n","embeddings = train_w2v(train, w2v_params, [ratk_logger, loss_logger])\n","\n","# Save results\n","pickle.dump(ratk_logger.recall_scores, open(os.path.join(\"/content\", f\"recall@k_per_epoch.pkl\"), \"wb\"))\n","pickle.dump(loss_logger.training_loss, open(os.path.join(\"/content\", f\"trainloss_per_epoch.pkl\"), \"wb\"))\n","\n","# Save trained embeddings\n","embeddings.save(os.path.join(\"/content\", f\"embeddings.wv\"))\n","\n","# Visualize metrics as a function of epoch\n","plt.plot(np.array(ratk_logger.recall_scores)/np.max(ratk_logger.recall_scores))\n","plt.plot(np.array(loss_logger.training_loss)/np.max(loss_logger.training_loss))\n","plt.show()\n","\n","# Print results on the test set\n","print(recall_at_k(test, embeddings, k=10))\n","print(mrr_at_k(test, embeddings, k=10))"]},{"cell_type":"markdown","metadata":{"id":"6F0OaTwC4NJK"},"source":["## Tune word2vec with ray"]},{"cell_type":"markdown","source":["In the previous section, we simply trained word2vec using the default hyperparameters-but hyperparameters matter! In addition to the learning rate or the embedding size (hyperparameters likely familiar to many), word2vec has several others which have considerable impact on the resulting embeddings. Let’s see how.\n","\n","The default word2vec's parameters in Gensim library were found to produce semantically meaningful representations for words in documents, but we are learning embeddings for products in online sessions. The order of products in online sessions will not have the same structure as words in sentences, so we’ll need to consider adjusting word2vec’s hyperparameters to be more appropriate to the task."],"metadata":{"id":"84gKhlj9v3uN"}},{"cell_type":"markdown","source":["![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAooAAAB3CAIAAAALhfY1AAAgAElEQVR4nO2dz0sbW//43/Pl8w9kI5MSkTwUCvciF6SCE7DwBHR1vdRMFZxwC9K6kmdj7aYZXcRJN1U3F1etFPohI9ROLNdnVSEXWsgELIEi9wMFuYMYmsFN/oT5Ls458yO/TGI0E+f9WsU4c37Nmfevc07enGVZgCAIgiCIn/h//W4AgiAIgiC1oHpGEARBEN+B6hlBEARBfAeqZwRBEATxHaieEQRBEMR3oHpGEARBEN/xP5dewXHcDbQDQRAEQYJGi7PNl6tnAKhWq71rDIJcC6FQCCdqO+BAuQn4aAS8+30nFAq1+C8GtxEEQRDEd6B6RhAEQRDfgeoZQW6KHx8eh0KhxQ8X/W5I7zjeumqPLj4shkKhxx9+9K5Rg8jXrdAtmxtktodCj7WLC+1xKBQKvTq+wXq3bqSy66Vj9UwG+rFmz6Krv5+DxPGrW/LggwAVChT7qR1vdSMpGt1VL1IHUsgeb4VquIqyJKW5S6j/xu8cv3INhv00f3x47BF93cL0Fp2XXwEALs6+XbXYXuKZEt11+Ti7dAgzr/+uvhOHet6+Gi60x84E+3F2eN313RToPSO3kwvt8b2nhzNvvler1erfr2dAme651rz/++uHAB///MwUz/FfCgDIy4+uXSD1HvlT1ebdoztdlzP++5sZgMM/dTbYXz8rAPDiP1co80Y5fhWafgnyUbVarVaPZPi4dK+Hbt+PD49/Xjp8+Pp7tVqtfn/9EJSpxx9+wJD4rlqtVt/6YOb8+PA4NK3QFlY/vYDDp/eIDdEJF2f/5/xBe/d8vOYiYgZ1XnhL7j+rVqvV6rPaygaQHqpnr41sx/Hoh62tRY+16FzjMdBopOvxIr2SPr9XzJjzWrKNv1x87AQ3vrqsQPaO1Zd5bLtZzntIWuIu/+LDYmj6JQAo03YvXOXTb+rbgPSDs38OAeCXkSEAgDuP3hHZR0QPALycpk+nnYlUfxdl6MGvblV0/PklAMgP7jcp1o0n0O0NQdVPqr5Q++Y67q8Tlnj1uf6+IeG3GYDD/36mg0JMln+Pe96pxs60J9B9/KpOnlzBmWsboldmRogxQWT983H4uhX6eekQ4PDpPbdAqHnETeWVDfHtfiLzcujRW2YMkYf+6tg7Sm6h1KC664B6vRlqKIw/r1ar1Wf3AaDhzKSPbOvVY/fTOX51b+kjABwu/ez0wyvkQ6EQNYOe3Xd3nz3rV8f2eD5eZCHbRq/V8avQvaeHpC7nmlZvU+M2gzdq0s9Xj9FD9Tz+7Ei2RdWF/ueh2434+G0kU61WP8kAyhQRcMdbzIr8/mbGa6Ad/rLsmhMACjyoVqvf38zAx6VV7QLg4kNq6fDFJ+oY1Zi3H3/5D7WejremFOo/Hcnwctr9YivwgFiv8HHpD9ikn1/+wUTDvaWPM6//dpc/9Ojt99cPgfgZz4gInlLgxSdiYxIruK4NSH8Y+dcMACh/eQ2kO4/e/f16BgBefKpWn423OZFq73LwqCLqJj64pNjWtJhUfYG8uX+/noHDpdSHCwD4unXv6SFpYfXfoNTfcufBb05QwTFZLrTVpY/ER//++uHh0s/tG68tZEXPGRr5CTzeP+H+s+qRDAAzb74TH7eRiKDUySsXd0ZmAODl5+Z9JzqbSkuglk2r6noKeV6N+LoVYuK0bmYefvsXFaGHT1c//IDx50RUzrz+mwUh7ArsjhB1sNOGqfHrZrVafSdCw9dq/Hn10wtaV20gvZM2w9et6Zfgurj/zlWX6vnw6T1mZEw77+f9BzITVWf/HFI3gvDwtwd3AGD8wQsAUD5/hQvtDwVg5tcHQ0zGuZ4TM10ZZIK6ROHQo7csVHJn5BcA+L8z5xk/HBmhn8afVdkDuzMyA3D4z5m3TPIqEh+LvpZnPwB+fPjjJWszkTVMbbs5zi4dsraN/1sGOFzKsgfqtAHpD0Piu+9vZuDltG0NN3rZ2pxIzaGq6OyMLR+SedWq2Ja0mlTXi2KPlMc9pW/ByC/g6ebMv0YA6CtfBw0qnP0A+HH2DYjJQiKcxLgh79q3s/Ysj5ayovcQce+IuCbBjxYiok5euaCmnjPazUyN41fTCgC8+EQ8gXYk0rVCoiAkHEVmpsuCmflN8IrQppDgxC8jdwBgZIS9O62hMbDOX6uO2uxe/h9/7gvnqq2fJaln5s13ZqccbzkaevzBC4CXf37OjJy9bEu6HT69F3raVQu+boWmGhjuNZAFyK4qAPi4dO/j0qVXKVOhy9uB9IMh8V1VBAC64De9OPK9fm2vvYnUopIHv87AR+Xz199H/usySa9WbD8mlfypPXnkWTVowpDw2wwcKn8d//6vP21rwysrOqZ7WdE548+r1ecAQJ7j0r2fRurXTQHaFRG13Hn0rvoIAAAuPizeW5p6PPL3u0c113zdmn4JAPInd73dVXdtHP5zBtCpGzI08hPAx29nP2D8ztnZxw49mau+ra3aPCS++/RPaJpNM5eO6xs93hpGLJSz7Gfb2m0NjTwTOtgWcbw1pbDNLJ8a2e8AAPDjw+rTQyB7HEhwsiPY5ojWm2XoFhJCw3cY6Tsu589LexOpJdSf++t/zz5SN/Hqxfp5UpFVg2+t/Rbq4X3+X1cUjbiDpGufXnRcb7ey4mqQWHQzL609EdGcZu4mmT8gH3kNpqtW1w4kwNkWNILSaQXPP8l0TXpaAflTz8V+S1q3mSy0eyLefaXXO7fvP5ABlJcKCx0wqGR0FqKGxP/ITuTn4sNiqzU2IgvIeraj9YnZ9eOyEwlkF0ZHu+3vPPrPC2dH7oX2uOFulPHk6xlndfN4CzeC+QjvaVpnVbgRbU6kZlBVpCiOm9ikWGIl1H8mzQOAQZhUQyO/QM1ye6OrHvw6A6AoL2uGnaxbubf10gWmus/OImhHsuLKkC2udNhrZY5NSxHRWF4BvdK1fODaS+huwatpBWDmzXd7802bEqknjCdd+wxcm6vH/y3bXTv+q07Ct8vFh8Xpb46lxewPYgaRK/Q/W8nqTt7Wztr8dcu7Y/GXkX6fNej5wSpmfNHFZsbDb3+whWpmEo4/I8v7oVAodG/pI/ypNxdE/10NhUL3nh7Cw9eb4hA9vEHuTcFvL6DBFgxgc5qsPv418vohwMvpNjeVjD8nu1dCtN7/kq0cQ4+WZWfn9p1H745ktro5rTz89vl695QiHfAg4zzBENlsRVet7FmxddzmRKq9qwaiisAlZ5sVO/7gBbCDOq5r/gLZVVGfJpWzGnrJGeX7z743bLkXElQAl8lC5f7PoVBoFX6VgekAusT+89ax55rP4LhxnciKK/Jj5PfqJ5mNBjmbR4Oc939//dDZud1ERABAvbxyIWyS3W1s4079mgK1S5zFb7oBqnl1vYWsjtPRDpHdUr/fuSCb40irpl+CfNSl+z7y04xr6xKbbC5ZvfrPL00inU3fVnva1FotnbT5GEbYCJPNa/1fe+ZapMugV3BctZPfTCenBp05bZ/z6yoeZZ9BfHb/8ouRIBPCH/dvDxwoN70djYGTVzc+GY63QtNgjw/RDrbpHDxCoVALFXxNP0vSXdwDQRAEub10vYQUSLrcud0EtjNzcH4hCEEQBLkh7jzafPPnPffBhIevvwfVdb6U3ge3EaQvYMy2TXCg3AR8NALe/b7Tl+A2giAIgiDdg+oZQRAEQXwHqmcEQRAE8R1trT3fTFMQBEEQJFC0UMFt7dy+VIUjSN/huMttTQRwoLwEfDQC3v2+09r7xeA2giAIgvgOVM8IgiAI4jtuv3o290RuXjXbvLqY6eDi62sGgiAIEmy6U896hmN0qXL0TDc3mup8Ru/wHn5Bs95LfMd19RifNCNg0ImaKfa7Ib7H3BMbvc7Omy7uBce2NNX5K8q3wcXVd/bk1XK/GxVUulDPeoaLgW4RCqPJcOBmMDIIlFWRi4Feyc71uyW+R9/gwgdiRa9JDul+0yviQTggVo6+EdZmK4582+jUIxh0EtlzKt6t82wCxqKRfrcoqHSsns29TTldSE3QP4W1SmUlzv4l1pmcpjovqnsZjwFezHBcTN5Phh23xjbSqaWmb3AcxxzlsipyXOZPVeTCyX05VmPIl1XRrq6Y8d6V0cmX9AXTM1xGZY10yRrbYBRVAzwle1tl7ol21foGx4oFc0/kat7houN1UNuTNUPf8JimtBlOXR2HB5BGmOpKcky37ImKNKWYiZ1kK/WhnbJRmss+oQPISyuKfBSIuSmsWdoCHYzo3UR/G9NfzC8aqE+EfjcjsHSsno3TXOJu1PUFz0/wPIC5J4YPRGJzVmY1l8mZS57GiSU6Ji2rZYCJlHWeTcxlKxaRnm4jfdUYFtUyCGtWRS3FNnQAU13RxHMr9ZukWQUFlILlvDwAAJG4CFqeqE+jBFAyiDosG40mlqzBDrEKSwJVhPpGODlasCzLsnbggKWvLavisCYSK/Jc1IYzOgA/KcJB3iRVnQCcGCYdE8guuqsy1e1S1r53xRNdENaYZaorQMSfp67oJmroHsBL71E3t8dEqvGyS9nI1XzDJnxgMPMHOWUqUOqJl95rEnWX9V0JxElckesbvdoaZuYPcsoKfcn5hVVlfZOtWCSY6hLi6ZxRv4xRzLvccSGezmlfTADgF3ayJ5uZjWVtdkdqFV3ho6OkWDN/MFbQx8jt+pE8Fq2fWMoqUe2RuDhHFLlpnNgt5KUVGt8zv2i59CqtNyKtpuXNPRMi0bF9wwCAcl4bLRRGiVmg59dr4j+Gsc96GpG0xkvOekYoZbclnpqorI+RuDgn54MRRUR8zURc2U/u0qloqtsyjDZ4o24nNPq1DNvBNfLMvU3ZloFIP+hUPZvGyTW0Yj1mB3tj6/a3vLQyJq+PrS5cIhOEKaVkmACGAdFoJAoHeRNM40SJt/VeGcZ++w0V4umSUQYoG3A3Gr0L2hcTykYpHfca2ELqPFsSvOFrD6Y6HwPdtlIhJ4XZAISTHbQHQa4P9zRe1gAC5EdOpGg4bYWrXbcKCgGMHPiOTtUzH59N5E7dK7SmXrxyxCtdsFyw2LWeEUpKmoS4W0JUcjFfmo3zkbgIhlHOaxCNXnIbIRrtZOsQUcn6UUmc5PlJEU4N84sGd+uqikga21tR2q7dOmfuLSdHC26rPKFW3CMQWIMd8RfONBYB7HXo4MBL29lE4EL6AABQ3E0G8Yn7i46D2/zCqrIesz1Cc285JiyrZV5aUWSmhzqLikzEXZFwU523d4fFSupOam0nexK7ZMtoJC6CsXtUEid5AD4+W8q/NWA23l4gjo+O5pJvqQWgH8mubrJWldXNdRoV5ydFON3Nn4jxCEAkLp7kd0/rlmfsXWkEEg+3KWbCB2JlzTFL+UkRpF12vZ7BtWfEX+iZYU3cDsaxQLIRlQkc/W0yF5yQvgv9SE60K0KRa8O6jEbXFJwTGC7Ht6KyXY5zWeYMVrJzzjb9QhoU3V2C/S+nQEWvVM6tQtpVyHk2AeRGekiGFeJQURMASsHecuW+RldYI+nOsrqGOWdvlLRSUy942skuZr0upMGp141zRoXdy5pRSHvGn/rNnroqlfP6EpFLqJ2otceEaoM0gaXuja4/e1bzarjn/23jEvnmiLLbSWMVoCuNxRrSa1qr4LYyVl16DYL0HZyobYID5SbgoxHw7ved1uN/+3/UE0EQBEEGDlTPCIIgCOI7UD0jCIIgiO9A9YwgCIIgvgPVM4IgCIL4DlTPCIIgCOI72jpYdTNNQRAEQZBA0UIF/88V70cQn4AnONsEB8pNwEcj4N3vO629XwxuIwiCIIjvQPWMIAiCIL7Dv+rZ3BO5+dpcT77BSVyhbwQ25RyCIAhyXXSunouZmhzG5p4o7vVGjZp7ol0yv6BZ7wcgSY6wZllrtzArqvtZ3Br0DY7j0JxqgLknchznY4P4RqGjwXFck/RxrgtElm3vFlFWxUZdw0lyw3TpPct1OYzboqyKHNcrXT54BLz7fcZU57kYFJy8aghD3+DCB2KlPsdXMClmwtIYyddUUUuxem3kuQCSK7dLXRUz3LAmnhdqZgNOkpunK/U8l82OJpcbqhlqdnmtTvvL4eSYbmkLPNh+jGOLmeo8F5ZyssC+KWa4DR1Az7iddSeVsp5pYb0WM/W2bV2NpJCMygzhTNExil0JrUVxT2XFNTClXcGD2tJadN9FXUc88QlTnefYODiFu3W8Y8g375dTGRsEp4RihttQ1Xl3B+uexeBj7i0nRwu3Ms5xVYqZ2Em2MgiRqptBP5IVPUUmCr+wquxrea+E0SFaOW91wSCjZ4RS9lyTIt6vcZL0gy695/hiFqTdWl1VVsVhTSSpYc+jm7asX2Ff6gp1u4uZGLCUyVTT89J7lhDaMwmEJ2pCPqJVmV80UJ8IoGe4GNCMzqvGcI2GNtXtEs1Qey5qK81qJMga7NC2Cdwy+XyeLQmOJs5JRpzcqUMDU9oDK80poVH3HRp1ZCJl327uLWuzFaZUWOFWRTwIE6Vr7onhA5GkpK3MamEnbFvfEtA3OHsQVk/DjoZe12Cb+QobevNnMcDwCxrq5sZMpG7NU+4FpnHi/jMancsZXu0rTEi8rb3KRgnGojXKbIARUladbgacJP2h261hEWmHinIH84sG6g59tJG4OCfniwBgGPts+k7ElX3DAICJlC0rhSkld2q0qIqfFBPreaLq8gcgTvJQzMvpQmqCFhBP57Qvbq1nGPvsjYpIGplVTWtUVok7OxFXICFO8qzxJfudTKhP6J0T8cssZVaaU0Kj7ts060hE2lFLmxuZ5QNxx/G2WeHAx2eJyWLmD3LKCn1t+IVVZX2TWSr1LdHz60rBPQgHeTpq6VXy1PhJMXFi3A53GUGuH1NdSYItHxCkp3S/c5tf2MmexGp2D+WkMIsgh5P75LtolOppgGJenotGAWjMliDIl9Rka/pyXgMxTlTdeozdz8XWa24QUufZksDi0e4ocZs1NiUanevilvruu2jSEX5hdWxdHltpbLHy0bFO2wEAALJTWfeDMBg0WstAkB5iqvNhbbZSt1yFIL3hKgereGlFkbdVtzuYUEmclZKaAABe2mbKUoDCe4m3FwJpxPjSvQZ8fDZRMkzziwazcfoqpAvuimrfkIikkX+cZ0vbqtlxjc0wjP1Ob2nQfQ9NOqJvxEppxR1jd2MapY7bDgCgeCq71dEqYS0Q3UR6DR8ddf9pGPuJRrFrPcOhbkaul6ude55IFUaTMSlH/uInRdeCtH0y2FRXjFWmr0kUyDjN2WXoR5e7cfykCAe7uySyDQATcVcU11TnvWvPzvYxAADYN4zOa3RjB4HNvU15jrnv7dKg+w7NOlLMxE6yO2sp7woC88JB35VyypRgW0hO81iYuhFCPC1vsvXmHh6HQ5DbhDClyMwsdr3yesbeUFlWRS4GDbZ5IkgvuerPkghrrv33EUk7j27SkOJm9PxJlCoeV0yVE9UyCGsFhQV183eziXUaJI/eTchCo2OpkbgIsmxHtkFIWavGMI2iGys7cc/FkqYDLX1YE89TAjStsR0So8YyqUkaY+5vNDonx5qciayjtvvu8WvQkWKGE0rZbYlnKwhsQBQ4olHwklqhK9YTqcqsRlYUwgdipeXuJ2HNWj2lqw/h09WdyVaNbvosBhW6Qz4s5diCQpuP79bD1n0EGfaT4QazNGBMpCpqKVb7yjuYX7QcgMyWz7jbdVqSnQSJyZBLDtu9w0nSB9rKWHXpNa0oZrijuLNptpjhtqMDtEHf3BOXYad7M7ln3dczXD5e738jjKtO1MCAA+Um4KMR8O73ndbjf+0/6tlglXQ0Oii6+eoEvPsIgiBId7SVUPIq8As72fmwK2uWUrAC5AEGvPsIgiBId1x/cBtBbgScqG2CA+Um4KMR8O73nT4HtxEEQRAE6RRUzwiCIAjiO1A9IwiCIIjvaGvt+WaagiAIgiCBooUKbmvnNu4dQPwPbnJpExwoNwEfjYB3v++09n4xuI0gCIIgvgPVM4IgCIL4Dn+o52Lm+n7bmSQWvPEfxdUzTi5L/3GdA44gCIJcnc7VczHDeRWPv3Mf6fl1pVCfcfJaMNV5O8uCkKL5NJE+QtNg+NdO8g00EUJtbmw6gLcs68NluHLDBy5ZuKvvjbP4IDdHl96zncQwUOgbmOZocKBZ/yrZuX63xPfoG1z4QKzU5kHXM1wMdJILtSIehANi5egbYW2W5q0vjCbDgYszJbLnLFf6eTYBY43SXSM3QVfqeS6bHU0u11vTZVV0jE09w8wuc08U99SMY43qmQaWqcGsNo+xRkLTHuO9mOE2VHW+gaa0L6aR27IqcjEZ5Fit7W+q86K6l6l3C1gyNU8iRbvY2Em2QnNG1XkVZVXkwsl9uy6WvLmYcbfT3BNdbeOapTV0OuIZzxZDQcoxXZ8b9LRJQNsu2TXyRae2wbSdTXUlOaZjAKMNipnYSbZBFrWyUZrLPqEDyEsrinwUCEUlrDnBtujdRH8b01/MLxqoTzBJQL/o0nuOL2ZB2m3/Zc1JRtyyLKuShWSYy8eZZero+HUNtom9JmrDVLvoG1wMCsSMWz0Ni7UXe7Irui8uQEzcMyEiaVZBgYbB7VzylLZiTFq2zYjwgUjM5srdTarMihlWbCULyd0iqcv2KtjtzeqaeJKdk/PU7TDzB5BdFKCsisOaSEzU8+hmjYYuq5snWdqMWY0MUYMaveNWSMsxbpl8rqilmKOJ7Z6yYfHg9pBWjWGijE11u0Qt6HNRWxnESAkvvUfd3B4TKathhtOykav55sQYwJlwFcz8QU6ZCpR64qX3mkTdZX1XAnESE+z1jW63hkWkHY8OuIQENcH4+GwC0nEy34UpJXdq0CvSq3ROROIi1Wd6fl0psEzJwpSSO8ibNRc76Pn1RHaRXXy59WBfLMTTOaMMVHduUznFT4qJ9bwOYBqlxN0osMaXDBMAhDVb9Nu3N4OPzyao21HOayDGI8Qm3anrL6Ns5PYNMi78gkaUfdMa2VAIUwrMifEIa7wjSV3D4h5DQjEvpwvukrUvJoBh7LMqIpI2OMm5kV4yEVf2qT0KYKrbcoByodLo0TJsB9fIM/c25QaSFrk5ut+5zS/sZE9i17AcxUdH7c9yzN6gIMg9r6mOXHKYVTeczNHWjDGVZuYPcmNEQDmhaS62fkmhjqb/osFsnAi4nBRmBYST+94bJlIVtcQ6zhzrTmpsSiTaIFS37owxK1lInWdLAmtBMFYckTrc02BZAwiQHzmRsizLsnZgpdmS0K0ngJED33GVg1W8tKLI26rh/o65fVfANE7sz0rBcnHtbpxrT4TFgucTqcJoMsxxHBfWZiupCbDXNWnEOH1ZqZG4OFcyymb+wIkUJdSKpyavhc4vaHRDjlraJCvZHdXYjPpwJQCkPWNMI/MRSWN7Q0qB3AaIALingQhgr0MHB17aziYCF9IHAIDibjKIT9xfXO3c80SqMJqMSUzmR6JjwOK0ZaPUUVHreWqjFneT+0p8AgCEeFreZGullx3fEp6okHxLy9DfJjvf0cDHZ50SnJPBxczm3YpHdYFhOP6unr/cl+Xjs6C93SWRbQDgJ0VX7F3PeNeene1jAACQOzU6r9ENiVcDiU8mmPtOmYgr65tsJZttZyurortJPTC5kIFGzwxr4nYw1jjKquiKGOlvk7nghPRd6Ed1sgK5ca76syTCWsF1GkNI6YpMomErWmcFpSFPg9il7Dnd8yWsWaunNAgcPl3dmWxVAL+gFYDGaWNQ6OKgM7+gVe5u0pDudrSyGDUBTABwAtEk0uXqJpePqglZIMosGp2TY42iwfykCOuyHdmGiKSdR1lNm9HzJ9Gypxl2R8IHYmVNaF5jOyTGTpdpFH2UDUskmliP0b5YqwYN6YeNlZ04aZ7OWjCsieepgYxw0eXDcHIf6NAFNEp5KWzDvyDDPgkUiS6LjUxRLShrkBFJswrAVnZiJ9nK2kBO/ytRzMTWldWb+K0IpBVtZay69Jrbi6nOL8O2LZtMdT5srAzQbpGa9t9mgj1ROwAHyk3ARyPg3e87rcffHz/q6V/cUWVCAg/pIwiCINdNWwklA4yQ0se4YS7J/k6oFQ3VM4IgCHLNYHAbuSXgRG0THCg3AR+NgHe/72BwG0EQBEEGDFTPCIIgCOI7UD0jCIIgiO9oa+35ZpqCIAiCIIGihQpua+c27h1A/A9ucmkTHCg3AR+NgHe/77T2fjG4jSAIgiC+A9UzgiAIgvgOv6hnfeN6cxeS8jmu/Z+qZpCfbp5vnrWJpHpscQGCIAiCdEjH6tncE716Ts90p/ZukrK6uU4yJ7qyO9CUCR7qk2KZHWbeQhAEGWR0j2S0HQ9XynmvnGQpVTDrTK/p2nt2Uj0OAA3zHNOM65XsHNiZnuvzXNHUy9eeahpBEMQHlI3SXNZJR09Fn54Z1sRzmnF+TAqzSKepzoe1WXp5AWIt0/4indF9cDsnLatl71eeMK+esT8TP3Ujw4wyUS3ahpjX7T7K1JpsTlzadTENOIuNosouU478q6yKggwAsB5rx8sn1YnzIjUSaeN1p4N1Lay/nTb32mL1CIIg10WDFNdCyrIT30Wjc/b3hrHvpJ4UFrNwkEf93Cu6VM9KWgHIJd92EspYh7hlVdQEQC65DTv0s8cLlyFuke/3k8t7JgCYe2JsHRTdsqyCAnLMoxTFnVq/1lTnw8l9pWBZ1nk2sZ8Mb+gQkTRdAagLbrdmdqfOmTbVlWQuXbDchddQzMTWIaFWLMsqpKGTrMwIgiA+oGzkwGBOjljrgwFAOa/tK/EJenHt4t++YdxAI4NBt97z1JPsHMB6rAMHMR0XAPjoGAC1zsjn3KnzNJUpAQD4STEBkDvIm6DvSjkAMhWEeBpgP7lr11hv4pXz2j7AXDQKAJG4OAewnu9OQY7Vm4/AS+8ti+Rmj0THAODEqLET3QvVwprVgTWAIAjiE9Y12LYsy7L0sVW+KSwAAAJGSURBVOSw42PQ0OAK7NiSLRIX5xwXS3+bzBHxi/SCroPbvLSiAIC8vXn9W6fkGMdxHBdb7+LeklFv/XWNs5ssJjf6P7+gFdKQk8LNNpohCIL4momUZcexJ+KKS4QKa5ZlWdY2LDsrd7z0vjDGJN7mSSIxG8dtOr3iCgerJlKFNMB+rsGuqx6jFCyH1ERH945Fe5aeWc8IMmtMQWlyEZ3BViU712h5HkEQZKCJSDtqouSEDoWUxbaM7Y+t1u2uRbrmSueehcVswv03CfkSivmG/mVryCM3v2g5gMRsnAfhiZoAkPPEUitmLtm4TwLaZPGDBLrT8R6Hl0noxr3i4ul1xrta00PjAEEQ5NrRN1zno4p5eU6MR8h5Wucw7a6Uq1v+M9X5GOi4nNdLrvazJBFpR3UraOGJmoD9ZJjjuCNo5l+24mCZ47iwlIO57M4CDyxcLAscx3GcICdO8i39UV56X8nOyTGO44aTublsZa2Hs8XVuxUQ08D2r9FF8fCGrkM0O5dLDnMcF07uJ7LnOFkRBBkkhDWrADG6iCdA4b3Ek/Oluv1trKRW3FFMfYPjuLCx0mloE7mEtjJW4W+mI/4HJ2qb4EC5CfhoBLz7faf1+PvlRz0RBEEQBLFB9YwgCIIgvgPVM4IgCIL4DlTPCIIgCOI7UD0jCIIgiO9A9YwgCIIgvqOtg1U30xQEQRAECRQtVDAeekMQBEEQ34HBbQRBEATxHaieEQRBEMR3oHpGEARBEN+B6hlBEARBfAeqZwRBEATxHf8fpHfJ0OOpk5AAAAAASUVORK5CYII=)"],"metadata":{"id":"aAcFrEVPv5dO"}},{"cell_type":"markdown","source":["This table shows the main hyperparameters we tuned over. For each one, we show the starting and ending values we tried, along with the step size we used. The total number of trials is computed by multiplying each value in the Configurations column.\n","\n","We trained a word2vec model using the best hyperparameters found above, which resulted in a Recall@10 score of 25.18±0.19 on the validation set, and 25.21±.26 for the test set. These scores may not seem immediately impressive, but if we consider that there are more than 3600 different products to recommend, this is far better than random chance!"],"metadata":{"id":"pqWaBbQxv7Em"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":258},"id":"W-rvETTgsepP","outputId":"93a8ae6c-cc79-45df-8b08-899c017f8135"},"outputs":[{"data":{"text/html":["== Status ==
Memory usage on this node: 2.6/12.7 GiB
Using AsyncHyperBand: num_stopped=308\n","Bracket: Iter 40.000: None | Iter 10.000: 0.206
Resources requested: 4.0/4 CPUs, 0/0 GPUs, 0.0/7.36 GiB heap, 0.0/3.68 GiB objects
Current best trial: a0670_00440 with recall_at_k=0.244 and parameters={'size': 16, 'window': 1, 'ns_exponent': 0.7999999999999996, 'alpha': 0.1, 'negative': 19, 'iter': 10, 'min_count': 1, 'workers': 6, 'sg': 1}
Result logdir: /content/big_HPO_no_distributed
Number of trials: 512/25872 (16 PENDING, 4 RUNNING, 492 TERMINATED)

"],"text/plain":[""]},"metadata":{"tags":[]},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["2021-06-11 09:06:44,670\tERROR tune.py:545 -- Trials did not complete: [tune_w2v_a0670_00492, tune_w2v_a0670_00493, tune_w2v_a0670_00494, tune_w2v_a0670_00495, tune_w2v_a0670_00496, tune_w2v_a0670_00497, tune_w2v_a0670_00498, tune_w2v_a0670_00499, tune_w2v_a0670_00500, tune_w2v_a0670_00501, tune_w2v_a0670_00502, tune_w2v_a0670_00503, tune_w2v_a0670_00504, tune_w2v_a0670_00505, tune_w2v_a0670_00506, tune_w2v_a0670_00507, tune_w2v_a0670_00508, tune_w2v_a0670_00509, tune_w2v_a0670_00510, tune_w2v_a0670_00511]\n","2021-06-11 09:06:44,678\tINFO tune.py:549 -- Total run time: 6750.68 seconds (6746.07 seconds for the tuning loop).\n","2021-06-11 09:06:44,680\tWARNING tune.py:554 -- Experiment has been interrupted, but the most recent state was saved. You can continue running this experiment by passing `resume=True` to `tune.run()`\n"]},{"name":"stdout","output_type":"stream","text":["Best hyperparameters found were: {'dataset': 'ecomm', 'k': 10, 'size': 16, 'window': 1, 'ns_exponent': 0.7999999999999996, 'alpha': 0.1, 'negative': 19, 'iter': 10, 'min_count': 1, 'workers': 6, 'sg': 1}\n"]}],"source":["from ray.tune import Analysis\n","from ray.tune.schedulers import ASHAScheduler\n","\n","# Define the hyperparameter search space for Word2Vec algorithm\n","search_space = {\n"," \"dataset\": \"ecomm\",\n"," \"k\": 10,\n"," \"size\": tune.grid_search(list(np.arange(10,106, 6))),\n"," \"window\": tune.grid_search(list(np.arange(1,22, 3))),\n"," \"ns_exponent\": tune.grid_search(list(np.arange(-1, 1.2, .2))),\n"," \"alpha\": tune.grid_search([0.001, 0.01, 0.1]),\n"," \"negative\": tune.grid_search(list(np.arange(1,22, 3))),\n"," \"iter\": 10,\n"," \"min_count\": 1,\n"," \"workers\": 6,\n"," \"sg\": 1,\n","}\n","\n","use_asha = True\n","smoke_test = False\n","\n","# The ASHA Scheduler will stop underperforming trials in a principled fashion\n","asha_scheduler = ASHAScheduler(max_t=100, grace_period=10) if use_asha else None\n","\n","# Set the stopping critera -- use the smoke-test arg to test the system \n","stopping_criteria = {\"training_iteration\": 1 if smoke_test else 9999}\n","\n","# Perform hyperparamter sweep with Ray Tune\n","analysis = tune.run(\n"," tune_w2v,\n"," name=expt_dir,\n"," local_dir=\"ray_results\",\n"," metric=\"recall_at_k\",\n"," mode=\"max\",\n"," scheduler=asha_scheduler,\n"," stop=stopping_criteria,\n"," num_samples=1,\n"," verbose=1,\n"," resources_per_trial={\n"," \"cpu\": 1,\n"," \"gpu\": 0\n"," },\n"," config=search_space,\n",")\n","print(\"Best hyperparameters found were: \", analysis.best_config)"]},{"cell_type":"markdown","metadata":{"id":"jjdH1oSHrt2p"},"source":["Ray Tune saves the results of each trial in the ray_results directory. Each time Ray Tune performs an HPO sweep, the results for that run are saved under a unique subdirectory. In this case, we named that subdirectory big_HPO_no_distributed. Ray Tune provides methods for interacting with these results, starting with the Analysis class that loads the results from each trial, including performance metrics as a function of training time and tons of metadata.\n","\n","These results are stored as JSON but the Analysis class provides a nice wrapper for converting those results in a pandas dataframe."]},{"cell_type":"markdown","metadata":{"id":"RexEQbOd3TNR"},"source":["## Explore the results of the full hyperparameter sweep\n","Next, we're going to look at how the Recall@10 score changes as a function of various hyperparameter configurations that we tuned over. We tuned over three hyperparameters: the context window size, negative sampling exponent, and the number of negative samples.\n","\n","We want to look at the Recall@10 scores for all of these configurations but this is a 3-dimensional space and, as such, will be difficult to visualize. Instead, we'll \"collapse\" one dimension, while examining the other two. To do this, we aggregate the Recall@10 scores (taking the mean) along the \"collapsed\" dimension."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":949},"id":"O7lbr2-5sebq","outputId":"a2704370-f4b4-464a-87ee-6b4b6f7ed347"},"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
recall_at_ktime_this_iter_sdonetimesteps_totalepisodes_totaltraining_iterationexperiment_iddatetimestamptime_total_spidhostnamenode_iptime_since_restoretimesteps_since_restoreiterations_since_restoretrial_idconfig/alphaconfig/datasetconfig/iterconfig/kconfig/min_countconfig/negativeconfig/ns_exponentconfig/sgconfig/sizeconfig/windowconfig/workerslogdir
00.2375.543207FalseNaNNaN101faab4240d3746cbb4c51569db0690512021-06-11_08-49-15162340135571.839448456843b5d652723e6172.28.0.271.839448010a0670_004190.100ecomm10101190.6116.01.06big_HPO_no_distributed/tune_w2v_a0670_00419_41...
10.2034.436352FalseNaNNaN10fe00d91e28cf49beb45a0f47b900d2ae2021-06-11_07-54-42162339808275.372504222523b5d652723e6172.28.0.275.372504010a0670_001870.010ecomm10101190.6110.01.06big_HPO_no_distributed/tune_w2v_a0670_00187_18...
20.03610.416867FalseNaNNaN9965436df76d64eb1badb38eceda079982021-06-11_08-42-17162340093755.470819431083b5d652723e6172.28.0.255.47081909a0670_003930.001ecomm10101160.4116.01.06big_HPO_no_distributed/tune_w2v_a0670_00393_39...
30.2365.960799FalseNaNNaN98a3090a631b447bd9fb9ee249ef02a302021-06-11_08-52-44162340156454.629899472863b5d652723e6172.28.0.254.62989909a0670_004340.100ecomm10101130.8116.01.06big_HPO_no_distributed/tune_w2v_a0670_00434_43...
40.0411.409388TrueNaNNaN10cce93e60ce3a4ff4a0db54a8e47fa3d92021-06-11_08-06-00162339876038.842165278103b5d652723e6172.28.0.238.842165010a0670_002410.010ecomm1010110-1.0116.01.06big_HPO_no_distributed/tune_w2v_a0670_00241_24...
..........................................................................................
5030.0235.446685FalseNaNNaN39f3c574035ac47ac850519bf18e371842021-06-11_09-01-30162340209022.201976510823b5d652723e6172.28.0.222.20197603a0670_004710.001ecomm1010110-1.0122.01.06big_HPO_no_distributed/tune_w2v_a0670_00471_47...
5040.0152.176694TrueNaNNaN10e927f2c1c10e42908c4c5b26d8ba5b862021-06-11_08-17-32162339945241.176507330623b5d652723e6172.28.0.241.176507010a0670_002940.001ecomm101011-0.4116.01.06big_HPO_no_distributed/tune_w2v_a0670_00294_29...
5050.2111.697303FalseNaNNaN9de953f60dd084976b0a01b62918adea52021-06-11_08-33-35162340041519.104989397823b5d652723e6172.28.0.219.10498909a0670_003590.100ecomm1010110.2116.01.06big_HPO_no_distributed/tune_w2v_a0670_00359_35...
5060.0241.774843FalseNaNNaN4d60beb2097b64164acbcb80fa85800192021-06-11_07-14-48162339568813.84004034543b5d652723e6172.28.0.213.84004004a0670_000030.001ecomm101014-1.0110.01.06big_HPO_no_distributed/tune_w2v_a0670_00003_3_...
5070.1982.985146FalseNaNNaN83ad21430e1004d108a4683096833aeb22021-06-11_07-46-29162339758925.868437189243b5d652723e6172.28.0.225.86843708a0670_001540.010ecomm1010170.4110.01.06big_HPO_no_distributed/tune_w2v_a0670_00154_15...
\n","

508 rows × 29 columns

\n","
"],"text/plain":[" recall_at_k ... logdir\n","0 0.237 ... big_HPO_no_distributed/tune_w2v_a0670_00419_41...\n","1 0.203 ... big_HPO_no_distributed/tune_w2v_a0670_00187_18...\n","2 0.036 ... big_HPO_no_distributed/tune_w2v_a0670_00393_39...\n","3 0.236 ... big_HPO_no_distributed/tune_w2v_a0670_00434_43...\n","4 0.041 ... big_HPO_no_distributed/tune_w2v_a0670_00241_24...\n",".. ... ... ...\n","503 0.023 ... big_HPO_no_distributed/tune_w2v_a0670_00471_47...\n","504 0.015 ... big_HPO_no_distributed/tune_w2v_a0670_00294_29...\n","505 0.211 ... big_HPO_no_distributed/tune_w2v_a0670_00359_35...\n","506 0.024 ... big_HPO_no_distributed/tune_w2v_a0670_00003_3_...\n","507 0.198 ... big_HPO_no_distributed/tune_w2v_a0670_00154_15...\n","\n","[508 rows x 29 columns]"]},"execution_count":112,"metadata":{"tags":[]},"output_type":"execute_result"}],"source":["analysis = Analysis(\"big_HPO_no_distributed/\", \n"," default_metric=\"recall_at_k\",\n"," default_mode=\"max\")\n","\n","results = analysis.dataframe()\n","results"]},{"cell_type":"markdown","metadata":{"id":"FHKEwe9u3LDv"},"source":["The Analysis objects also has methods to quickly retrieve the best configuration found during the HPO sweep.\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WnKAoaER2vkp","outputId":"1f50520a-3267-4d0c-fb99-d7a566fbc95b"},"outputs":[{"data":{"text/plain":["{'alpha': 0.1,\n"," 'dataset': 'ecomm',\n"," 'iter': 10,\n"," 'k': 10,\n"," 'min_count': 1,\n"," 'negative': 19,\n"," 'ns_exponent': 0.7999999999999996,\n"," 'sg': 1,\n"," 'size': 16,\n"," 'window': 1,\n"," 'workers': 6}"]},"execution_count":113,"metadata":{"tags":[]},"output_type":"execute_result"}],"source":["best_config = analysis.get_best_config()\n","best_config"]},{"cell_type":"markdown","metadata":{"id":"tGk8zNkp3Pkr"},"source":["While the results dataframe contains the final Recall@10 scores for each of the 539 trials, it's also nice to explore how those scores evolved as a function of training for any given trial. Again, the Analysis class delivers, providing the ability to access the full training results for any of the trials. Below we plot the Recall@10 score as a function of training epochs for the best configuration."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":276},"id":"y8QF0YZY3GXZ","outputId":"01f09f26-14ee-4003-d621-7135cf1f0861"},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEDCAYAAAA4FgP0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1xUdeI+8GcAQa7C4Ix4QwEvKIRK3kG0DfPXqrtpKvNV1LKv6ZqhfnNTyaStMMW2NNbSrVzTdKWQSHcrysokA/GKghrIZUQRhgEEhvvMnN8f5BSBgcnhAPO8X69ezpnDkYfU88zn3D4yQRAEEBGR2bKQOgAREUmLRUBEZOZYBEREZo5FQERk5lgERERmzkrqAPeipqYGaWlpUCgUsLS0lDoOEVGnYDAYUFRUBF9fX3Tv3r3J+k5VBGlpaViwYIHUMYiIOqUDBw5g9OjRTd7vVEWgUCgANPwwbm5uEqchIuocCgoKsGDBAtM+9Nc6VRHcORzk5uaGfv36SZyGiKhzudshdZ4sJiIycywCIiIzxyIgIjJzLAIiIjPHIiAiMnMsAiIiM8ciICLqwArLa7DjWCaCor7FP09kifI9OtV9BERE5kAQBCRlFePDU2p8mV4IvVHApME9MWWoUpTvxyIgIuogyqrrcfjsDRw4pUZWUSWc7bphSaAH5o91x8Ce9qJ9XxYBEZHELt0ow4fJanyaehM19UaMcnfG3+eOwHS/3ujeTfwHbLIIiIgkUFNvwNHUfHyYrEbqjTLYdrPErFF9sWDcAPj27dGuWVgERETtKLtIhwOnriP27A2UVddjkNIBL80cjtkP9oNT926SZGIREBGJTG8w4tgVDT5MVuP7a1pYWcgwzdcNoeMGYLynHDKZTNJ8LAIiIpEUltfgUEoe/p1yHQXlNejTozuemzoEIWP6Q+nUdIIYqYhaBJs3b0ZqaipkMhnCw8Ph5+dnWpecnIw33ngDFhYW8PDwQGRkJCwsGm5rqKmpwYwZM7BixQrMnj1bzIhERG3qzqWf+5PV+PJyIQxGAUFDFHj5zz74g7cSVpYd7/Yt0YogJSUFarUaMTExyMrKQnh4OGJiYkzrN23ahH379sHNzQ1hYWFITEzE5MmTAQDvvPMOevRo35MlRET3o6yqHrHnGi79zP7p0s+n2uHSz7YgWhEkJSUhODgYAODl5YWysjLodDo4ODgAAOLi4kyv5XI5SktLAQBZWVm4du0apkyZIlY0IqI2c+lGGfYn5+JIar4kl362BdGKQKvVwsfHx7Qsl8tRVFRk2vnf+VWj0eDkyZNYtWoVAGDr1q148cUXER8fL1Y0IqL7Ul1nwNGL+TjQAS79bAvtdrJYEIQm7xUXF2P58uWIiIiAi4sL4uPjMXLkSPTv37+9YhERtVpHvPSzLYhWBEqlElqt1rSs0WgaTZys0+mwdOlSrF69GoGBgQCA48ePIy8vD8ePH0dBQQGsra3h5uaGiRMnihWTiOg31RuM+PpKIT5Mvt4hL/1sC6IVQUBAAKKjo6FSqZCeng6lUmk6HAQAW7ZsweLFixEUFGR6b/v27abX0dHR6Nu3L0uAiNpVnd6ISzdvIzm7BKdySnAmtwRVdYafL/0c2x9Kx45z6WdbEK0I/P394ePjA5VKBZlMhoiICMTFxcHR0RGBgYGIj4+HWq1GbGwsAGDGjBkICQkRKw4RUbNq9Qak5pXhVHYxTuWU4Ky6FNX1BgDAkF4OeNy/H6YMVWDyEEWHvPSzLYh6jmDt2rWNlr29vU2v09LSfnPbZ599VpRMRGTeauoNOH/9Nk7lFONUdgnOXS9Frd4IAPB2c0TImP4Y7ynHmIFyuDrYSJy2ffDOYiLq0qrrDDh3vRTJ2Q07/gt5t1FnMEImA4b3dkLo+AEY5yHHWA85nO2spY4rCRYBEXUplbV6nFX/tOPPKcHFG7dRbxBgIQN8+/bAEwEDMc5DjtED5ehh23mv9GlLLAIiaqSksg4xp/NgbWUBV3tryH/6z9Wh4Vcbq451k1RFTT3OqH/+xJ92swx6owBLCxke6NsDTwV6YpynHKMHuMCxE1/iKSYWARGZ3CitwqI9Kcguqrzr1zjYWP1cDqaCsPm5NBysTa9d7W1ga922xVFWXY/TOSUNx/hzGnb8RgHoZimDXz9nLJvsiXEernhwgAvsbbiLaw3+XyIiAMDVgnIs3pOC6joDYp4ejyG9HFFcWYeSyjqUVNY2vNbV/eK9OuSX1SAtvwwllXWoNzS9aRQAbLtZNhpR/FwgNqbXd0pD7mANe2vLRtfm366qw6mcEpzKbtj5X75VDkEArC0tMLK/M1Y+NAjjPF3h7+7S5qVjLlgERISUnBI89cFp2Flb4uPlEzHUzREA4GLfupOngiCgvEb/c2noGoril6VRXFkHra4WGQUVKK6sM12p82u/PCSlNwjI0FRAEAAbKwuMcnfGqocHY5yHK0a5O3eaZ/l0dCwCIjOXkF6AZ/99Hv1cbLFvyVj0c7G7599DJpOhh2039LDtBo9WPGlTEARU1RlQrKtDcWVt09LQNRSKUQBm+PXGOE9XjOjfo8Odn+gqWAREZuzgqevYGH8Jfv2cseeJMZC3cgRwv2QyGextrGBvYwV313svHmpbLAIiMyQIAqK/uYY3vsrAlKEKvL3AH3bW3B2YK/7JE5kZg1HAS0fSsT9Zjdn+fbH1cT9066KPTqDWYREQmZGaegP+76ML+OxSAZYFeWL9o95d4umZdH9YBERmorymHk/vO4Pk7BJsnD4M/zvJU+pI1EGwCIjMgKaiBk/sOY2MwgpsDxmJx0b1lToSdSAsAqIuLkdbiUV7TqFYV4f3nxiDyUMULW9EZoVFQNSFXbpRhif+lQIBwMGl4zGyv7PUkagDYhEQdVGJmUVYvv8snO2ssf+psfBUOLS8EZklFgFRF3QkNR/PfXQBXgoHfLBkLHo5da2pFaltsQiIupg93+fg5f9cxlgPOd5dNJrP3KcWsQiIughBEBCV8CPeOZ6FaT69sEM1ig9lo1YRtQg2b96M1NRUyGQyhIeHw8/Pz7QuOTkZb7zxBiwsLODh4YHIyEjU1tZi/fr1KC4uRm1tLVasWIGHHnpIzIhEXYLeYMSGuEv4+OwNzB/njlf+7AtLC94oRq0jWhGkpKRArVYjJiYGWVlZCA8PR0xMjGn9pk2bsG/fPri5uSEsLAyJiYmorKyEr68vli5dips3b2LJkiUsAqIWVNcZsPLgOXx9VYNVDw/G6uDBvFuY7oloRZCUlITg4GAAgJeXF8rKyqDT6eDg0HDlQlxcnOm1XC5HaWkpHnvsMdP2t27dQq9evcSKR9Ql3K6qw5K9p3E+7zZefcwXoeMHSB2JOiHRikCr1cLHx8e0LJfLUVRUZNr53/lVo9Hg5MmTWLVqlelrVSoVCgoKsGvXLrHiEXV6+bersWhPCq4XV+Ht+f549IHeUkeiTqrdThYLQtNp7IqLi7F8+XJERETAxcXF9P6hQ4dw5coV/PWvf8WRI0c4zCX6lYzCCizekwJdjR4fLBmLCV6uUkeiTky0Z88qlUpotVrTskajgULx863tOp0OS5cuxerVqxEYGAgASEtLw61btwAAw4YNg8FgQElJiVgRiTqls+oSzN2VBL1RQMyyCSwBum+iFUFAQAASEhIAAOnp6VAqlabDQQCwZcsWLF68GEFBQab3zpw5gz179gBoOLRUVVXVaKRAZO6OXS7EgvdOQW5vjbi/TMTwPk5SR6IuQLRDQ/7+/vDx8YFKpYJMJkNERATi4uLg6OiIwMBAxMfHQ61WIzY2FgAwY8YMqFQqvPDCC5g/fz5qamqwadMmWFhwwgwiAPjoTB42xF2CTx8n/OuJMXB1sJE6EnURop4jWLt2baNlb29v0+u0tLRmt/n73/8uZiSiTkcQBLx9PAvbEn7EpME9sSv0Qdjb8F5Qajv820TUgRmNAl7+z2Xs/SEXfx7ZB9vmjIC1FUfJ1LZYBEQdVK3egLUfX8TR1Hw8FeiBF/44DBa8W5hEwCIg6oB0tXos338W31/TYv2j3lgW5MnLqEk0LAKiDkarq8WT/zqNy7fK8frcEZjzYD+pI1EXxyIg6kCuF1dh0Z5TKCivwbuLHsQfvPmYFRIfi4BIQoIgILe4Cqeyi3EqpwTfXNVAJmuYVtLfnffQUPtgERC1I0EQkFVUiVM5xTiVXYLk7GJoKmoBAD0drBE4uCfWBA/BICWnlaT2wyIgEpEgCMjU6HAquxjJ2SU4lVMCra5hx690tMF4T1eM85RjnIcrvBT2PCFMkmARELUho1HAj4UVph1/Sm4JSirrAAC9e3THpME9Mc5DjnGerhjoascdP3UILAKi+2AwCrhyqxynchoO85zOLcHtqnoAQF9nWzw0VIlxnnKM93BFf7ktd/zUIbEIiO6B3mDE5VvlpuP7KbklqKjRAwDc5XZ4ZHgvjPNoONzTz8VO4rRErcMiIPoN9QYj0m6WmT7xn8ktha62Ycfv2dMeM/x6m3b8vXvYSpyW6PdhERD9Qp3eiEs3byP5p0/8Z9WlqKozAAAGKR3w55F9MM7TFeM95FA6dZc4LVHbYBGQ2arTG5FbXImMwgpkFOpwVl2Cs+pS1NQbAQBDezlizoP9MM7DFWM95FA48rHP1DWxCKjLq6k3ILuoEpmaClzT6JBZqEOmpgK5xVUwGBumUJXJAG83J6jGuGO8pxxjPVwht7eWODlR+2ARUJdRXWdAVlHDTr5hZ6/DNY0O6uJK/LS/h6WFDANc7TBY6YBHfXtjcC8HDFY6wlNhj+7dLKX9AYgkwiKgTqeyVt/wyV7TsNO/VqhDhqYCN0qrIfy0w7eykMGjpz2G9XbEn0b0Me3wB/a0g40Vd/hEv8QioA6rvKYe1zQ6XPvpUE7mT4d1bt6uNn2NtaUFPBX2GNnfBXMf7I/BSgcM7uWAAa726GbJCVyIWoNFQJIrq6pvtKO/c2inoLzG9DU2VhbwUjhgzEAXzO/ljkFKBwxWOsBdbgcr7vCJ7ouoRbB582akpqZCJpMhPDwcfn5+pnXJycl44403YGFhAQ8PD0RGRsLCwgJRUVE4e/Ys9Ho9li1bhkceeUTMiCQhg1HAigNnkZBeaHrPtpslBvdywMRBrhisdDR9wu/nYgdLzs5FJArRiiAlJQVqtRoxMTHIyspCeHg4YmJiTOs3bdqEffv2wc3NDWFhYUhMTISNjQ0yMzMRExOD0tJSzJo1i0XQhUV/k4mE9EI8FeiBwEE9MUjpgL7OtpyOkaidiVYESUlJCA4OBgB4eXmhrKwMOp0ODg4Nj9eNi4szvZbL5SgtLcXMmTNNowYnJydUV1fDYDDA0pIn97qa7zO12PF1Jmb798XG6cP4DB4iCYl2cFWr1cLF5eeJNeRyOYqKikzLd0pAo9Hg5MmTmDx5MiwtLWFn1/B8ltjYWAQFBbEEuqCCshqsOnQegxQOePUxX5YAkcTa7WSxcOe6vl8oLi7G8uXLERER0ag0jh07htjYWOzZs6e94lE70RuMePbf51Bdb8A7of6ws+b1CkRSE+1foVKphFarNS1rNBooFArTsk6nw9KlS7F69WoEBgaa3k9MTMSuXbvw3nvvwdHRUax4JJHXv8zA6dxS7FCNxCAl/3yJOgLRDg0FBAQgISEBAJCeng6lUmk6HAQAW7ZsweLFixEUFGR6r6KiAlFRUdi9ezecnZ3FikYS+fpKIXZ9l4X549zx55F9pY5DRD8RbUTg7+8PHx8fqFQqyGQyREREIC4uDo6OjggMDER8fDzUajViY2MBADNmzAAAlJaWYvXq1abfZ+vWrejTp49YMamd5JVU4f8+SoVPHydsmjFc6jhE9AuiHqBdu3Zto2Vvb2/T67S0tGa3CQkJETMSSaBOb8TKg+dgNAp4e4E/n+lD1MHwTB2JbvNnV5B6owy7Qv0xwNVe6jhE9Cu8N59E9d+Lt7D3h1wsCfDA//PtLXUcImoGi4BEk6OtxLrDFzHK3RnrH/VueQMikgSLgERRU2/AXz48CytLGf4x3x/WVvyrRtRR8RwBieKlI+m4WlCBfz05Bn2dOak7UUfWYhHU19fj8OHD+OGHH0yPiFAqlZg0aRJmzZrFR0BQE4fP3sCh03l45iEvPDRUKXUcImpBi0Xw/PPPw93dHUuWLIGrqysEQUBhYSESEhKwYcMGREVFtUdO6iQyCiuwMT4N4zzkWBM8ROo4RNQKLRZBUVER3nzzzUbvubu7Y8yYMQgNDRUtGHU+lbV6/OXDs7C3sUL0/4zihDFEnUSL/1JlMhm+/PJL1NfXm96rq6vD0aNHYW1tLWo46jwEQUD4J5eQo63EW/8zEkqn7lJHIqJWanFEsG3bNuzYsQNbt25FTU3D1IF2dnaYMGECtm7dKnpA6hwOplzHpxfy8dzUIZjo1VPqOER0D1osAjc3N7z22mvNrisvL2/zQNT5pN0sw9+OXEbQEAWeeWiQ1HGI6B7d10HclStXtlUO6qTKa+qx4sA5yO2tsT1kJKeZJOqEWhwRHDhw4K7rCgsL77qOuj5BEPDXj1ORf7saMcvGQ27Pc0ZEnVGLRbB3715MmDABSmXT68H1er0ooahz2HMyFwnphdg4fRgeHCCXOg4R/U4tFsHOnTvx6quvYuPGjU2uEjp16pRowahjO3e9FK99dgVTh/fCU4EeUschovvQ4jmCIUOGYPfu3bCyatoZ69evFyUUdWyllXVYeeAcejt3x+tzR3DyeaJOrlUni21tbWFhYYG6ujrcvn3b9L6Pj49owahjMhoFrPnoArS6Orw9/0H0sO0mdSQiuk+teuhcfHw8Pv/8c7i5ucHJyQklJSVwcnLCypUrYW/PiUbMyTvfZeH4j0V45TFfPNCvh9RxiKgNtFgER48exZkzZzB79mw88MADpvmDb926hVdffRUTJ07EpEmTONm8GUjKKsbfv/wRM0f0Qeg4d6njEFEbafHQ0Mcff4xNmzbh9OnTmD59OqZNm4b58+cjMjISNTU16N27910vMd28eTNCQkKgUqlw8eLFRuuSk5Mxb948qFQqbNiwAUajEQCQkZGB4OBgfPjhh23w41Fb0VTUIOzQeQzsaY/XZj/A8wJEXUirzhFYW1vj6tWrOH78OBISEhAZGQm5XI6lS5fC398f586da7JNSkoK1Go1YmJiEBkZicjIyEbrN23ahLfeeguHDh1CZWUlEhMTUVVVhVdeeQUTJkxom5+O2oTBKGDVvy+goqYeby/wh4MNp7Eg6kpaLII7n9T1er3pfICHhwciIiLw3nvvwcLCotED6e5ISkpCcHAwAMDLywtlZWXQ6XSm9XFxcXBzcwMAyOVylJaWwtraGu+++26z9yyQdLYfy0BSdjFe+bMvvN2cpI5DRG2sxY92fn5+OHHiBFauXInFixdj5syZcHd3R25uLuzt7VFRUYFu3ZpeOaLVahtdVSSXy1FUVAQHBwcAMP2q0Whw8uRJrFq1ClZWVs1epkrSOf6jBtHfXMO80f0wd3R/qeMQkQhaHBEsX74cu3btgq2tLXbv3g17e3ukp6fD0dER4eHhePnllzFnzpwWv5EgCE3eKy4uxvLlyxEREQEXF5ff9xOQaPJvV2NNzAV4uznib3/ylToOEYmkxY/fTk5O2LlzJ6KiolBaWgp/f384OzsjNzcXTz31FGbMmIFHH320yXZKpRJarda0rNFooFAoTMs6nQ5Lly7F6tWrERgY2EY/DrWVeoMRz/77POr0Ruxc4A9ba05JStRVteo4jIuLC1577TXodDpcunQJxcXFGDFiBJ588knTIZ5fCwgIQHR0NFQqFdLT06FUKht97ZYtW7B48WIEBQW1zU9CbSrqi6s4qy5F9P+Mgpei+T9jIuoaWiyCa9euNVpWKBSmT/YFBQUYNKj558/7+/vDx8cHKpUKMpkMERERiIuLg6OjIwIDAxEfHw+1Wo3Y2FgAwIwZM+Dj44OtW7fi5s2bsLKyQkJCAqKjo3mPQjv7Mr0A7ybmYNGEAZg5oo/UcYhIZC0Wwd/+9re7rpPJZNi3b99d169du7bRsre3t+l1Wlpas9vs37+/pUgkouvFVXju41T49euBF6YPkzoOEbWDFouAO2bzUVNvwIqDZyEDsHO+P2yseF6AyBy0WATjx49v9i5SQRAgk8mQlJQkSjBqf5H/vYK0m+V4d9Fo9JfbSR2HiNpJi0WQnJx813UnT55s0zAknSOp+difrMbTQZ6YOryX1HGIqB21+u6tvLw8HDx40PQY6vr6epw+fRrfffedaOGofVzT6LD+8EWMHuCCv04bKnUcImpnrZ68fv369Rg0aBDS09MxZcoUWFhY4OWXXxYzG7WD6joDnjlwDt27WSJ6/ih0s2z1Xwki6iJa/a/eysoKjz/+OJycnDBt2jRERUXxCaFdwIufpiFDU4HtISPRu4et1HGISAKtPjQkCAJSUlLg7OyMmJgYuLu748aNG2JmI5F9dCYPsWdvIOzhwQgaomh5AyLqklo9Iti2bRvs7OywceNGXLhwAfv27cO6devEzEYiunSjDC/Gp2GilytWPTxY6jhEJKFWF4GTkxMKCwvh5uaG1157DdOmTcPYsWPFzEYi0BuM2PntNTz+zg9wsbPGDtUoWFpwkhkic9bqIlizZk2jQ0G1tbV47rnnRAlF4rhaUI5Zb/+AbQk/Ini4EkefDYTC0UbqWEQksVafI6ioqMDixYtNyyEhIfjPf/4jSihqW3V6I945noV/fJuJHrbd8PYCf/zxgd5SxyKiDqLVReDg4IAPP/wQ/v7+MBqNSE5OhqOjo5jZqA2k3SzD2o9TcbWgAn8e2QcRM30gt7eWOhYRdSCtLoLXX38d77//PrZv3w4LCws88MADiIqKEjMb3YdavQFvfZ2JXd9lw9XeGu8uGs07homoWa0uAkdHR6hUKty4cQOjR49GXV0drK35ybIjOn+9FM/HXkSmRoe5D/bDxunD0cOu6XSiRETAPRTB3r178cUXX6C6uhqffvoptm3bBoVCgaefflrMfHQPauoNeOOrDLyXmI1eTt2x98kxmDJUKXUsIurgWn3V0LFjx3Do0CE4OTkBAMLDw/H111+LFozuzencEjy6IxH/PJEN1Vh3fLkmiCVARK3S6hGBwWAAANMjqWtra6HX68VJRa1WVadH1Bc/4oOkXPR1tsWB/x2HgEE9pY5FRJ1Iq4tgxowZWLRoEdRqNSIiIpCcnIwnnnhCxGjUkh+ytFh3+CLySqrxxMSB+Ou0obC3afUfKRERgFYUgSAIOHr0KEpKSvDwww9DoVDA2toay5cvx6FDh9ojI/1KRU09tnx+FQdOXcdAVzt8tGwCxnrIpY5FRJ1Ui+cIIiIikJSUhJ49eyIxMRH5+fkAgCeffLLF33zz5s0ICQmBSqXCxYsXG61LTk7GvHnzoFKpsGHDBhiNxha3IeC7jCJMe/ME/p1yHUsneeDzVUEsASK6Ly2OCDIyMkyf/OfMmYPAwECMHz8e7733Hvr163fX7VJSUqBWqxETE4OsrCyEh4cjJibGtH7Tpk3Yt28f3NzcEBYWhsTERNja2v7mNuasrLoekf+9jI/O3MAgpQNi/zIR/u4uUscioi6gxSLo1q1bo9dDhgzBjh07WvyNk5KSEBwcDADw8vJCWVkZdDodHBwcAABxcXGm13K5HKWlpbhw4cJvbmOuvr5SiPBPLkGrq8OKKV4Ie3gwunfjxPJE1DZaPDT064nrm5vIvjlarRYuLj9/YpXL5SgqKjIt39m5azQanDx5EpMnT25xG3NTWlmH1YfO46kPzsDFzhrxKwLw/P/zZgkQUZtqcUSQlpaGOXPmAGg4cZyTk4M5c+ZAEATIZDLExsa26hsJgtDkveLiYixfvhwRERGNCuC3tjEXX6Tdwsb4dNyuqsPq4MFYMWUQrK04jSQRtb0Wi+Do0aO/6zdWKpXQarWmZY1GA4Xi51mwdDodli5ditWrVyMwMLBV25gDra4WEZ+m47+XbsG3rxP2PzUWw3o7SR2LiLqwFougb9++v+s3DggIQHR0NFQqFdLT06FUKhsd69+yZQsWL16MoKCgVm/TlQmCgCOp+XjpSDoqaw3467SheDrIk5PJE5HoRLv7yN/fHz4+PlCpVJDJZIiIiEBcXBwcHR0RGBiI+Ph4qNVq06GlGTNmICQkpMk25kBTXoMX4tPw1eVCjOzvjG1z/DC4Fx/xTUTtQ9TbUNeuXdto2dvb2/Q6LS2tVdt0ZYIg4PC5m3j5aDpq9Ua88MdhWBLowakjiahd8XkEErlVVo0NcZdw/McijBnogq2P+8FTYR6HwYioY2ERtDNBEHDodB42//cK9EYBL80cjkUTBsKCowAikgiLoJ3tS1Ij4kg6Jni6YuvjfnB3tZM6EhGZORZBO6quMyD6m2sY5yHHgf8dx1EAEXUIvDaxHR04pYZWV4v/mzqEJUBEHQaLoJ1U1enxzvEsBA7qiXGerlLHISIyYRG0k/1JahRX1mHN1MFSRyEiaoRF0A4qa/XYfSIbQUMUeHAA5w4goo6FRdAOPkjKRUllHdYEczRARB0Pi0BkFTX1+OeJbDw0VIFRnEiGiDogFoHI9p7Mxe2qeqwOHiJ1FCKiZrEIRFReU493E7MRPEyJEf2dpY5DRNQsFoGI9nyfg/IaPUcDRNShsQhEUlZVj/e/z8Ejw3vBt28PqeMQEd0Vi0Ak73+fjQqOBoioE2ARiOB2VR32nMzFo75uGN6H00wSUcfGIhDBu4nZqKzjaICIOgcWQRsrqazD3pO5+OMDvTHUjdNNElHHJ2oRbN68GSEhIVCpVLh48WKjdbW1tVi3bh1mz55tes9oNOLFF1+ESqXCwoULkZWVJWY8UfzzRDaq6g1Y/TDvIiaizkG0IkhJSYFarUZMTAwiIyMRGRnZaH1UVBSGDRvW6L2vv/4aFRUVOHToECIjIxEVFSVWPFFodbX44Idc/GlEH04+T0SdhmhFkJSUhODgYACAl5cXysrKoNPpTOvXrFljWn9Hbm4u/Pz8AADu7u7Iz8+HwWAQK2Kb++eJbNTqDQjjaICIOhHRikCr1cLF5edn68jlchQVFZmWHRyaTtQ+ZMgQfP/995EVCP0AAA0kSURBVDAYDMjOzkZeXh5KS0vFitimNBU12JeUi8dG9oUXJ6Enok6k3aaqFAShxa+ZPHkyzp07hwULFmDo0KHw9PRs1XYdwe7vslFvEPAsRwNE1MmIVgRKpRJarda0rNFooFAoWtxuzZo1ptfBwcFwde34s3lpymvwYbIas0b1hUdPe6njEBHdE9EODQUEBCAhIQEAkJ6eDqVS2ezhoF+6evUqNmzYAAA4ceIEhg8fDguLjn+F69vHs6A3Cnj2D4OkjkJEdM9EGxH4+/vDx8cHKpUKMpkMERERiIuLg6OjI6ZOnYqwsDAUFBQgJycHCxcuxLx58zB9+nQIgoA5c+bAxsYGr7/+uljx2kxBWQ0OplzHHP9+GODK0QARdT6iniNYu3Zto2Vvb2/T67feeqvZbbZs2SJmpDb39vFrMBoFrORogIg6qY5/3KUDu3m7GodS8jB3dH/0l9tJHYeI6HdhEdyHnd9egwCOBoioc2MR/E55JVX4+EweQsb0R19nW6njEBH9biyC32nnt9cggwzPPMTRABF1biyC3+F6cRViz97A/HHu6N2DowEi6txYBL9D9DeZsLSQ4S9TvKSOQkR031gE9yhXW4m48zexYNwA9HLqLnUcIqL7xiK4R299k4luljIsn+IpdRQiojbBIrgH2UU6xJ+/iYXjB0DpyNEAEXUNLIJ78NbXmbCxssSyyTw3QERdB4ugla5pKvBpaj4WTRyAng42UschImozLIJW2vH1Ndh1s8SyII4GiKhrYRG0QkZhBf5zMR+LJw6E3N5a6jhERG2KRdAKO45lwt7aCksn8UohIup6WAQtuHKrHP+9dAtPBgyEC0cDRNQFsQhasONYJhxtrPC/gRwNEFHXxCL4Den5ZfgivQBLAj3Qw66b1HGIiETBIvgN249lwqm7FZYEekgdhYhINCyCu7h0owxfXS7E0kme6GHL0QARdV2izlm8efNmpKamQiaTITw8HH5+fqZ1tbW12LRpEzIzMxEXFwcAqKysxLp161BWVob6+no888wzmDRpkpgR7+rNYxlwtuuGJwIGSvL9iYjai2gjgpSUFKjVasTExCAyMhKRkZGN1kdFRWHYsGGN3vvkk0/g4eGB/fv3Y8eOHU22aS8X8m7jm6saLJ3kCcfuHA0QUdcmWhEkJSUhODgYAODl5YWysjLodDrT+jVr1pjW3+Hi4oLbt28DAMrLy+Hi4iJWvN/05lcZcLHrhsUTB0ry/YmI2pNoRaDVahvtyOVyOYqKikzLDg4OTbaZPn068vPzMXXqVISGhmLdunVixburs+pSfJdRhGWTveBgI+qRMyKiDqHdThYLgtDi13z66afo06cPvvrqK3zwwQd4+eWX2yFZY9uPZcDV3hqLJgxo9+9NRCQF0YpAqVRCq9WaljUaDRQKxW9uc+7cOQQGBgIAvL29odFoYDAYxIrYxOncEiRmarF8shfsrDkaICLzIFoRBAQEICEhAQCQnp4OpVLZ7OGgXxowYABSU1MBADdv3oS9vT0sLS3FitjEm19loKeDDULHczRAROZDtI+9/v7+8PHxgUqlgkwmQ0REBOLi4uDo6IipU6ciLCwMBQUFyMnJwcKFCzFv3jyEhIQgPDwcoaGh0Ov1eOmll8SK10RydjF+yCrGizOGw9a6/cqHiEhqoh7/WLt2baNlb29v0+u33nqr2W127NghZqRmCYKAN77KgNLRBgvGubf79ycikhLvLAaQlFWMlJwSrJjihe7dOBogIvNi9kUgCALePJYBN6fuUI3laICIzI/ZF8H317Q4nVuKZx7iaICIzJNZF4EgCHjzqwz06dEd88b0lzoOEZEkzLoIvssowrnrt7HyD4NhY8XRABGZJ7MtgoZzA5no62yLOQ/2kzoOEZFkzLYIvv1Rg9S82wh7eBCsrcz2fwMRkXkWQcO5gUy4y+0w25+jASIyb2ZZBMeuaHDpZhme/cMgdLM0y/8FREQmZrcXvHOl0EBXO8wa1VfqOEREkjO7IkhIL8TlW+UIe3gwrDgaICIyryIwGgVsP5YBT4U9/jSij9RxiIg6BLMqgi/SC3C1oAKrOBogIjIxm73hndHAIKUDZvhxNEBEdIfZFEFxZR1ytJVY+8hQWFrIpI5DRNRhmM18jApHG5zf9AgnpCci+hWzGREAYAkQETXDrIqAiIiaYhEQEZk5UY+VbN68GampqZDJZAgPD4efn59pXW1tLTZt2oTMzEzExcUBAD7++GMcOXLE9DVpaWk4f/68mBGJiMyeaEWQkpICtVqNmJgYZGVlITw8HDExMab1UVFRGDZsGDIzM03vzZ07F3PnzjVt//nnn4sVj4iIfiLaoaGkpCQEBwcDALy8vFBWVgadTmdav2bNGtP65uzcuRMrVqwQKx4REf1EtCLQarVwcXExLcvlchQVFZmWHRwc7rrtxYsX0bt3bygUCrHiERHRT9rtekpBEFr9tbGxsZg1a1aT9w0GAwCgoKCgzXIREXV1d/aZd/ahvyZaESiVSmi1WtOyRqNp9Sf8U6dOYePGjU3evzOiWLBgQduEJCIyI0VFRRgwYECT90UrgoCAAERHR0OlUiE9PR1KpfI3DwfdUVhYCHt7e1hbWzdZ5+vriwMHDkChUMDSkpPNExG1hsFgQFFREXx9fZtdL1oR+Pv7w8fHByqVCjKZDBEREYiLi4OjoyOmTp2KsLAwFBQUICcnBwsXLsS8efMwc+ZMFBUVQS6XN/t7du/eHaNHjxYrMhFRl9XcSOAOmXAvB++JiKjLMZs7izdv3oyQkBCoVCpcvHhR6jiSi4qKQkhICB5//HF8+eWXUseRXE1NDYKDg003N5qzI0eO4E9/+hNmz56N48ePSx1HUpWVlVi5ciUWLlwIlUqFxMREqSOJwiyewtbSzW3mJjk5GZmZmYiJiUFpaSlmzZqFRx55ROpYknrnnXfQo0cPqWNIrrS0FDt37sThw4dRVVWF6OhoTJkyRepYkvnkk0/g4eGB5557DoWFhVi8eDG++OILqWO1ObMogrvd3Naak9dd0ZgxY0yP+3ByckJ1dTUMBoPZnoDPysrCtWvXzHqHd0dSUhImTJgABwcHODg44JVXXpE6kqRcXFzw448/AgDKy8sb3RvVlZjFoaGWbm4zN5aWlrCzswPQcM9GUFCQ2ZYAAGzduhXr16+XOkaHcOPGDdTU1GD58uWYP38+kpKSpI4kqenTpyM/Px9Tp05FaGgo1q1bJ3UkUZjFiODXeH68wbFjxxAbG4s9e/ZIHUUy8fHxGDlyJPr37y91lA7j9u3b+Mc//oH8/HwsWrQI3377LWQy85zV79NPP0WfPn3w/vvv4+rVqwgPD++S55HMogju5+a2rioxMRG7du3Ce++9B0dHR6njSOb48ePIy8vD8ePHUVBQAGtra7i5uWHixIlSR5OEq6srRo0aBSsrK7i7u8Pe3h4lJSVwdXWVOpokzp07h8DAQACAt7c3NBpNlzyMahaHhgICApCQkAAA93RzW1dVUVGBqKgo7N69G87OzlLHkdT27dtx+PBhfPTRR5g7dy5WrFhhtiUAAIGBgUhOTobRaERpaSmqqqq67HHx1hgwYABSU1MBADdv3oS9vX2XKwHATEYEzd3cZs4+++wzlJaWYvXq1ab3tm7dij59+kiYijqCXr16Ydq0aZg3bx4AYOPGjbCwMIvPi80KCQlBeHg4QkNDodfr8dJLL0kdSRS8oYyIyMyZb9UTEREAFgERkdljERARmTkWARGRmWMREBGZObO4fJToXt24cQMzZ85sMpFHdHT0fd17ER0dDRcXF4SGht5vRKI2wyIgugsPDw/s379f6hhEomMREN2D9evXw87ODtnZ2SgtLcVrr72G4cOH44MPPsBnn30GAHj44Yfx9NNP4+bNm1i/fj0MBgP69OmDrVu3AgAyMjKwbNky5Obm4oUXXkBQUJCUPxIRzxEQ3Su9Xo+9e/di1apV2LlzJ/Ly8vDJJ5/gwIEDOHDgAD7//HNcv34db775Jp544gkcPHgQSqUSaWlpABoe6rZ7925s3LgRhw4dkvinIeKIgOiu7synfYeHhwcAmJ5FNHLkSLz++uu4cuUKRowYASurhn9O/v7+uHr1Ki5fvowXXngBAPD8888DAE6cOAF/f38ADY9zqKioaLefh+huWAREd9HcOYL169fDaDSalmUyGWQyWaNHm9fX18PCwgKWlpbNPvL8TmEQdRQ8NER0j86ePQsAOH/+PLy8vDBs2DBcuHABer0eer0eqampGDZsGHx9fZGcnAwA2LFjB3744QcpYxPdFT+aEN3Frw8NAUD37t1hZWWFZcuW4datW9i2bRv69euHkJAQhIaGQhAEzJ07F3379kVYWBg2bNiAgwcPonfv3li5cqWpRIg6Ej59lOgerF+/HtOmTcNDDz0kdRSiNsNDQ0REZo4jAiIiM8cRARGRmWMREBGZORYBEZGZYxEQEZk5FgERkZljERARmbn/D3R3bpm9gBZpAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"tags":[]},"output_type":"display_data"}],"source":["best_path = analysis.get_best_logdir()\n","dfs = analysis.fetch_trial_dataframes()\n","\n","plt.plot(dfs[best_path]['recall_at_k']);\n","plt.xlabel(\"Epoch\")\n","plt.ylabel(\"Recall@10\");"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"z6sCIiAK3Rmt"},"outputs":[],"source":["def aggregate_z(x_name, y_name):\n"," grouped = results.groupby([f\"config/{x_name}\", f\"config/{y_name}\"])\n"," x_values = []\n"," y_values = []\n"," mean_recall_values = []\n"," \n"," for name, grp in grouped:\n"," x_values.append(name[0])\n"," y_values.append(name[1])\n"," mean_recall_values.append(grp['recall_at_k'].mean())\n"," return x_values, y_values, mean_recall_values"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":327},"id":"6MDZY3uQ3xDA","outputId":"11d3cdd2-949c-4bc7-ee82-24ee63c5ba46"},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABDAAAAFgCAYAAABNIolGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXwUVbr/8U919ZJ09o0kJGHfN4dFFEEcBRQB/cmIjgvoKFyQuI2KqIigouIyuAwKOoiyyHXc0FHh4jAw4BU3BB2BEVD2ELaQBLL3+vuDaw8xARrp7oTwfb9evEifqq7naZqcOv30qVOG3+/3IyIiIiIiIiJSj1nqOgERERERERERkRNRAUNERERERERE6j0VMERERERERESk3lMBQ0RERERERETqPWtdJxBqlZWVrF+/nrS0NEzTrOt0RESOyev1cuDAATp16kRUVFRdpxMW6pNF5HSg/lhEpP44Xp/c4AoY69ev5/rrr6/rNEREgrZgwQJ69OhR12mEhfpkETmdqD8WEak/auuTG1wBIy0tDTjyYjMyMuo4GxGRY9u7dy/XX399oN9qiNQni8jpQP2xiEj9cbw+ucEVMH6eEpeRkUF2dnYdZyMicmINeSqv+mQROZ2oPxYRqT9q65O1iKeIiIiIiIiI1HsqYIiIiIiIiIhIvacChoiIiIiIiIjUew1uDQwRObP5/X6+KC7m6+JiSrwe4kwrPRMT6ZWYiGEYdZ2eiIjIaUHnUxGpj1TAEJEGwe3z8VpeHjN27qDC62VgWhrxVisHXOXM2LmDaNMkt0lTbs7OxmbR5DMREZHa6HwqIvWZChgictor9XgY9u1aqnw+prVrz0UpKViO+nbI5/ez7GABj2/Zwvv79vJu127EWtX9iYiIHE3nUxGp71Q2FZHTmtvnY9i3a8l0OFh6dk/6p6ZWG2wBWAyDAalp/OPsnmQ4HAz7di1un6+OMhYRaZj8VeW4/uc5/FXldZ2K/Ao6n4rI6UAFDBE5rb2Wl0eVz8esTp2xnmAqq9Vi4dVOnan0+Xg9Ly9CGYqInBl829fi27YG3/a1dZ2K/Ao6n4rI6UBzvoBNuw/wQ95+rKaFjjnpNE9PDlusXWUH2XAon3JPFW3jM2iX0BjTCE8d6bCrkLyKrRx2F5LqyCTb2YooMzossVzeMgqrfuCwayfR1jRSotrjtKaGJZbf76WoaiPFVT9iNaNJdnQg1pYTllgApa6tlFRtwIebOHsH4h3twhbL7dlNletf+HwHsdna4rB1xmIJz3vm9Rbhdv8Lj2cHVmsTbLazMM3w/N/3+auodK2jyr0J05JIlO0s7LbsUz6u3+9nxs4dTGvXvtpgy+/ehN+9AbBg2DthWFsFtlktFh5s2ZJxGzfyXzk5p7wQmcu9gwr3v/D5SnHY2hNt74xhnN5da15eHpdccgk5Of/5verSpQtPP/10tf02btzIww8/TFFREUlJSTz88MO0axe+3w8Rqd88P6w88vfGTzHb9qnjbORk1HY+9ftd4F6P3/MjhiUBbJ0xzKzAc0J9PhURCcbpPcoOge937GHkS+9Q5fYCkBgTxau5w2jTOC3ksXaUHiT36znsrigGwDQszOh5A+ektgx5rDJ3Ce/vns2Gw18H2i5rfCO9UwdhCXHBxO/3seXwR3xT8HygLSfmt5zb6AGirAkhjQWwv2IN/7vnNvwcec+c1sb0zXyROHvTkMcqqdrMmr034vYdec8shoPuGXNIjPpNyGN5vHvYXziaKtd/vrlKS5pBXMzQkMfy+SopLZ1OWenLgbaYmJuJi5+AxeIMebzSin+Qd3B04HGU7SyyU1/Fbm18Ssf9oriYCq+Xi1JSAm1+1/f4i0aAv+LIYyMekudj2NoH9umXkkq518sXxcWcl5T0q+O73NvZcWA4bu+2/2ux0CTtDWKjLvjVx6wv0tPTWbJkyXH3ueuuu7jnnnvo378/y5Yt49577+Wjjz6KUIYiUteq/jYV/+4N/2mwHBlW+vdspnLG8ECzkdURx/97INLpyUmo7XxK5XL8h+4AwA9g7QRJL2GYmYFdQnU+FREJ1hl9CYnX52PBp98GihcAxWWVfLph23Ge9et9V7QjULwA8Pp9vLRpGeWeqpDH2le1q1rxAmDJnjc56NoX8lgl7t18e3BmtbZdZSsodm0JeSy3t5z1hTMCxQuAck8+Byu/D3ksgILyfwaKF3BkJsGuwwvw+73HedavU+VaX614AVB4aBIeb+jfM69nC2Wlr1RrKyt7DY/np5DHcnsPsLd4crW2Sve/qHStO+Vjf11czMC0tMA1un6/H3/5m4HixZHGw/grP6n2PIthMDAtjdWHijkV5a5vjypeAPjYV/wEXu/hUzru6WDTpk2UlJTQv39/APr168fBgwfZsiX0v/ciUj9Ze/w/sNr/0+DzVP8bwGo/sp/UazXOp94C/CWPV9/Jsx7c/67WFKrzqYhIsM7sAobXT17BoRrt+UXh+fBR7Kq5qNW+ykNUet0hj1XprRnL7Xfh9rlCHsvjq8Trr1mE8fhCv4iX119JhWd/jfYqb1HIYwFUePNrtJV7duELQwHD5yut0eb1FeL3VdSy9ynG8pfyf9+nVOP3l4U8lt9fhcd74Bg5nJoSr4f4aquf+8C7s+aO3prX58ZbrZR4Tu199PlqDtg83r34qDyl49YHpaWl5ObmMnDgQEaOHFmjMLF9+3ays6tfBpSTk8PWrVsjmaaI1CEzqwO2weOqFzGOZrVjGzwOM6tDZBOTk1bjfOqvAt/BGvv5azl3h+J8KiISrDO6gGG3mVx1Xpca7Rd1Dv0lHQCdE2te839Vk54kO2JDHivN0RjHL9ZOaB7TniR76NeliLFlkBZ1VrU2m8VJnL1JyGNFWZNpGT+sRntyVOeQxwJo5OxXoy077veYlmMM1k6B3dYasFVri3X+DtOaWfsTToHV2gzTrP7+WMzGWK3NQh7LZqaTGPP7X2aAw9b6lI8dZ1o57PnPN32GYYLzl7HAiBpUo+2wx0Oc1Tyl+FH2zkD1a36TYkdgtYT+ErRIiomJYciQIUyYMIHFixfTu3dvcnNz8Rz1b11RUYHD4aj2PIfDQXm57j4gciYxszpgu/h2MG2/2GDDdvHtKl6cJn55PsVsBNFX/mIva7U1pX4WivOpiEiwzugCBsD5HZox7v/1JTk2mvTEWB677hK6Ns868RN/hY6JWTzX/VqaOFOIt0UzpvWFXJYd+rUUANKiGjOqxYM0c7bDbomia+L5XJk9mmgzJuSxHGYcvRo9QLPYi7Ea0aRFnUW/xi+QEIYCBkDTuMF0SPovbJZ4Yqw5nJf+J5Id4RkgJTq60TF1Kg4zA5slmTbJ95Ea3Tcssey2DmSm/jd2WycMI5a4mD+QFHcPFsNx4iefJNNMJznlNRyOARiGE4fjQpKT52CaoS+WGIaN1PjbSIq9GYsRh8PWniZp84mydTzlY/dMTGTJgQP4/P+ZTWLY+0DcRLCkgiUd4p8Ae49qz/P5/Sw5cICzExJPKX6UvQs5Ka9ht7bAYiSQEncHiTHXnfYLmSUlJTFp0iSys7OxWCzcdNNNFBQUsH379sA+TqeTqqrqM68qKyuJiQl9HyMi9VxVGVhMMAww7Uf+tliOtMtp4ZfnU8OwYcSMBucNYMSCtR1G0iywVl+oOVTnUxGRYJ3xi3imxMVww2+7c2nXtpgWC8lxoV/A8GcO08aFGR3omtwUl9dLWlRcWD/oNI1py80tHqDSW06sNQGrxXbiJ/1KCY5mnJc+kUpvEXZLLLYwFEp+5rSl0yFpDM3jfodp2HFYw3fStJqxNI67gpTo8/HjJcraKGyxDMNCdFQfMm3v4veXYZqNwno3C5utA0nJL+PzFWGxJIZl8c6f2a1NyEicTGpcLoYlGqslNO9Zr8REok2T5QcP0j/1yOwiw0zGiLkBf9SlgAXDTKnxvGUHC4gxTXolnloeFsNOnPNioh098PtdWM1GGGG6q1AkHTp0iMOHD1e7C4nP58N61PTiFi1asGvXrsBjv9/Pjh07aNkyPDPYRKT+8vywEtxVGKlNsPa6Bs8Xf8VfsFN3IzmN1Ho+teZA3P0QMwqM6CN3IvmFUJ1PRUSCdfqPtEMkLSE2rMWLoyXaY2gUHR+Rb2mjTCeJ9tSwFi9+ZlrsxNjSw1q8+JlhGDhtjcJavDiaw5oS1uLF0UwzAau1cURuxWmxRGO1Ng5r8eJnhmHFZs0MWfHiyDENcps05fEtP+Hx+apvM9NqLV54fD4e37KF3CZNQ/Y7aDWTsVkzGkTxAmDdunXceOONFBYWAvD222+TmZlZraDRqlUrkpOTA3cdef/998nKyqJ58+Z1krOI1B3DHo153rXYr5qCmdMZ+7ApmL2uwbCF5zbgEnrHOp8ahhXDzKi1eBGO86mIyIk0jNG2iJyxbs7OxmGxMGr9uhpFjF/y+HyMWr+OaIvJTdk116SRI/r06cN1113Htddey8CBA1m8eDHTp0+noKCAIUOGBPb705/+xPz587n44ot55513eOaZZ+owaxGpK/ZBd2P7zaBAEdewWLB1HYx90F11nJmcDJ1PReR0cMZfQiIipzebxcK7Xbsx7Nu19F/9NQ+2bEm/lNTAreDgyDW6/ygo4ImtW4i2mLzTtSs2i+q3xzNq1ChGjRpVo/3jjz8O/Ny2bVvefvvtSKYlIiJhovOpiJwOVMAQkdNerNXKR9178HpeHuM2bqTc62VgWhrx1iOrqi85cIAY0yS3SVNuys7WYEtERKQWOp+KSH2nAoaINAg2i4XRTZrwXzk5fFFczOpDxZR4vDRy2pnX5SzOTUzUNboiIiInoPOpiNRnKmCISINiGAbnJSVxXlJSXaciIiJy2tL5VETqI837EhEREREREZF6TwUMEREREREREan3VMAQERERERERkXpPBQwRERERERERqfdUwBARERERERGRek8FDBERERERERGp91TAEBEREREREZF6TwUMEREREREREan3VMAQERERERERkXpPBQwRERERERERqfeskQ7odruZNm0ar7/+OitXriQjI4Onn36a5cuXB/aprKwkOTmZhQsXVnvuwoULefzxx0lLSwu0DR8+nOHDh0csfxERERGRUNMYWUTkxCJewMjNzaVz587V2saPH8/48eMDjx9++GFatmxZ6/MHDBjAk08+GdYcRUREREQiSWNkEZETi/glJLm5udxxxx3H3L5582ZWr17NtddeG8GsRERERETqjsbIIiInFvECRteuXY+7/cUXX2TUqFFYrbVPDvnhhx8YMWIEl1xyCRMmTKCkpCQcaYqIiIiIRIzGyCIiJ1avFvHcsWMH//rXvxgyZEit25s1a0a/fv2YOXMmH3zwAaWlpTzxxBMRzlJEREREJHI0RhYROSLia2Acz+LFixkwYAA2m63W7d26daNbt26Bx2PGjGHUqFGRSk9EREREJOI0RhYROaJezcBYsWIFffv2Peb2PXv2UFhYGHjs9XqPOY1ORERERKQh0BhZROSIelXA2LRp0zFXVgZ48803mThxIm63G6/Xy/z58/ntb38buQRFRERERCJMY2QRkSMiWsAoKChg4MCBDBw4EIARI0YwcOBA9u3bR3FxMRUVFdXuXw3wxhtv8PzzzwMwduxY4uPjGTx4MIMGDcJqtVa7tZSIiIiIyOlGY2QRkeBEdG5ZamoqS5YsOeb2TZs21WgbPnx44Ofo6Gjd31pEREREGhSNkUVEglOvLiEREREREREREamNChgiInJMK1asoG3btuTl5VVrz8vLo2PHjoEpzwMHDtR0ZREREREJKy1PLCIitaqoqGDatGkkJibWuj09Pf24U55FREREREJJMzBERKRW06dP5/LLLycmJqauUxERERERUQFDRERq2rRpE59//jl/+MMfjrlPaWkpubm5DBw4kJEjR7Jly5bIJSgiIiIiZxwVMEREpBq/38/kyZOZOHEiNput1n1iYmIYMmQIEyZMYPHixfTu3Zvc3Fw8Hk+EsxURERGRM4UKGCIiUs1bb71Fq1at6NGjxzH3SUpKYtKkSWRnZ2OxWLjpppsoKChg+/btkUtURERERM4oKmCIiEg1y5YtY9myZfTu3ZvevXuzZ88ehg0bxpdffhnY59ChQ+zatava83w+H1ar1oYWERERkfDQSFNERKqZNWtWtccXXXQR8+bNIzs7O9C2bt06Jk2axLvvvktycjJvv/02mZmZ5OTkRDpdERERETlDBD0D46OPPqKgoAAAl8vFs88+y9ixY5kzZw5+vz9sCYqISP3w/fffM3LkSAD69OnDddddx7XXXsvAgQNZvHgx06dPxzTNOs5SRERERBqqoGZgzJo1ixkzZvDmm2+SmprK1KlTWbhwIb1792bmzJmUl5eTm5sb7lxFRKQOLF++HIDs7Gxmz54daB81ahSjRo2qq7RERERE5AwT1AyMd955h8cff5x27dpRUVHB+++/z7hx45gxYwZPPvkkH374YbjzFBEREREREZEzWFAFjL1799KtWzcAVq9ejdvt5rLLLgOgffv27NmzJ3wZioiIiIiIiMgZL6hLSOLj4ykoKCAjI4Nly5bRuXNnEhMTASgsLCQqKiqsSYbblvwCfswrwDQttM1Oo0l6Uthi5ZUV8++ivZR73bRJSKN9QjqGYYQlVqnnELvLt1LqKSbZnkFWdHPsZnjeK7e3kgNVP3LIvRunmUxaVBuc1sSwxPL7/RRW/USRayumEUWqoy1x9oywxAIod2+n1PUDPr+XWHtbYu2twxbL49mLy/09Xl8xNmsrHPZOGIY9LLG83kO43OvxePMwzSwcts6YZkJYYvn9btzuDXjcP2KxJGC1dcZqzQxLLACXezNVrg0Yhond1hG7rWXYYnk8ebjd6/D7yrDa2mKzdcQwdIMnERE5vUX63C0iEoygChjnnXceEydOpHv37rz33ntMmjQJgNLSUl5++WW6du0a1iTDacP2vYx59l3Kq9wApCXEMOOPv6Nl49SQx9pZWsjoz95iS8lBAOwWkzl9r+fstCYhj1XuKWVR/ly+Lf400DY0azQ9kweEvGDi9/vZdPjvrNj3fKCtTfwA+qbfRpQZF9JYAHsr/sX/7L4bn//Ie5Zga8rFWU+RYM8KeaxS1498t+8mXN4DAJhGDF0z5hLv6BzyWB7PPg4U3U5V1Wf/12KQlvwqMc5BIY/l81dyqPQvFJc8G2hLiL2dpPi7sVhCX+SqqvwnxYU3Az4AbPaeJCTNDMtAqMr1PfkHhuH3lwBgsaSQmfoODnv7kMfyeHZSePAmPJ4f/q/FRkrqf+Nw9A55LBERkUiqrPwnRUedu+32niSG6dwtIhKsoL4mfOCBB2jZsiVffvklN954I1dffTUAn376KevWrWP8+PFhTTJcvD4fb//zu0DxAuDAoTL+d922sMRbW7A7ULwAcPm8PLd+BeUeV8hj7avcWa14AbBozzwOuvaGPNZhdz6f7X+5Wtvmw0s5WBX6f0e3t5w1B18NFC8ADrl3sL9yfchjARysWBEoXgB4/WXkHV6A3+8LeSyXe/1RxQsAP4XFE/B494c8ltu9leKS56u1HSp9Ebfnx5DH8noLOHzoQX4eAAG4XV/jca8LeSyAw6ULAsULAJ/vIGUV/xOWWG7Xt0cVLwDcHD70BF7v4bDEExERiYTazt2uMJ67RUSCFdQMjKSkJKZNm1ajvV+/flxyySWn7W3zPF4fW/YU1mjftb84LPH2V5bUaNtVVkS5x4XTGtrLBCp95TXaXL5KXL6qkMYBcPsq8Pgra8bzloY8lsdfxWH37hrtFZ6Dtex96spd22u0lbl/wud3YxqOkMby+Q7VaPP69uP3lUOIf8V8/hKOHpQc4cfnq/l/9FT5/RX4vDULZz5f6D/k+/1eXJ6NNdrdnp9CHgvA6yuo2ebdCVQC8WGJKSIiEm5+fwXeCJ27RURORtAXavv9flauXMnLL7/MY489RmFhIQ6Hg+3bt4cxvfBy2KwM7dOpRvsFXVqEJd5ZyTUvcRjW7DekOGJCHivFnontFx+wc6JbkWgL/aUxsbZ0GjnaVmuzGlEk2rNDHivKTKRt/JAa7WlR7UIeCyDV2a9GW+PYqzEtoS1eANhsrfhlpcIZdRmmGfr1PaxmE0yz+hRQ05KGzdo05LFMsxFR0b/7RasFq61VyGMZhkl8zHU12mOia/6fCQWbrWb/Ee28Bosl9L9nIiIikWKajYiO0LlbRORkBFXA2L9/P1dccQVjxoxhzpw5/Pd//zdlZWVs2bKFoUOH8s0334Q7z7C54KyWjBlyLlF2K/FOB/dfeyFdW4V+LQWALsmNebbnFaQ6YrBbTG5sdTZXNf9NWBbxbBSVxc3NHyTD0QQDg7Zx3bgyJxenNTbksaLMOPo1Hk/TmHMAgxR7cy7LeZIkR+jX9jAMgzYJg+mYeBUWw0a0mcxvMyaRFtUh5LEAEqN60CZ5ElZLAhYjmuYJt5PqvDAssey2DjRKmYfVbApYiIn+HUkJ94VlTQqbNZOMlDlE2c8FwGHrQUbKPKzW0P/fNwwHsXF/JCp6GGBimjkkJr+OzdYx5LEAoqMuIin+fgzDicWIJyVhCtGO88ISy2brQmLSy1gs6YANp/MmYmJu0CKeIiJyWvv53B191Lk7OYznbhGRYAV1CckTTzyB1Wrlo48+onXr1oFFO1u2bMkNN9zACy+8wPz588OaaLikJsQwesi5/L/eHbFYLDRKDP0H/J9FW21c1rQT5zZqhsvnJT06DqslfB90msd2YHTLR6n0lRNrJmA3Qz9r4GcpjuYMbDyZCm8xdouTKGv4ps/H2TI4Jy2XTklXYxo2nNaUsMWymfFkx19PmrM/frw4zMyw3TXGMKw4oy/Cbl+M31+GaTbCEuLLVI7msHchPWUePn8RFiMR0wzfe2a1NSch6U/Ext+LYURjmuGboWA100iMu4M45zDAEtbFxiyWaJzOy3E4euH3V2Ga6RiGLWzxREREIsVma05i0p+Ii8C5W0QkWEEVMFatWsWrr75K69Y1bx955ZVX8sYbb4Q8sUgyDIOM5Mhdr54WHb4iyS85rbE4iUw8mxmFLQyXO9TGYliJs0UmFoDDmh6xWFYzGUiOSCzTjMMk9HeKqY1h2LFacyIUywjLbJJjMc20iMUSERGJlEieu0VEghHU1/8Wi4WYmNrXaXC73WH7RlpEREREREREBIIsYLRr147p06fj8XhqbHvjjTfo1KnmQnYiIiIiIiIiIqES1CUkd9xxByNHjuS3v/0t3bp1w+128/jjj7Njxw7y8/N5/fXXw52niIiIiIiIiJzBgpqB0b17d9577z369etHXl4ejRs3Zv/+/ZxzzjksXLiQbt26hTtPERERERERETmDBTUDA47cceSRRx4JZy4iIiIiIiIiIrU6ZgHj5ZdfDvoghmEwZsyYkCQkIiIiIiIiIvJLxyxgPP/880EfRAUMEREREREREQmnYxYwNm7cGJaAbrebadOm8frrr7Ny5UoyMjJYuHAhjz/+OGlpaYH9hg8fzvDhw2s8f9GiRcycORO3202bNm144okniIuLC0uuIiIiIiKRoDGyiMiJBb0GBkBZWRl79+6lpKSEhIQEMjMziYqKOqmAubm5dO7cuUb7gAEDePLJJ4/73Pz8fKZMmcLChQtp3LgxTz75JM899xyTJk06qRxEREREROoTjZFFRE4sqLuQHDhwgFtuuYWePXsyZMgQrrnmGgYNGkSvXr0YP348RUVFQQfMzc3ljjvu+FXJLlu2jF69etG4cWMAhg0bxpIlS37VsURERERE6guNkUVETiyoGRgPPvgg69ev54477qBDhw44nU7KyspYv3498+fPp7S0lBkzZgQVsGvXrrW2//DDD4wYMYL9+/fTvXt3HnjggRrT3rZv306TJk0Cj5s0acLBgwc5dOgQCQkJQcUXEREREalvNEYWETmxoGZgfPXVV/zpT39izJgxnH/++XTv3p2+ffuSm5vLM888w+eff35KSTRr1ox+/foxc+ZMPvjgA0pLS3niiSdq7FdRUYHdbg88ttvtGIZBRUXFKcUXEREREalvNEYWEakuqBkYTqczMCXtlxo3bkxsbOwpJdGtWze6desWeDxmzBhGjRpVax4ulyvwuKqqCr/fj9PpPKX4IiIiIiL1jcbIIiLVBTUD45prrmHBggU12v1+PwsWLODaa689pST27NlDYWFh4LHX68VqrVlbad68OTt27Ag83r59O2lpacTHx59SfBERERGR+kZjZBGR6oKagVFVVcXSpUtZtmwZnTp1Ii4ujoqKCr799lu8Xi8XXHABDz30EACGYfDoo4+eVBJvvvkmP/30Ey+88AIWi4X58+fz29/+tsZ+/fv3589//jNbt26lRYsWzJkzhyFDhpxULBERERGR04HGyCIi1QVVwFiyZAkWy5HJGuvXrw+0G4aB1Wpl1apV1dqOpaCgoNp9q0eMGIFpmsydO5fnnnuOwYMHYxgG3bp1Y/z48QAsXbqU5cuXM3XqVNLT05k8eTK33norXq+XDh06MHHixJN7xSIiErQVK1YwZswYli1bRnZ2drVtX3zxBU8//TTl5eU0btyYqVOnkpGRUUeZioicvjRGFhEJTlAFjOXLl4ckWGpq6jFv6XSs+1sPGDCAAQMGBB4PGjSIQYMGhSQfEZGGqLS0lM2bN1NQUECfPn1wOp14PJ5apx0fT0VFBdOmTSMxMbHGtvLycu6++25effVVOnbsyLx585g8eTKvvPJKqF6GiMhp41T7XY2RRUSCE9QaGD8rLS1lz5495Ofn1/gjIiJ1y+128+ijj3Luuedy3XXXceedd3Lw4EF2797NpZdeyu7du0/qeNOnT+fyyy8nJiamxrYvv/ySnJwcOnbsCMCVV17JqlWrKC0tDclrERE5HYS63xURkeMLqoDxxRdf0K9fP84++2wuuugi+vXrF/jz82MREalbzz//PP/zP//DAw88wIcffkhUVBQAycnJtGjRgmnTpgV9rE2bNvH555/zhz/8odbt27dvJycnJ/A4JiaGxMREdu7ceUqvQUTkdBLKfldERE4sqHltjz76KE2bNuW+++4jISHhuLF7FVAAACAASURBVOtciIhI3fjwww+ZMmUK/fv3r9YeHR3NbbfdxsiRI4M6jt/vZ/LkyUycOBGbzVbrPhUVFTgcjmptDoeD8vLyX5e8iMhpKFT9roiIBCeoAsbevXt55ZVXaNKkSbjzERGRX6m0tJQ2bdrUui0hIYGKioqgjvPWW2/RqlUrevToccx9nE4nVVVV1doqKytrvdxERKShClW/KyIiwQnqEpJu3bqxffv2MKciIiKnolmzZixatKjWbStWrKBp06ZBHWfZsmUsW7aM3r1707t3b/bs2cOwYcP48ssvA/u0aNGi2uUiJSUlHDp0KOgYIiINQaj6XRERCU5QMzCeeOIJJkyYwMaNG2nXrh3R0dE19jn77LNDnpyIiATv+uuvZ+LEifz73//mnHPOwefzsWjRIvLz83n//fd5+OGHgzrOrFmzqj2+6KKLmDdvXrXbqJ5zzjlMmDCBb775hh49ejBnzhwuvPBCnE5nKF+SiEi9Fqp+V0REghNUAWPFihWsXr2aVatW1brdMAx++OGHkCYmIiInZ9iwYZimyaxZs1i6dClwZIG5Fi1a8PDDD3PllVee0vG///57XnjhBWbPnk1UVBTPPvssjz76KBUVFTRp0uSYt/oTEWmowt3viohIdUEVMKZPn86gQYMYMWKEFvEUEanHhg4dytChQyktLaWsrIzY2NhTXpdi+fLlAGRnZzN79uxA+znnnMOHH354SscWETndhaPfFRGR2gW1BkZFRQW33norHTt2JDs7m6ysrBp/RESkbn3wwQfs2bMHgNjYWNLT0wOD6NLSUh544IG6TE9EpMFRvysiEllBFTAuvfTSY14+IiIi9cP999/P0KFD+d///d8a2yorK/nggw/qICsRkYZL/a6ISGQFdQlJhw4dmDt3Lp9++ilt2rQhKiqq2nbDMBgzZkxYEhQRkeBdfPHF3HLLLYwePZo77rhDl/yJiISZ+l0RkcgJqoDx6KOPArBt27bAtdBHUwFDRKTuGYbBnXfeSb9+/bjvvvv49ttvefbZZ0lOTq7r1EREGiT1uyIikRXUJSQbN2487h/dgUREpO75/X4ALrjgAj744AMqKyu54oorWLNmTR1nJiLSMKnfFRGJrKAKGMezb98+rrrqqlDkIiIip+DoacsZGRksWLCAwYMHc+ONN/Laa6/VYWYiIg2T+l0RkcgK6hISgM8++4xVq1ZRXFxcrf3HH39k27ZtIU9MREROzs/fBP7MNE3uu+8+evTowYQJE+ooKxGRhkv9rohIZAVVwPjrX//KI488QmZmJnv37iUrK4vi4mJKSkro0aMHjz32WLjzFBGRE1i2bFmt113369eP999/n6+++qoOshIRabjU74qIRFZQBYy5c+fyyCOPcPXVV9O1a1dee+01cnJy+Oc//8krr7xCly5dwp2niIjUYu3atZx11lmYpsm+ffvYt2/fMfdt2rRpBDMTEWmY1O+KiNSdoAoYu3fv5vzzzwfAYrHg8XgAuPDCC6msrGTSpEnMnj07fFmKiEitrrvuOlatWkVKSgrXXXfdMW/f5/f7MQxDiy6LiJwi9bsiInUnqAJGdHQ0hw4dIjMzk8TERHbu3Enz5s0B6Ny5s67xExGpI/PmzSMhISHws4iIhJf6XRGRuhNUAaNXr15MmDCBv/zlL3Tv3p1nnnmGxMREEhMTmTNnDklJSeHOU0REatGzZ89afxYRkfBQvysiUneCuo3qfffdR3R0NG63m9zcXIqLi7nmmmsYOHAgb7/9Nrm5ueHOU0RETqCsrIyHHnqI7du3A5Cfn88111xD9+7dGTt2LIWFhXWboIhIA6N+V0QksoKagZGZmcmCBQsCjz/55BO+/vpr3G43HTt2JCsrK2wJRsKuvEK2bj+AaVpo2bwRmRkJYYu1p7SEHw4eoNzjonVSKm2TU8MWq8RdxvayPIrcJWREpdLMmYXdtIUlltvnYk/FDgpde4mzJpEZ3QynNTYssQAOVW3hkGsrVks0ifbWOG3pYYtV4d5FmXsTfr+XGFsbnPbmYYvl8RZQ5d6Az1eEzdoSh609hhH03Y5PitdXisv9bzyePKzWLOy2DpiWuLDE8vu9uNw/4Pb8hMWSgN3WCauZFpZYAJXurVS6N2FgEGXrgMPWJGyxXJ69lLl+wOcvI9rWmmhbm2NeDx1uU6dOZc2aNdxyyy0ATJo0if3793P77bfz8ccf8+yzz+quUSIiIdSQ+12f30tR1VaKXTtxmLEkO1rjtNa844qISCQF/cmosrIS0zSx2WzExMTQpEkTtm7ditUang9XkbL5p73cNeEtSkurAMhIj+fpR6+iaU5KyGPtOlzMLX//kA0F+wGIslp5Y8hV9MgIfQGozFPOGzs+5O/7Pgu03d56BBc1OjfksQC+L17FO3kvBR73TB7ApZnDiTZjQh6roOJfrMi/Fa//yHuW6GhL7/SniLWH4d/RtYV1+0ZR5d0NgNUST5f0ucQ5OoY8lse7n32F91JetfT/WkwyU14nNrp/yGP5/S5KSudQePjxQFtS3L0kxOdiMaJCHq+y6n/ZVzACOLIAcJTjQlKTnsVqzQh5rArXBrbsvxavrwgAm5lJi0ZvEGVrE/JYVe7d/FhwK2WufwFgYKdd+nzio+pmSvGKFSt4/vnnycrK4uDBg6xatYrp06fTv39/unXrxh133FEneYmINFQNud/NL/+GT3bfhx8vAFnOnlyQ8QBOa/i+fBMROZGgLiH59ttv6du3Lxs2bABgyZIlXH755dx+++1ccsklfPnll2FNMlx8Pj9/W/RdoHgBsHffYb76ZmtY4q3Zlx8oXgBUejw8t3oV5W5XyGPtLN9TrXgB8OrWd9hbcSDksQ5W7eXD/NeqtX1duJR9lbtCHsvjq2Bd4SuB4gVAcdUmDlatC3ksgMKKlYHixZH4h8kveRO/3xfyWFXuDUcVLwC8HCh+AI8n9O+Zy72FwsNPVmsrKpmG270l5LE83oMcLHqAn4sXAJVV/8TlXh/yWACFpW8FihcAbu8eDpV/EpZYpa7vAsULAD8udhU9g9dbGpZ4J3L48GGys7MB+OKLL4iKiqJv374ANGrUiIMHD9ZJXiIiDVVD7XcrPMWs2v9coHgBsLv8awoqf6zDrEREgixgTJs2jYEDB9Kx45FvnZ9++mkuvvhivv76a4YPH85LL710giPUTx6Plx+37q/Rvm1HQVji7SktqdH2U3Eh5W53yGOVeMpqtFV4K6nwVoY8VpWvgipfRY32ck/N13uqPL4KDru21YzlPvY92E9FuavmifrI5QKhf8+83qIabR7vbnz+mu/lqfL5DsFRg5L/a8XnKw55LL+/DI93Zy051Hy9px7LS7nr+xrtFa5NIY8F4PLW7D8qPVvx+svDEu9EGjVqxKZNR17r3/72N8455xzsdjsAO3bsIDExsU7yEhFpqBpqv+vxVVDq3lOjvcp7qA6yERH5j6AKGD/88AM33XQTNpuNjRs3kp+fzy233EJ8fDy///3vAx336cZutzKwf6ca7ef1bBmWeF3Sak6X/13rDqREO0MeK9ORhu0Xayc0dWaR6gj9tYsJtlQyHU2rtVkNGymO0F8e4DATaRY3uEZ7clT7kMcCSHFeWKMtPXYopsUR8lh2Wwug+toJMVEXYzVDv76HzZqDaam+BoXFkozVGvq1IkxLI5xRQ37RamC1hv73zDBMkmKuqtGe6Lwk5LEAYmw1/9+lxlyBzQz9JWjBuPzyy7n77ru57LLLWLVqFTfeeCMAW7ZsYcqUKVx4Yc3/zyIi8us11H432ppMs9gLftFqkGDPqZN8RER+FlQBAwisdfHZZ5+RmZlJ27ZtA9vcYZhBECnn92rN7393NlarhegoG2NuuoCzOmWHJdZZjTKY2vdi4u0OTMPg6raduK5Dl7As+JftzODBDmNJdxy5TrF9XEv+2PoG4myhX5MixhrH1U1up5mzHQDJ9nRubPYAjRyh/3c0DAstE4bSPO5yDCzYLHH0SHuQZEfNQlQoJESdTfPEe7EYTgxsZMffTKoz9GtSADhsHchIeRXT0ggAZ1R/UhImYrFEhzyW1ZpFesocbNYj/242azsyUuZis4Z+YGKxRJGUMJ7oqIEAmJY00pL/gsMenvcsPro/jeJuxcCBxYgmI2E8MVHnhSVWjKMLzZOfxmpJBCykxAwlPe5GDMMMS7wTuf322xk/fjw9evRgxowZ9OrVC4C9e/fStm1b7rvvvjrJS0SkoWqo/a7V4qB76iiaxvQBINpM5qLMR0hxhH49KRGRk2H4/X7/iXa65ppr6Nq1K4MHD+aOO+6gf//+TJgwAYAFCxbw1ltv8eGHH4Y92WDk5eXRr18/li1bFrgm8UQ8Xh/79h/CtFhIbxQf9jsI5JccxuX1khkbhyPMi6AWuw5T5q0gyRaP0xr6D8JHq/SWU+IuJsp0EmcL75RJr89FuWcfFsNGjC30Mz2O5vf7qfTsBnxEWRuH7a4gP3N79+H3lWGaGZiW0M/OOZrXV4TXW4hpScY0k8Iay+erwOvdg2FxYjXD/Z55cXnyMAwLNjM77L/TVZ49+P1V2M1MLCcxO+fX9FenmzPhNYrI6e9M6Kt+1RjZV0mZ5wBWI4oYW/juHiYicrTj9VdBfRK78847yc3NZc6cOTRu3JhRo0YBR1Zenjp1Ko8++mjQybjdbqZNm8brr7/OypUrycg48kHmpZde4qOPPsLv99O+fXumTJlCXFz1Wzp+9dVXjB49mszMzEDbgAEDuOeee4KOXxuraSErM7wf3o7WOC4+YrES7fEkEpl4UaaTKDO8H7h/ZlrsxEVoGqNhGETbIjeYsZnpEKEv8E1LEqYlMv/3LZZoLJYWEYllGCYOW9MT7xgiDmvmiXcSERE5jno5RrZE6bIREalXgipg9OrVi5UrV7J9+3Zat25NdPSRb/JbtGjByy+/TJ8+fYIOmJubS+fOnau1LVmyhCVLlvDuu+/idDq55557ePXVV7nrrrtqPL9Lly7Mnz8/6HgiIiIiIvWdxsgiIicW9BoY8fHxdOnSJVC8AGjSpMlJFS/gSOf8y3tit2zZkqlTpxIbG4vFYqFr1678+KNu0yQiIiIiZwaNkUVETiy8F/PXomvXrjXaWrduXe3xp59+ytlnn13r8/Pz8xk5ciR5eXm0bduWBx98kPT00N+lQUTkdFNYWEhycujvNCQiIrULZb+rMbKIyIlFvIBxIjNnzuTgwYOMGDGixra0tDQuvvhi/uu//ou4uDieeuop7r33XubNm1cHmYqI1C+9e/emXbt2nHfeefTu3ZsePXpgt9t/1bE++eQTZsyYQVVVFUlJSTzyyCO0aVN99fm2bdvSvHnzwOP09HTmzp17Sq9BROR0Esp+90Q0RhYRqWcFjGnTprFq1Spmz56N01lzMcgWLVpUux3Vbbfdxrnnnkt5eXmt+4uInEn+8pe/8M0337B69WrmzZuHxWKhe/fugYF1+/btgzpOfn4+kydP5r333iMrK4u5c+cyYcIE3n333Rr7LlmyJNQvQ0TktBGqfvdENEYWETmi3hQwpk+fztq1a5k3bx6xsbG17lNQUIDX6w1Mh/N6vRiGgTXMtyIVETkdnH/++Zx//vkAuFwuvvvuO1avXs3nn3/OK6+8gs1m4/PPPz/hcaxWK9OmTSMrKws4spDzn//857DmLiJyOgpVv3s8GiOLiPxH0It4lpaWsmDBAh566CFuueUW9u/fj9frPeVOGWD9+vV88MEHvPzyy8fsmAGWLVvGbbfdRllZGQDz5s2jV69eYZuqJyJyujIMA9M0sdlsmKaJYRh4PJ6gntuoUSN69+4NgMfj4f3336dfv3617jtu3DgGDRrE9ddfz9q1a0OWv4jI6eZU+t1j0RhZRKS6oMqyW7du5cYbb+Tw4cO0atWKTZs2UVVVxY4dOxg9ejTPP/88/fv3P+FxCgoKGD58eODxiBEjME2THj16UFJSwlVXXRXYlpWVxezZs1m6dCnLly9n6tSpXHXVVWzfvp0rrrgCi8VCq1atmDp16q942SIiDc+nn37KmjVr+Oabb1i3bh0pKSl069aNCy+8kHHjxtVYw+JE5s6dy4wZM2jSpAkvvfRSje1XX301119/Pe3atWPx4sWMHTuWpUuXEh8fH6qXJCJSr4Wq39UYWUQkOIbf7/efaKeRI0diGAbPPPMMSUlJdO3alQ8//JCcnBxee+01Fi1axHvvvReJfE8oLy+Pfv36sWzZMrKzs+s6HRGRYwp1f9WuXTsaN27M1VdfzRVXXEFGRsYpH9Pv97No0SKeffZZFi9eTFRU1DH3veyyyxg3bhwXXHBBoE19soicDn5tXxWOfjdc1B+LyOnieP1VUJeQrF27lnHjxpGUlFRj24ABA3Q/ahGReuDuu++mZcuWzJo1i2HDhnHnnXcyf/58Nm7ceFLH2bJlS+DyQMMwGDJkCGVlZWzbti2wT1lZGVu3bq32PK/Xq+utReSMEqp+V0REghPUSNPpdOLz+WrddvjwYWw2W0iTEhGRkzd69GhGjx6N1+tlw4YNrF69ms8++4zp06fj9/vp3r07L7/88gmPU1hYyPjx43nvvfdIT09nzZo1uN1ucnJyAvvs3buXa6+9lnfeeYemTZvy2WefUVRUxFlnnRXOlygiUq+Eqt8VEZHgBFXA6Ny5M1OnTuXPf/5ztVkYlZWVvPTSS/To0SNsCYqIyMkxTZMuXbrQqlUr2rVrR/v27fnb3/7GypUrg3r+2WefzdixY7npppvw+XzY7Xaee+45ysrKuOaaa/j4449p2bIlEyZMYOzYsfh8PhISEpgxY8ZxF5kTEWmoTrXfFRGR4ARVwBg3bhwjRoygb9++tG7dmqqqKu68807y8vIwTZM33ngj3HmKiMgJ7Nu3jzVr1rB27VrWrl3L5s2biY6OpmfPntx8882cd955QR/r+uuv5/rrr6/R/vHHHwd+vuKKK7jiiitCkruIyOkolP2uiIicWFAFjFatWrF48WLeeecd1q1bR0JCAnFxcVx66aVceeWVJCcnhztPERE5gQsuuAC73c5vfvMbBgwYwOTJk+ncuTMWS9B3zBYRkZOgfldEJLKCKmC8/fbbXHrppYwePTrc+YiIyK80e/Zsunfvftw7hYiISOio3xURiaygChgPP/wwU6ZMoU+fPlx22WX069cPh8MR7txEROQk9O7dmxUrVrBgwQL+/e9/U1paSlxcHJ06deLmm2+mZ8+edZ2iiEiDon5XRCSygprf9umnn3L//fdTXl7OvffeS69evRg3bhwrV67E6/WGO0cREQnCRx99xC233EJJSQmDBw/m5ptvZuDAgRQUFPCHP/xBi8mJiISY+l0RkcgKagZGampqYEG3oqIili5dyt///nduvfVW4uLiGDhwIJMnTw53riIichyzZs1i7Nix3HnnnTW2PfXUU0yfPp0LLrigDjITEWmY1O+KiETWSa8wlJSUxNVXX82sWbOYOXMm6enp/PWvfw1HbiIichK2bdvG0KFDa9129dVX8+OPP0Y4IxGRhk39rohIZAU1A+Nnbrebzz//nKVLl7J8+XKKi4vp2rUrEydODFd+IiISpOjoaIqLi2nSpEmNbSUlJdjt9jrISkSk4VK/KyISWUEVMBYvXsw//vEPPv30U0pLSznrrLMYPXo0l156Kenp6eHOUUREgnDOOefw1FNP8eSTT5KTkxNo37ZtG4899hi9evWqw+xERBoe9bsiIpEVVAHj7rvvpkOHDtxyyy1ceumlZGVlhTsvERE5Sffffz833HADF198MYmJicTGxlJSUsKhQ4do0qQJL7zwQl2nKCLSoKjfFRGJrKAKGH//+99rnRonIiL1R1ZWFosWLWLp0qVs2LCh2u38BgwYoKnMIiIhpn5XRCSyjlnAeOihh7j//vuJiYlh1qxZxz2IYRg8+uijIU9OREROTlRUFJdddhmXXXZZXaciInJGUL8rIhI5xyxgrFq1CpfLRUxMDKtWrYpkTiIiEqSbb775pPZ/7bXXwpSJiMiZQf2uiEjdOWYBY/ny5bX+LCIi9Yfb7a72eMuWLZSWltKyZUucTieHDx9m69atpKSk0KVLlzrKUkSk4VC/KyJSd4JaA+OGG27gxRdfJD4+vsa2zZs388ADD/Dee++FPDkRETm++fPnB35euHAhS5cu5ZlnniE2NjbQvm/fPiZMmMBFF11UFymKiDQo6ndFROqO5Xgb8/Pzyc/P5+uvv2bXrl2Bxz//2b17N6tXr2bz5s2RyldERI5h5syZ/PGPf6w2iAZIT0/nnnvuYcaMGXWUmYhIw6R+V0Qkso47A2Pw4MFUVlZiGAbDhg2rdR+/38+5554bluRERCR4+/btw+/317rNMAz27dsX4YxERBo29bsiIpF13ALGmjVr2LRpE0OHDuWRRx4hLi6uxj7x8fH07NkzbAmKiEhw2rZty0MPPcRDDz1Eu3btsNvtuFwu1q1bxzPPPEObNm3qOkURkQZF/a6ISGQdt4BhsVho37498+bNo2vXrthsthr7lJaW8vbbbzN8+PCwJSkiIic2ZcoUxo4dy+9//3vgyLd/fr8fv99PamqqpjKLiISY+l0RkcgKahHPnj174vF42Lx5M4cOHQq0+/1+vvvuO1588UUVMERE6li7du34+9//zldffcVPP/1EeXk5TqeT5s2bc+655+JwOOo6RRGRBkX9rohIZAVVwNi4cSO5ubns2bOn1u0DBgwIaVIiIvLr2Gw2+vTpQ58+feo6FRGRM4L6XRGRyAmqgPHUU0/Rvn17pk6dypgxY3jqqaewWCx89NFHJCQkMGXKlHDnKSIiJ1BUVMRLL73E999/X2223NE++eSTCGclItJwqd8VEYmsoAoYGzZs4M0336Rly5YYhkGHDh3IyclhwIABPPbYYzz33HPcdddd4c5VRESOY+LEiaxatYqePXvSokULDMOo65RERBo09bsiIpEVVAGjsrKS6OhoAJxOJ0VFReTk5AAwfPhwhg8frgKGiEgd++qrr5g+fTrnn39+XaciInJGUL8rIhJZlmB2atOmDX/9619xuVw0b96cd999N7Btx44dVFVVhS1BEREJjsPhoGnTpnWdhojIGUP9rohIZAVVwBgzZgyzZ88mLy+P6667jrfffptLL72Ua6+9lttuu42+ffuGO08RETmBoUOH8vHHH9d1GiIiZwz1uyIikRXUJSQDBgzg448/JisrixYtWmCxWFi8eDEul4vc3FxuvPHGoAO63W6mTZvG66+/zsqVK8nIyABgzpw5vPXWW/h8Pnr06MHkyZOx2+01nr9o0SJmzpyJ2+2mTZs2PPHEE8TFxQUdvzb5+UVs316A1bTQrHkajRrFn9LxjqeguJSf8gqodHlolplMs8zksMUqdVewtXQPRa5SMqOTaR6bgc0S1Ft+0jw+D3sqd3HQtZ84awKNo3OINmPCEguguGonRa4dWC1RpDha4LSmhC1WhXsvJe6f8Pk9xNlbEWPLDlsst7eIcvcmPN5DRNma4bS1xjCCqjOeNK+vggr3Zqo8u7FbM3Ha2mJanGGJ5ff7qfJswuXeimlJwGFrh9UM53u2i3LXjxiGBae9DVHWxmGL5fIcoMy9Ca+vAqetBU57y7DFOpGcnBzmzJnDmjVr6NSpU+DSv58ZhsGYMWOCOtYnn3zCjBkzqKqqIikpiUceeYQ2bdpU22fjxo08/PDDFBUVkZSUxMMPP0y7du1C9npEROq7UPa79W2M7Pf72V62j13lB4i1RtMiNoNEe+yvPp6ISCgE/Wm2efPmgZ8HDhzIwIEDf1XA3NxcOnfuXK3tu+++Y968eXzwwQfExcVx5513Mn/+fEaOHFltv/z8fKZMmcLChQtp3LgxTz75JM899xyTJk36VbkAbNmyj/H3/pXi4nIAcpqkMGXKleTkhP7D1Z6Cwzz0l8X866d8AGKi7Lw47ko6tcgMeawyTyXzti3l7V2fAmBg8FDH67ko4zchjwXw/aGvmbf9Rfz4AbgwbTCXZF5JtBl9gmeevH0V/2ZR3j24fUfes4yozlzUeCJxtoyQxyp1bWf13jsp9WwDwG5J5tzMV0hwtA15LJe3kG2Fj3Cw/CMADGy0b/QqidGhv67W53dzoPQtdhQ9EmhrkvQg6XE3YDFqDopOVXnV5+w8MBw/LgDiogeTmfQ4VjMt5LHKXJtYt+8m3N79AERZm9Kx0Syc9hYhj1Xp2cPmA/dyqOpLACyGk87pc4mP6hryWMGYPHkyANu2bWPVqlU1tgc7kM7Pz2fy5Mm89957ZGVlMXfuXCZMmFDt8kGAu+66i3vuuYf+/fuzbNky7r33Xj766KPQvBgRkdNAqPpdqH9j5G+LfuK+717F7fcC0LdRZ/7Y5nckO07ti0MRkVMRVAHjoYceOu52h8NBdnY2gwYNolGjRsfdNzc3l65du/LSSy8F2pYsWcKgQYOIjz8y8+HKK6/kxRdfrNE5L1u2jF69etG48ZFvU4cNG8YNN9zwqztnn8/Pxx9/FyheAOzaeZDVq7eGpYCxbkt+oHgBUFbpYtbfvuCpWy8jym4LaaztpXsDxQsAP36e3fQe7ROakBkd2lkfB6v28/au2YHiBcA/DyzirMSeNI9tc5xnnjyPr5JvCl4PFC8A9lauY1/FhrAUMPZXrAoULwBcvkJ2Hn6PTqn3h3xmRJlrQ6B4AeDHzZbCiXRJX4gtxDNMKt1b2Vn0eLW2nUVPkhDVB6c9tN+ge7yF7CmaECheAJRULCIx5lrioi8MaSyAfSXvBYoXAJWeHRys+AdO++iQxyqp+legeAHg85ezvXgaHdP+wv9n787jq6rO/Y9/9hkzzyEkEEKYQQEFcRZvAUUBi7ZVQewPFRxAe60VB1QUi5Y6UBQEvSoitV6V6tUqiFTBWVoVEHBAkUAgRCDzeHJypt8fqdF4GA5ma6aaDQAAIABJREFU7xMI3/fr5UvO2jv7eUI4K2s/Z6217XZrZrMcyObNm025jsPhYM6cOXTq1AmAU045hXnz5rU45+uvv6ampoYRI0YAMHz4cGbMmMHWrVvp3r3tZqGIiESTWf0uHF5j5OrGOh7++uXm4gXAe3s3MTr7JE5ya6adiLSdiAoYmzZtoqSkhLKyMmJjY0lKSqK6uhqPx0Nqaiput5uSkhLmzZvH4sWLGThw4H6vdfzx4Z9Mbt++nWHDhjW/zs3NpaCgYJ/ndenSpfl1ly5dKCsro6qqiuTk5Ei+lRZ8Pj9ffVkc1r712z2HfK1IFJeEPx/8m50l1HsaTS9gVPhqw9pq/R5q/R5T4wB4AvV4AvVh7bX+atNjNQbrKfd+G9Ze49tteiyAam/4wKTCu5FgqBG7EWNqLF+gLKzN69+JP1SLE3MLGP5gJSH8P2kN4AtUmBoHIBispdG/Naw9ECw1P1bIT3Xj+rD2Wu/npscCaPR/F9ZW1/g1/lANdqJfwDBLhw4dmovRfr+fl19+meHDh7c4Z/v27XTu3HI51fd9twoYIiKH7nAaI9cFGthRXxLWXtEYPr4UEYmmiAoYt912G7Nnz+bhhx/mhBNOAJrWxb333ns88sgj3H333XTt2pU77riDBx98kGeeeeaQkvB4PC3W8sXExODxhN9oezwe0tJ+mD3gcrkwDAOPx/OzOme328mIs45hy5aWN78nnmTN4LtPfvgMgZEn9iEl0fwbnZzYdByGHf+PKud5cR3o4D70v6eDSXGmkeXuxB7vruY2u2Enw51leqxYewrdE4ezqfLvLdozYsxf0gHQIe4Mdtb+o0Vb58Qx2G3mFi8AYp1dw9pSYs7EZcEyC5ejMw5bGv5geXOb3ZaE29HJ9FgOeyYJMSOpbXjjJzmYv6TDZjjoED+WGm/LIkZ63AjTYwH7nK2SGT8Kl4X7e/zUyJEjeeGFF0hJSeHss8/GMIwDnr9y5cqIr71kyRIWLlxIly5dWnwiCE39sdvtbtHmdruprw8vZoqItCdW9rs/1VZj5FRXIqdlHMMHpS0/AMiNyzjka4mImCmiAsa9997L9OnTm4sX0LSm78wzz8TpdPLHP/6R559/nqlTpzJhwoRDTiI2NpbGxh+ml3s8HuLiwm/q4+LiWpzn9XoJhUL7PDdSZ5zRm8LCUt5YsRG73cbFF5/EgAFdDv6FP8Ox+R25YdyZPPp/H9LQ6OcXg3vw618MwGY78C++nyMvPotZAybywFd/p7yxhm4J2dza92KSLdh8KcGZxG+7Xsv/Fj5KccNOkhwpjOtyFVkx5t8MG4aNY1LPp9a/h2217+EwYhiSMYkOMdZMZ0yPGUzPlKvYWrmYIAFyE8+nY9zwg3/hzxDv6kfP9Llsq5iJP1hFousE8lKnW7KxZoyjE70yH2Nr2TS8/h247J3pnvEgMU7z/+3bbLF0SLmVYEU19d6PsBlJdEy9mxjnMabHAkiLG069bwvf1TyPgY1OSVeQEnOKJbESXf3plnYX2yseIBiqJy12GDmJl2MY1myWuy+DBg3C4XA0//lgA+lDMXHiRP7f//t/LF++nHHjxvH6668TE9NUvIuLiwt7hHZDQwPx8dZt3isicjiwst/9qbYaI8fYXVzZ41xq/fV8VllAgiOG63qNpUei+WM7EZFDEdEoe9u2bS2quj+WlZXVvP7PMAwCgcA+zzuQbt26UVhY2Py6sLCQHj16hJ2Xn5/PJ5980vx6+/btZGZmNq8L/DmyspL57/8+mwsvPBGbzUZ2dgp2uzVPfUiIczP+rEEMPb47Pl+AjulJxLrNXTryPbth45SMfvzPkOup9XtIdyWT5LJuSntuXD7X9byTal8FsfY4UlzWfQKd7OrML7JvZ4hvEnbDRaIz27LBg9uRTq/Ua8hN+CUhAsQ6OmG3mb/JJYDNcJGZMJbEmBMIButw2bNx2K3bKCsxZgjHZL2EL1iG05aO02Hdpyoxzp7kpj+FL1CMzRaHy5FrXSxHNt3SbicncSKGYRDjyLWsoOCwJ5CTeClpsf9FMOQlxpFj2ZNc9mf27NnNf7700ks59thjW33NrVu3smfPHk499VQMw2DMmDHMmjWLbdu20bdvX6Cp3965c2fz14RCIQoLC7V8RETaPSv63f1pyzFyXnwW9w68gr0NlcTaXXQ0eQ81EZGfI6I79by8PO677z5KSlquhduzZw9z586lQ4cO+Hw+Fi5c2Dy4PRTnnnsuy5cvp7S0FL/fz1//+ldGjx4ddt6IESNYs2ZN89q/p59+mjFjxhxyvJ9yOh106ZJB585plhUvvmcYBp0zU8jPSbesePFjmTEp5CdkW1q8+F68I4Hs2FxLixffc9piSHV3JcmVY+knH9C0LCHe1YUEV75lxYsfi3F0Is7Vy9LixfecjgziXL0tLV58z25PJMbV29Lixfdshos4VzdinfmWz4YwDINYZy7xrh5RL1781G9+8xtOPvlkfv/73/P3v/+d4uLwPX4iUV5ezs0338yePU37Aa1duxafz0du7g8/ux49epCWltb81JGXX36ZTp06tXhilYhIe2dWv7s/bT1GjnfEkJ/QUcULETlsRLwHxrXXXsvQoUNJSkoiMTERr9dLWVkZNpuNv/zlL9TV1bF69WoWLVq03+uUlpZy6aWXNr/+7W9/i91uZ8mSJVxxxRVMmDCBUCjEqaeeyvjx4wF48803Wb16NbNnzyYrK4u77rqLa6+9lkAgQL9+/bjjjjta+VcgItI+rFixgo8//phPP/2URx99lBkzZpCXl8epp57Kaaedxsknn0xCwsGXkQ0ZMoQpU6Zw+eWXEwwGcblczJ07l7q6OsaNG8eyZcsAePDBB5kxYwbz588nPT2dBx54wOpvUUTksGJWv6sxsohIZIxQKBQ6+GlNn8i99dZb7Ny5k8rKStxuN3l5eQwfPrz5kU3l5eX7XWoSLUVFRQwfPpxVq1aF7ZAvInI4sbq/2rVrF5988glr167lww8/ZO/evXz+uTVPZNkf9ckiciQwq686HPrd/VF/LCJHigP1VxHPrU5LS+Oiiy466DkiItL2iouLWbt2LevXr2ft2rXs2bNH+1OIiFhI/a6IiPUiLmB89tlnLF68mM2bN1NaWsorr7xCeno6TzzxBNdff72VOYqISASee+451q5dy9q1aykvL6d///4MHjyYW265hUGDBpGYaP2+KiIiRxP1uyIi0RVRAePtt9/muuuuo0+fPgwbNoxnn30WgMrKSpYuXUpsbCxXXXWVpYmKiMiB3X333eTk5HDxxRczbtw4UlJS2jolEZF2Tf2uiEh0RfTIjXnz5nH55Zfz0ksvccstt2C32wHIycnhrrvuYunSpZYmKSIiBzdnzhyGDh3Kq6++yumnn85vfvMb/vznP/PWW29RUVHR1umJiLQ76ndFRKIrohkYBQUFzJ07d5/HjjnmGHbv3m1qUiIicuhGjx7d/Hi90tJSPv30Uz7++GMefvhhCgoK6NatW/NjT0VEpPXU74qIRFdEBYyMjAwKCwvp2rVr2LEdO3aQnJxsdl4iItIKGRkZ9OzZk5qaGurr66mpqWHbtm1tnZaISLulfldExHoRFTBOPfVUZs6cyYwZMzjppJMwDIPGxkbWr1/PPffcw/Dhw63OU0REDmLt2rWsW7eueRf86upqevTowcknn8ydd97JiSee2NYpioi0K+p3RUSiK6ICxi233EJhYSFTp07FMAxCoRBjxowB4IQTTuDmm2+2NEkRETm4CRMmkJ2dzcknn8ztt9/OKaecQmZmZlunJSLSbqnfFRGJrogKGAkJCfz1r39lw4YNbNiwgbq6OhITExkwYAADBgywOkcREYnAG2+8sc+lfiIiYg31uyIi0RVRAeN7AwcOZODAgVblIiIiraBBtIhIdKnfFRGJrogKGBUVFSxYsICNGzdSVVW1z3NWrlxpamIiIiIiIiIiIt+LqIBxxx138OGHH3LiiSfSrVs3DMOwOi8RERERERERkWYRFTD+/e9/M3/+fM444wyr8xEREQt4vV4qKyvJyspq61RERI4K6ndFRMxni+Qkt9tNXl6e1bmIiEgr9O3bl7Kysn0e27ZtG2PHjo1yRiIi7Zv6XRGR6IpoBsYFF1zAsmXLmDp1qtX5iIjIIXrllVcACIVCrFixgoSEhBbHQ6EQH3/8MV6vty3SExFpd9Tvioi0jYgKGLm5uTz99NOsXbuWY489ltjY2BbHDcPg6quvtiRBERE5sGXLlrFp0yYMw+Cee+7Z5zmGYTBp0qQoZyYi0j6p3xURaRsRFTDuuusuoGkq3Icffhh2XAUMEZG28+STTxIKhejbty+vvvoqaWlpYeckJibidrvbIDsRkfZH/a6ISNuIqICxefNmq/MQEZFWMAyD1atXk5OTs99zPvnkE4YMGRLFrERE2i/1uyIi0RfRJp4iInL4O++883j++efD2uvr65k5cyYTJ05sg6xERNov9bsiItGlAoaISDtx1VVXcd999zFx4kSKiooA+PDDDxkzZgzvv/8+jz76aBtnKCLSvqjfFRGJLhUwRETaiauvvprXX3+d5ORkzjvvPK655hquueYaRo8ezfLlyznzzDPbOkURkXZF/a6ISHSpgCEi0o5kZ2dzww03kJWVxTvvvMOJJ57I5MmTiYmJaevURETaJfW7IiLRs98Cxuuvv059fT3Q9KzrxsbGqCUlIiKHzuv1MnfuXMaOHcvAgQN59tlnqa2t5ZxzzuGVV15p6/RERNod9bsiItG136eQ3Hzzzbz66qt069aN6dOnM3To0H0+IkpERA4P55xzDgDz589vnrb8/PPPs2TJEu6++25eeuklnnnmmbZMUUSkXVG/KyISXfstYHTq1InLLruMrl27EgqFuPbaa3E6nfs81zAMlixZYlmSIiJycEOHDuWmm24iISGhuc0wDC677DKGDx/OnXfeGfG1Vq1axbx582hsbCQlJYW7776bXr16tTind+/e5OfnN7/OysrS7wIROaqY2e+KiMjB7beA8dBDD7Fo0SKqqqowDAOXy7XfAoaIiLS9u+++e7/HcnNzWbx4cUTX2bNnD7feeivPPfccPXr04Nlnn+XOO+/c56MC33jjjZ+dr4jIkc6sfldERCKz3wJG3759efDBBwEYNmwYDz30EKmpqVFLTEREDt1nn33G4sWL2bx5M6Wlpbzyyiukp6fzxBNPcP3110d0DYfDwZw5c+jRowcAgwcPZu7cuVamLSJyxDKj3xURkchE9BSS1atXk5qaSiAQYOvWrWzcuJGCggKCwaDV+YmISITefvttJkyYQFFREcOGDcPn8wFQWVnJ0qVLefzxxyO6Tnp6OkOHDm1+/d577zFw4MB9njtt2jRGjRrFhAkTWLduXeu/CRGRI4hZ/a6IiERmvzMwfiwYDPLggw+ydOlS6urqmtsTExOZOHEi1157basTeeONN3jooYdatG3bto21a9c2ryssKipi5MiR5ObmNp8zYMAA7r///lbHFxE50s2bN4/LL7+cadOmATQv+cjJyeGuu+7i/vvv56qrrjqka65Zs4YlS5bsc2+Liy66iAkTJtCnTx9ef/11pkyZwptvvklSUlLrvxkRkSOAFf3uj2l8LCLSUkQFjHnz5vH888/z29/+lv79+xMfH09tbS3r1q3jiSeeICYmhkmTJrUqkXPOOad5J2doeozrihUrWmyKBE2bxGnNtYhIuIKCgv0u9TjmmGPYvXv3IV3vrbfeYtasWTz22GPNy0l+bNasWc1/HjVqFI8++ijr169v3olfRKS9M7vf/SmNj0VEWoqogPHqq68yc+ZMfvnLX7ZoP+uss+jWrRtPPPFEqwsYP+b1enn44Yd54oknTLumiEh7l5GRQWFhIV27dg07tmPHDpKTkyO+1kcffcS9997LU089Rffu3cOO19XVsWfPHrp169bcFggEcDgi+rUiItIumNnvHozGxyIiEe6BsXfvXgYNGrTPYyeffDLFxcWmJvXiiy8yaNAgunTpEnastraWqVOncs455zBp0iS2bt1qamwRkSPVqaeeysyZM1m9ejV1dXUYhkFjYyPr16/nnnvuYfjw4RFdx+PxMH36dObPn7/P4gXA7t27GTduHIWFhQB88MEHVFRU7HevDBGR9sisfjcSGh+LiEQ4AyMtLY2CggI6d+4cdmzLli2mPp0kGAzy1FNP8dhjj4Udi4+PZ8yYMVxxxRXk5OTw9NNPM3XqVJYvX65P/UTkqHfLLbdQWFjI1KlTMQyDUCjEmDFjADjhhBO4+eabI7rOqlWrKC8vb17T/b1FixZx9dVXs2zZMrp3785tt93GlClTCAaDJCcns3DhwrBpzSIi7ZlZ/e7BaHwsItIkol5t5MiR3H777Vx//fUcd9xxJCQkUFNTw7p165g/fz6jR482LaH169cTFxdHz549w46lpqZy5513Nr++/PLLWbBgAdu3b9/n+mwRkaNJQkICf/3rX9m4cSMbNmygtraWxMREBgwYwIABAyK+zpgxY5oH4D+1bNmy5j+ff/75nH/++a3OW0TkSGVWv3swGh+LiDSJqIAxbdo0SktLmTFjRot2wzAYM2YMN954o2kJvfPOO/vdAK6qqorq6uoWuywHg0FVl0VEgOnTp3P77bfvc+C8bds25s6dy7x589ooOxGR9ida/a7GxyIiTSLaA8PtdjN37lzeeecdFixYwH333cfChQt5++23uf/++3G5XKYltHnz5v2uud60aRMTJ06kvLwcgKVLl5Kdnd2iwxYROVq98sorNDY27vPY1q1befvtt6OckYhI+xatflfjYxGRJodUms3KyiIrK8uqXICmjeEyMjKaX2/cuJGHH36YRYsWcfrpp3PJJZcwfvx4DMMgKyuL+fPnY7fbWxVzb1E5O775DrvdRl6fHNKyzNsx+qcqauv59rsyGhr9dM1KJTcjxbJYdb5Gvqkspayhjs4JKfRITsdhi6hmdcgCoQC7G4op85aQ6EwiJ6YzbnuMJbEAqht3UdW4A4fNTaqrGzEO6/4eG/wl1Pm+JRgKkODqRqwjx7JYvkA19b5v8AeriXXkEevshmEYlsQKhrx4fVvw+YtxOrJxO3pis1n3M/P4tuL1bcduSyLW2QuH3br3WaO/GI9vCwY2Yly9cNmt67caAxXUNH5LIOQh3tmVeGf45mpW69OnT/O/k9NOO22/5/Xr1y9aKYmItGvR7nfbYnwMUFBdRmFtOYlONz2TM0l2xbb6miIirXHYzS177bXXWrweMGAAixYtan49efJkJk+ebFq87V/t4s7xCyjZ1VS17jGwC9OfuJKc/EzTYnxvd0UNf3z+TT7c3LRrf3JcDI9N/RX9cs2/uarzNfI/n/+beRs/AsBh2Fhw5lhG5vUyPRbApqr1PFkwnyABAEZ1vICzskZbUsQobdjMyqI/4A1WA9Ap7iROy7qZBKcVf4872LDnBmp8XwEQY89mUMdHSXSZ//foC1SwveJBdte+AIDNiOGYDk+SEnuy6bFCIT+Vdf/HroqbgRBgkJNyL2kJ4zEMp+nxaho+5puSiQRDHgDS435NbuptOO3ppsfyNH7D1pIraAzsACDW0Zf8zMeIceabH8u/l89L/0iJ5x0AHLYkTuz4P6S4+5se60A++OADPvvsM6677jquueYa4uLiws5JTk7mrLPOimpeIiLtVbT73WiPjwE+KdnBpPefwxPwAXBBXn9uGTCC9Jh4U+OIiByKw66AEU2hUIg3/vZhc/EC4NsNO1j39pfk5O97nWFrbNz+XXPxAqCqvoHHVvyLBy4fjdtp7o/im8rS5uIFgD8U5JY1Kzg2PYtOCeZ+8l3mLeHZwiebixcAr+9+mWOSB5Ifb+7mUf6gl/Vli5uLFwC76v9NScMXlhQwyuo/bC5eADQEvmNXzUv0TrvV9JkRtY1fNRcvAIKhBraUzWBg9lJcdvOe9APg9RVQXHE7TcULgBDFlXcS5z6JWFdvU2P5A1Vsr7izuXgBUFb/EunxY0mOHWpqLICyupeaixcAHv9XVHneIsZ5pemxqrybmosXAP5gNd+Uz2dQ1sM4bNH7lCojI4MRI0Ywe/ZsRo8ebeqyPhERCdfe+92qRg93r1/RXLwAeLlwE+d1OZYzOu57KYuISDQc1QUMn9fPF//6Nqz968+2MwbzCxg7SyvD2r7YuYdaj9f0Akappy6srdLbQKW3wfQCRn2gjrpAbVh7ta/K1DgAjcE6Shs2h8dq3GV6LICqxs/D2ioa1hIMebEb5s4uaQyUhLU1+LcRCFaDyQUMf7CcED9ds+snECwFzC1gBILVNPjCf2aNgb2mxoGmmSV13n+Htdd515seC8DjLw5rq2r8An+wJqoFjO9dcMEFlJSU8NVXX1FdXU0oFAo757zzzot6XiIi7VV77XerfQ18XRU+LilpCB/viYhEU0R3zWvWrGHQoEG43W6r84kqV4yToecP5tuNO1q0D/6FNevE+3QKX5YyfEAPUhLMv9HpnJCM3TAI/OgXaV5iCh3jEkyPlexMJdOVRUnjnuY2GzbSXeYvw4mxJ5GXMJTNVS+3aE93hz9WzAwZsadSXNsyVlb8udgt2Csi1hG+d0JyzMk4bRn7OLt1XI4cbEYywdAPRSabkYDT3sn0WA57Okkx/0V1wzst2mMceabHMgwHKXHnUde4rkV7cuxw02MBJDrDP4XKihtu+oyZSP3jH/9gxowZ+91QzjCMI3IgLSJyuGqv/W66O54zs7rz7p6tLdq7xLfN7zcRke9FVMCYOnUqK1asoGPHjlbnE3VnjB3Mls928P6ra7HZbZw36b8YcKo1+0Qc2zWba0edwuMrP8YXCHBSz1wuOfN47BZsrNkzJYNHzhzLLR+toLrRS5eEZB4+4zzSY81ft5jkTOay/Cks3raQ0sa9xNrjuLTLlWTHmn8zbDMcHJN6IVWNO/jOsxab4eS4tIlkxFpTdEqNOZEuib9lR82zQJCsuJFkx59jSax4V196pP+JgvJ7CIbqiXP2oVvaHTjs5v/MXI4u5GU8zs6y/8Yf3IPDlknn9IdxO7uaHstuiyM3ZTrbykqo932BzYilc8ptxLmOMT0WQErsOdQ3bqCi/h+AQUbCBBJj9r/BWmskuY+lV+oNfFvxCEF8pMYMplvy5dgs2EckEgsWLGDYsGFcccUVpKWlWbYBrIiINGmv/W6cw8XNA4dT8nEtX1buIdbu5JYBw+mX2v7uBUTkyBJRAeP888/n0Ucf5fbbb293a/w6dsngD49MZPyNo7DZDXLyO+B0WbOyJjkuhitGnMhZx/Wi0R+gU3oSCTHWzGpx2Gycm9ebY9M6UtXooWNcIhkWFC++lx/fg2m976LSV0G8PZ50t/mzL76X7OrC8Jw/UeP7DrvhJMnVCZthzc8sxpFJr7QbyU26iFAoQJwzF7tFSwPsthg6JlxISsxJBIJ1uB3ZOC38JD8h5jS6Zy0jECzFbsvA5ci2LFacqze9O/wvXn8Rdls8bkeeZYM8t7MTXdLuo2PSVMCO29kFm2HN+8xlT6J78mV0jBtOMNRArKMTTnuiJbEisXfvXp588km6dIn+k1BERI5G7bnf7ZXcgb+eeSm76qqId7jokpDabgo0InLkiuiur7a2lvXr13PKKafQo0cP4uNb3ggbhtFiJ+QjTUysi/x+5s8W2BeH3UZ+VlpUYgHkJiaTi3WPq/yxJGcySc7oxHLZ40m3m7tB6P7YbS4SXNHZsMowDGKd5i+t2J+mooV1hYsfc9iTLX106o/ZbbHEuvpEJZZh2ElwdY1KrIPp3bs3e/fubZcDaRGRw1F773eTXbF6dKqIHFYiWruwe/dusrOz6devHy6XC5/P1+K//a37ExGR6JkxYwbz5s3js88+w+fzHfwLRESkVdTviohEV0QzMJ555hmr8xARkVa69tpr8Xg8jB8/HgC73R52zuefhz9ZR0REfh71uyIi0XVIGwfU1tbyzTffUFpayumnn05cXBx+vx+H46h+GquIyGHhwgsvbOsURESOKup3RUSiK6LKg8/nY/bs2SxduhS/349hGPzzn/+koqKCyy67jKeffppOnaKzh4SIiOzbdddd19YpiIgcVdTviohEV0QFjIceeogVK1Ywffp0hgwZwsUXXwxAWloa3bp1Y86cOfzlL3+xNFERETm4kpISXnnlFb788ktqa2tJTExkwIABjB07ltRU655qIyJytFK/KyISPREVMF599VVmzZrFiBEjWrTHxsZy3XXXMWnSJEuSExGRyG3atInLLrsMv99Pfn4+8fHx7Nmzh3/+8588/vjjPPPMM3TvHp0n6oiIHA3U74qIRFfEj1Ht1avXPo8lJyfj8XhMTUpERA7dnDlzGDJkCA8++CAJCQnN7VVVVdxwww088MADPPbYY22YoYhI+6J+V0QkuiJ6jGrXrl1Zvnz5Po+988475OXlmZqUiIgcug0bNnDDDTe0GERDU6H5xhtv5NNPP22jzERE2if1uyIi0RXRDIwJEyZwxx138OWXX3LSSScRDAZZvnw5xcXFvPzyy8ycOdPiNEVE5GCCweB+nwoVGxuL3++PckYiIu2b+l0RkeiKaAbGb37zG2bPns3WrVu555578Hq9PPTQQ3z66afMnDmTX//611bnKSIiB9GvXz/+9re/EQqFWrSHQiGeeOIJ+vbt20aZiYi0T+p3RUSiK6IZGAAXXHABF1xwAbW1tdTV1ZGQkEB8fLyVuYmIyCG4/vrrmTx5Mu+++y79+/cnISGBmpoaNm7cSEVFBU8++WRbpygi0q6o3xURia6ICxh79+7lrbfeYtu2bTQ0NBAXF0ePHj0YPnw4aWlpVuYoIiIROPnkk3nxxRd59tln+eKLL6itrSUhIYGhQ4cyceJE7YQvImIy9bsiItEVUQFj1apV/OEPf6CxsZHMzExiY2Opr68REZk+AAAgAElEQVSntLSU2bNn88gjj3DqqadanauIiBxEnz59mDVrVlunISJy1FC/KyISPRHtgXH//fdz+umn88477/Dee++xcuVK3n//fd59912GDBnCPffcY3WeIiKyH9XV1SxYsGCfxwoKCrjvvvtoaGiIclYiIu2X+l0RkbYRUQGjuLiYP/zhD2RlZbVo79ChA9OmTaOoqMiS5ERE5MBqa2uZMGECjz/+OMXFxWHHt23bxosvvsiUKVO0G76IiAnU74qItJ2IChjdu3enoqJin8eqq6vp1q2bqUmJiEhknn76aSoqKnj55ZfJyckJOz58+HCWLl3Kli1beOGFFyK+7qpVqxg7diznnnsu48eP55tvvgk7Z/PmzYwbN46RI0cybtw4Nm/e3KrvRUTkSGBVvysiIgcXUQFj5syZPPzww6xZswaPxwNAY2MjH330EfPmzWPmzJlW5igiIvuxcuVKrr/++gMWkvPz87n++ut56aWXIrrmnj17uPXWW5kzZw4rVqxgzJgx3HnnnWHn3XDDDUyePJmVK1dy5ZVXctNNN/3s70NE5EhhRb8rIiKR2e8mnscccwyGYTS/DgaDXHHFFQAYhtH8vGubzcbVV1/Nv//9b4tTFRGRnyoqKuKEE0446Hknnngi9913X0TXdDgczJkzhx49egAwePBg5s6d2+Kcr7/+mpqaGkaMGAE0feI4Y8YMtm7dql33RaRds6LfFRGRyOy3gHHNNde0KGCIiMjhx2aLaCIdPp8v4mump6czdOjQ5tfvvfceAwcObHHO9u3b6dy5c4u23NxcCgoKVMAQkXbNin5XREQis98Cxu9+97to5iEiIj9Dz549+eCDD8jPzz/gef/85z/p2bPnIV9/zZo1LFmyhCVLlrRo93g8uN3uFm1ut5v6+vpDjiEiciSxut8VEZH9228B46c++eQTvvrqK2pqapqXj3zPMAyuvfZa05MTEZEDu+CCC5gzZw7HHXcc/fv33+c57777Lo8//vg+97E4kLfeeotZs2bx2GOPNS8n+V5cXBxer7dFW0NDA/Hx8Yf2DYiIHGGs7HdFROTAIipg/PnPf+bpp58mPj6e5OTksOMqYIhIWwh56/Gt/h+cw67GcMe1dTpt4sILL+Ttt99m/PjxjBw5kjPOOIPs7GyCwSA7d+5k9erVvPvuu4wcOZJf/epXEV/3o48+4t577+Wpp57a55KQbt26sXPnzubXoVCIwsJCLR8RkXbPqn5XRKS9sWKsHlEB45VXXuGOO+7g0ksvNSXoTxUVFTFy5Ehyc3Ob2wYMGMD999/f4rzNmzczc+ZMKioqSE1NZebMmfTp08eSnETk8Bfcvo7gtrUEt6/D3vv0tk6nTdhsNhYuXMjixYv529/+xvLly1tstNylSxfuuOMOLrnkkoiv6fF4mD59OgsWLNhvQaJHjx6kpaXx2muvcd555/Hyyy/TqVOng06pFhE50lnR7+6PxsgiciSzYqweUQHDZrNx5plnmhJwf7KysnjjjTcOeM4NN9zAjTfeyIgRI1i1ahU33XQTr732mqV5icjhy//Vu03/3/zeUVvAgKY+etKkSUyaNIni4mJKSkowDIOOHTvSoUOHQ77eqlWrKC8vZ9q0aS3aFy1axNVXX82yZcsAePDBB5kxYwbz588nPT2dBx54wJTvR0TkcGd2v3sgGiOLyJHKirF6RAWMiy++mJdeeonf//73pgT9OfTIPhHx/mM2oV1f/NBga+rCQt99Q8PCH2aIGZ2OwT12erTTOyzk5OSQk5PTqmuMGTOGMWPG7PPY98ULgN69e7N06dJWxRIROdKZ0e+2hsbIInK4iMZYPaICxlVXXcXll1/OueeeS58+fYiJiQk7Z/bs2T8rge/V1tYydepUCgoK6NSpE7fddluLTleP7BMRxwlj8e3ZAv7Gpoagv+X/ARwuHCeMjX5yIiIiFtAYWUSOFNEYq0f0IOtbbrmFDRs24HA4KC0tpaioKOy/1oiPj2fMmDHcdtttvP7665x22mlMnToVv/+Hb1SP7BMRe6d+OEdPA4dr3yc4XDhHT8PeqV90ExMREbGAxsgiciSJxlg9ohkY7733HvPnz2+emma21NTUFo+Zuvzyy1mwYAHbt29vfnSfHtknItDUMXL27/CtnAcB348OOHGe/TsVL0REpN3QGFlEjjRWj9UjmoGRnp5u6c7yVVVVLR7HBxAMBnE4fqiv6JF9ItLMWwc2OxgG2F1N/7fZmtpFRETaCY2RReSIZOFYPaICxrRp03jkkUeoqKhodcB92bRpExMnTqS8vByApUuXkp2d3eKRUT9+ZB+gR/aJHMX8X70LPi9Geheco27ASO8Cvkb8m99r69RERERMozGyiByJrByrR7SE5JlnnuG7777j9NNPJyMjY5+beK5cufJnJ3H66adzySWXMH78eAzDICsri/nz51NaWsqkSZP0yD4RacFwxWI7dTyOgedgGDZsnY7Bv2EFoe++aevURERETKMxsogciawcq0dUwMjLyyMvL6/VwQ5k8uTJTJ48Oaxdj+wTkZ9yjfpDi9eGzYbz+NFw/Og2ykhERMQaGiOLyJHGyrF6RAWM1j4iVURERERERESkNSIqYBQXFx/0nJycnFYnIyIiIiIiIiKyLxEVMIYNG4ZhGAc856uvvjIlIRERERERERGRn4qogPGXv/wlrK2uro4NGzawYcMGpk2bZnpiIiIiIiIiIiLfi6iAMWrUqH22X3jhhbzwwgssX76cM88809TERERERERERES+Z2vtBU499VRWr15tRi4iIiIiIiIiIvsU0QyMA3nnnXdwOFp9mTZVVlzBjs1F2B128vp1JjkjybJY1bUeCovKaWjw0aVzGlkWxmrw+dhaXk5ZXT2dkpPJT0vFdpC9TH6uYCjIdw17KPWWkexMIiemIy67y5JYAOXeMnY37MZld5Edk0O8I96yWB5/JeWNOwiGAqS6upDgTLcsljfgodRbhCdQS5qrI2nubMtiBUJ+Khp3UusrIcGRQaq7C3bDuvdyTeMuqn1FuOwJJDu74rJb9zNr8JdS01iAYdhJdHbD7Ui1LJYvUEONrwB/0EOCM484p3U/MxERkWjaWVbJzrJKEmLcdOuQTkKMdWM7EZFIRHS3cvbZZ4dt4hkKhaiqqqK6upqLL77YkuSiofCrImb+6gGKvm560kr/M/py09PXkp2fZXqs0rJaHl60inc++gaA9NR4Hpjxa3p2Mz+Wx+fjb+s/4/533ycEuB0OFowdw39162Z6LICNlV/w0JZH8YV8GBiMy/0VZ2X9ArfdbXqsHfWFPLxlLlW+SgAGp5zAuC4TSHWZf5Na1fgd//zuAYo9GwFIcXZmTOe7SXfnmR6r3l/D+yV/Z03ZPwBw2+K4JG8GefH9TI8VCgXZUv0Ob373AEECGNgY0fFG+iSPwGbYTY+317OJVcXTaAzWANAn+dcclz4Ztz3Z9Fg1jdv4ZM80anxbAUh1D2Rwh3uId+aaHqvBX8rnZXPYVbcCALc9jZM7LiDF3df0WCIiItH0WWExU596hWqPF4Dxpw7k2rNOISU+to0zE5GjWURLSAYNGhT23+DBg/nlL3/J7NmzmTFjhtV5WiIUCrFy8dvNxQuATe9/xbo3N1oS74tvipuLFwBlFXX89cV/0djoNz3WltIy7vtP8QLA6/dz84qVFFdXmx6rzFvO/xQsxhfyARAixHM7X6LIc/DH7x4qX9DHsuJXm4sXAGsrP6WgbqvpsQB21K1tLl4AVPqK+LJqJaFQ6ABf9fPsbtjWXLwA8AbrWVb8KPX+GtNjVTQW8dbuOQQJABAiyOo9D1HRWGR6LG+gho/3/qW5eAGwueolyhq+OcBX/XxFta83Fy8AKrwb2F3/viWxKryfNxcvALyBcjaXP0og2GBJPBERkWio9jTwp1febi5eADz30Qa+Ki5pw6xERCKcgfHnP//Z6jzaRKPXx2erPw9r//Jf3zD6qrNMj7djV3lY26avdlFb7yXNZe7U/b21dWFt5fUeKj0N5CSZu2ylxl9L9T5usit9VabGAfAE6tla921Y+56G3abHAtjdEP544KK69QQyGnEY5s4uqfWH//so8e7AE6glzpFoaqz6QCWB/xScvhcI+aj3V5g+u6QxWENZ49f7yMH8QVAw5KfE8++w9nLPZ3RPvsT0ePX+8IJPuXcDjcFqYm0xpscTERGJhhqPly937Q1r31tl/ocqIiKHYr93zcXFh/bpeU5OTquTiTZ3jIvTLjiRLesKWrQfP6y/JfG6d80MazvlhO4kJZh/o9MpORGbYRD80UyBnKREMuPN33cgxZlMhiuN0sYfbsANDDJc5u8VEWePZ0DSQN4ve69Fe+dY85cHAHSOO54vq1a2aOueeDoOm/lLY1Kc4UuJOsf2Id5h/jKLBEcGLlscjcH65janEUOCI8P0WDG2FDrGDma3Z+1PcjC/z7AZDrLjh1HhbTmLqkPcKabHAkhw5oe1ZcWejsuWYkk8ERGRaEiJi+WkHrn8+9udLdo7pZk/JhERORT7XUIybNgwhg8fvs//hg0b1uL4iBEjopmzqX4x7jROOHtg8+uRl/+C435xrCWx+vXK4ZILTsRma9pPpE/Pjlw89gQcDvP3HOiens79544kzukEIDM+nr+MHkVmggUFDFcyU3tcSYqz6Zea2+bmmm5X0CnW/M0MHTYHZ3c8h65xTTeOBgbnZI0iP96avT1y446jX/LI5td5cSfQK+kXlsTKisnnnI6TmzfSTHVlMzrnKmLscabHSnHlcG7OHbhtTTM73LYEzul0OymuTqbHctrjGJLx3yT+Zw8KGw4Gp19Hmrun6bEAcuLPIit2aPPrzvGjyYy1poCR6j6WXsmTMWh6Dye7+tAz9QrsNm1yJiIiR674GBc3jTmTvIymgrzDbuPGUWfQJyf8wzgRkWja7wyMVatWHfALA4EAzz33HM888wwdOnQwPbFoyenekTuW/oFdW3Zjd9jo1DObmDjzP10HSEmKZdIlpzHyv/rhbfTTqWMKSYnWbITkstsZ268vx+VkU+HxkJ2YSMdEc5ch/FjvxB7MOuZ2yhorSHTGk+XuELbxq1myY3P4fa8bKWnYi9PmomNMFg6b05JYCc4M/qvD7zgu9VcEQwFSXJ1wW/T0DLc9hiHpo+iecBzeYD3Jzg4kOq17ekbXhBMZ3/VR6gPlxNlTSXZZ9/SMtJienNv5UWp93+G0xZPk6ozNoieexDs7MbjDbOp8OzAMG/HOLjhsVr3PkumVdjWdEs4hEGog3tkZl12zL0RE5MjXJyeTv065mOKKKuJj3HRJT8Fhj2j7PBERy+z3DqJTp/1/ErtmzRr+9Kc/sWvXLq699lquuOIKS5KLlvikOHoNtuYT/J9yOR10y4tO9dowDLqmptI11bqb4B9Lc6eS5o5OrARHAgkJCVGJ5bTHkGnvHpVYdsNOZow1y2H2JdnVkWQ6RiVWrCONWEdaVGI57fGk2KPzJBC74STJ3SMqsURERKIpPTGO9ETzZ4KKiPxch/QR6M6dO5k9ezZvv/025513Hk8++SRZWeY/AlRERERERERE5MciKmDU1dWxcOFCnnnmGfr27csLL7zAgAEDrM5NRERERERERASIoIDx4osvMnfuXOx2O7NmzWLs2LHRyEtEREREREREpNl+Cxhr167l3nvvpaCggCuuuIIrr7yS2FhrNsITEZHDi8/nY86cOSxevJh3332Xjh3D90rp3bs3+fk/PEo2KyuLJUuWRDNNERERETmK7LeAMWHCBBwOB7/85S+x2WwsWrTogBe67rrrTE9ORETaxtSpU+nfv/9Bz3vjjTeikI2IiIiIyAEKGEOGDAGaNu7cuXPnAS9i1eMyRUSkbUydOpXjjz+eBQsWtHUqIiIiIiLAAQoYzzzzTDTzEBGRw8jxxx8f0XnTpk3jyy+/JDU1lRtvvJFBgwZZnJmIiIiIHK1sbZ2AiIgcmS666CImT57M66+/zoQJE5gyZQrV1dVtnZaIiIiItFMqYIiIyM8ya9Ys+vTpA8CoUaPo0KED69evb+OsRERERKS9UgFDREQOWV1dHQUFBS3aAoEADsdBn84tIiIiIvKzqIAhIiKHbPfu3YwbN47CwkIAPvjgAyoqKhg4cGAbZyYiIiIi7ZU+KhMRkRZKS0u59NJLm1//9re/xW63s2TJEiZNmsSyZcvo3r07t912G1OmTCEYDJKcnMzChQtJSEhow8xFREREpD1TAUNERFrIyMjgjTfe2OexZcuWNf/5/PPP5/zzz49WWiIiIiJylNMSEhERERERERE57KmAISIiIiIiIiKHvcNqCcmqVauYN28ejY2NpKSkcPfdd9OrV68W5/Tu3Zv8/Pzm11lZWSxZsiTaqYqIiIiIWE7jYxGRHxw2BYw9e/Zw66238txzz9GjRw+effZZ7rzzTp5//vmwc/e3NltEREREpL3Q+FhEpKXDZgmJw+Fgzpw59OjRA4DBgwfz7bfftnFWIiIiIiJtQ+NjEZGWDpsCRnp6OkOHDm1+/d577zFw4MB9njtt2jRGjRrFhAkTWLduXbRSFBERERGJGo2PRURaOmyWkPzYmjVrWLJkyT7X7l100UVMmDCBPn368PrrrzNlyhTefPNNkpKS2iBTERERERHraXwsInIYzcD43ltvvcWtt97KY4891jxd7sdmzZpFnz59ABg1ahQdOnRg/fr10U5TRERERCQqND4WEWlyWBUwPvroI+69916eeuop+vfvH3a8rq6OgoKCFm2BQACH47CcSCIiIiIi0ioaH4uI/OCwKWB4PB6mT5/O/Pnz6d69+z7P2b17N+PGjaOwsBCADz74gIqKiv2uBRQREREROVJpfCwi0tJhU5pdtWoV5eXlTJs2rUX7okWLuPrqq1m2bBndu3fntttuY8qUKQSDQZKTk1m4cCEJCQltlLWIiIiIiDU0PhYRaemwKWCMGTOGMWPG7PPYsmXLmv98/vnnc/7550crLRERERGRNqHxsYhIS4fNEhIRERERERERkf1RAUNEREREREREDnsqYIiIiIiIiIjIYU8FDBERERERERE57KmAISIiIiIiIiKHPRUwREREREREROSwpwKGiIiIiIiIiBz2VMAQERERERERkcOeo60TEBEREZH2JxQKsaayko8rK6kJ+Em0OzgxJYVTUlIwDKOt0xMRkSOQChgiIiIiYhpfMMhTRUUs3FGIJxDgnMxMkhwOShrrWbijkFi7nald8riic2ecNk0GFhGRyKmAISIiIiKmqPX7+c36dXiDQeb06cuw9HRsP5ptEQyFWFVWyr1bt/Lynt28ePwgEhwajoqISGRU9hYRERGRVvMFg/xm/Tqy3W7eHHIiIzIyWhQvAGyGwVkZmbw15EQ6ut38Zv06fMFgG2UsIiJHGhUwRERERKTVnioqwhsM8sSx/XEcZGmIw2bjyWP70xAMsrioKEoZiojIkU5z9oDKslp2bSvB7rDTKT+TxORYy2LV1zeyo6gMr9dPp5wUMtITLYvl8wfYUVxOZbWHrIwkOndMsSwWwI6ySoqra0iLi6Vreiouh92yWHvqatleU0GMw0H3pHQSXC7LYlU3etheV4o/GCQvIZ10d4JlsRoDPoobdlPnr6eDO4PMmHTLYgVDQYo9JZQ3VpLqSiYntgN2w7qaZklDOXu9pcTaY8mJ7UCM3W1ZrOrGGnZ59mIzbOTEdiDRGW9ZrHq/l+11JXj8PnLj0+kQk2RZrGjy+XzMmTOHxYsX8+6779KxY8ewczZv3szMmTOpqKggNTWVmTNn0qdPn9bF9frY+XUx1WU1ZOVlkt0tq1XXO5BQKMTO3ZXsraghLSmOLtlpOOzWvQd211dTWFtBvMNFt6R04hzW9Vt1/lpKvMWEQiEyY3JIcFj3u8Yf9FDVWIgvWEeiszPxTit/Zn68vm34giU47Vm4HfkYFvZbAf9OQoGdGLYUbI7uGIZ1/VYwUErIvw3DcGDYu2PYretLQsEa8BcQCnkxHPkY9szWXzMUYuGOQub06duieBEK+cBfAMEKsOdgOLo0H3PYbNzevTvTNm/mytzcVm/sGQqFCPi3EwwWY9jScTi6YxjOVl3zaLd3RwnfFewlPjmOzr2ziYmLaeuUROQod9QXMIoK9vLnG/6XrV8WAzDkzD5c98cL6JCTanqssopanlz8Hiv+uQmA7I7J3HvXr+mW3/qBw095G30se/tzHnr6bQLBEHGxLmbf+EuG9M8zPRbAR1t38LsXXqPO24jDZmP6OWfy60HHEOM0f+CwubyEK1f9HztrqwC4sGd/bh50Bplx5hcWiusruWfjMt7fuwWAvsnZ3D/oN3RNzDA9Vp3fw/Lit/i/Xa8TIkSiI4Fb+kylZ2I302OFQiHWlG7gL988Q2PQh9Nw8Ptel3J65vHYLLgZ+LZmO3/a/ChVvhoMDM7vdDbn54wgwYLCQrFnDw99s4QttYUADEzuw5Qe48mKMf9nVu6t5dEtb/F/Oz8BIDsmlb8MnkCvpGzTY0Xb1KlT6d+//wHPueGGG7jxxhsZMWIEq1at4qabbuK111772TE9dQ28tnAli6b/L8FgkISUeP74j1vof0bfn33NA/nwswJuX7CcBq8fh93GrZeP4JzT+uK0oPj6ZcVurvrgBXZ7agD4bY8T+N0xZ5DmNv89UOrdwws7/4dva78EID+uF5fkTSXDHV6Eai1voIpN5Uv4qvJ/AYi1Z/CLnAdJj2ldIWtfQiE/FXX/oKj8ZkI0Yhgx5KU/QnLcSNNjAfi9n+Ipv4JQqBKw4U78A674yzFs5heDgr5vaaz8HSH/ZgBs7rNxJs/EZje/LwkF9hKs/jMh76tNDfau2FMWYjh7teq6ayor8QQCDEv/ofAeCnoIeV6AmvuAABiJkLIAw31y8znD0zOoDwRYU1nJqamtG3s1et+hpuJqQqE6wEFC8r3ExF1oaeGpPfv6k2+547w/U7m3CsMwuPjmsVx081gSU637IEdE5GCO+iUkq15Z11y8APjk3c2s/3CLJbG++qq4uXgB8N3uKp5d+i98Pr/psbYVlTPnqdUEgiEA6j2N/PGRFZSU15gea091Dbf83xvUeRsB8AeD3LPibbaWlJseqzHgZ+GmfzUXLwD+vmUT60u+Mz0WwL9KtjYXLwC+qvqO14o2WBKrsG4nL+1aToimn1mNv5YnC/6XWn+96bGKPSXM/U/xAsAX8vPQN39jl6fE9Fh1fg+Ltv2dKl/Tv70QIV7etZJt9dZMGf6gZG1z8QJgQ9Vm1lV8aUmsL6qKmosXAN81VPD4t6vxBnyWxIumqVOn8t///d/7Pf71119TU1PDiBEjABg+fDhlZWVs3br1Z8fctrGQJ275G8H/rIevrazjgcsXULG36iBfeeiKS6qY+dgbNHib+l9/IMjsp96ksNj8fqve38iDm1Y3Fy8Anvn2Uz4v3216LIAvqtY2Fy8AttV/w4bKjy2JVd7wdXPxAsATKGVt6Tx8AfP7rQbfVorKbyJE0++aUKiBHWW/x+srPMhXHrpgoJyGqlv+U7wACOKteZCAb7PpsUKhIH7P883FC4Cg958Evf8yPRZAyLfuh+IFQGA7wfrFTTMlWuHjykrOycxsueeFfwvU/AkI/Cd4DaGqWwgFfvhdYzMMzsnM5JOqSloj4N9FTeXv/lO8APBTWzWdgP/bVl33aFVXVcfC6xdT+Z/+NxQK8fx9r/Dt+m1tnJmIHO2O6gKGt8HHJ+9+Hdb++afbLYm3vbAsrG39hkJqahtMj7W3LLxQUVZZR0WV+YPKsrp6SmrrWrSFQrC7utb0WFVeL2u+2xHWvqUy/O/WDGvLw2N9sHcLDRbcoJY1VoS1ba8votZn/t9jha8Kb7Dl9+AL+aloNP9GsdZfxze14QOeUm/499tagWCAtRVfhLV/UfWN6bEAdtSVhrWtK99Gtc9jSbxoOv744w94fPv27XTu3LlFW25uLgUFBT87ZklR+Pv4u4I9VJdW/+xr7k95VR019d4WbYFgiJJKC/qtxgY+KdkZ1l5U17obtv3ZXLMxvK36M0ti1frDizAlno14g+b/zPyBPYRo2W8FQ7X4A+YXXkPBCoL+8A8zgoHifZzd2mAegt73wmP51pkfCwj5wgu6Ie+HEGrdv/2agJ+knzxNJBTcR5Eu+B0EW77XkxwOavyBVsUPBksJBX/6eyVIIGBNobC9qy6v5ct/hf/uLNlpzXhLRCRSR3UBwx3j5KRh4VOTB5xk/pR9gPyu4VPYhxyfT1Ki+XtuZKUn8tOlpJlpCaQmmz9dOT0+ng6JLa9rGJCdZP4022S3mzNyuoa190o1f3kAwJD08CU3Q7N6EWM3f2lMuistrK1bfB6JDvOnaqa6komxtVx/77Y5SXMlmx4ryRFP38TuYe2Z7vDvt7XsNjtD0sKXPQxIMX86O0DX+PDlX0PSu5PkjLMk3uHE4/Hgdreclu12u6mv//lF0g654e/jzr2ySc40fy+A9OR4khJaruW22210SDW/30pxxXJyh/C+pEuCNfsS9U0aGNbWL3mQJbESHOFLHDrEHo/bZv7PzGHPxqBlv2UzEnGYsH/DTxn2NGyO8H7DZu9keiyMOOzuYeGxnIPNjwUYzmPC29xnNi3vaIVEu4Nqf8sZpYZtH0tg7J3B1vK9Xu33k9jKpVs2WyaG7ad9iA273fylU0eD5PQkjj09/D2QlWfNeEtEJFJHdQEDYNjY4+l7/A8Dy9NGHsvAk8NvtszQr28OY0cf1/w6LzedcReeiMOC9db5uencfOVZzWu5E+NjuOu6c8lMM/9mOCspgQd+fS7JsU03M067nbvPG0GPTPNvUF12B1f3P4keyT9c+9Lex3F8pjV7DpyU2Z0R2T8UuQam5jKm8wBLYuXHd2Zc7lhs/3lbpjqTmZQ/nngLboZzYjK5sffE5iKG2+bihsnxuZEAACAASURBVF6/JSfW/BuBWEcsl+dfSJqr6WbNhsFFuaPJj881PRbAaRmD6JfUo/n1kNT+HJfSz5JY/ZI7My7vFAyaqoVd4tKZ3P0XuO3tf3uhuLg4vN6WMxgaGhqIj//5RdL8/l2YMvcy7P/pt1Iyk5j21LWkZJpfWMvOTOaP15xLfOx/3gNOBzOuPJu8bPP3P4p1OLmx/y/IjW96DxjA5N4nc2yqNf3WMUmD6ZP4QxGjZ8Kx9E8eYkmstJjeHJt6GfznPRDv6MigjOtw2s3vt2Kc3chNn4thNBWebP+/vfuOiuJcwwD+LEUQRAHFEiuWXZAiEFQUKyoQAtcYC+QoGhX1WmK/sfeGaKKxJnajMWqMmiheFYyaWKNoYogUQRFQQRCkS33vH9ydMLDgIrsL4vs7x3Oc2dmZd76deXb2Y4qkHlo13AQ9XdXf20lLywT6DfxL/CDWhp7RfGjrqv5+LBKJBNp1h0Ki8893i5b+v6BVp6vKlwUAEl0HSOoO+2eEtgxaBqMgkVQtt7oYG+NsUhKKiP4ZqdsBMFoC4P+d/hITSOqvhUT7nx/BRUQ4m5SEzg2q1qGnrfMe6htvgUQi7zzTQ70G66Gt06FK831XGdSvi4kbRqNR8+LjLS0tLfguGYp29ubVXBlj7F0nISr5TfP2i4+PR79+/XDhwoUypzeXJz01C09ikqGto4UWbcxgYKS+Oyy/epWHuPhUvMrNR/PmJjA1Vt/TEQoKixD3LBVpGTlo0tAIzRqr/kdASfGpaUhIy4CxYV20MTVR6938k3OyEJOeiro6ujBvYKLWu/ln5r/C46wXwlNIjOuo76/r+UX5eJbzHFmFxU8haain+h9TckSEZzlJSMlPK34Kib5Zle8AX5EXuS///xQSfTSv2wS6Wuq7M3xmflaJp5CYwVBHfZ/Zq4I8PM5+gZyCPLQ0NEVDPeX/ivkmeaVpMplM4VNIoqKiMGrUKFy9ehVA8fbk5OSEw4cPw9z8nwPcyq5jQX4B4iOeIT0lA03bmKFxK9V3qpUUn/gSSakZMKlviFZNTaClpb594HlOJmIzU2GoWwfm9Uyhr6O+fSCnIAtJuc9QBEJjvfdgoKPG75qiXKTnxaKAslFP5z0Y6KrvMyMqQm7BIxQUJkFXuyn0dNuobVkAUFTwBEWF8ZBoNfj/U0jU95kVFaWACmIggS4kOm0h0VLfZ0ZF2UDh/59Cot0GEu2qP/GKiGB39Qq+sLBE/0aNSozPBwpigKKX/38KifgslqDkJPwnPBx3nXuo5DuooCAWRYVPoaXVENo6bSGRKPdHorchj6vqTdYx+ckL4SkkLWXvQVePn+rCGFO/ivKq9v+ZUAn1TQxR30R9Bwol6evXQYf26nvEXEk62lowb6G+x3CW1sKkAVqYqLeTRK5RXUM0qquZz6yerj6sjNVw2rACulq6aGWomWVJJBK8Z9AY76GxRpbXUM8YDfXU+yhfuXq6hpDpauavRPo6dSCrBU8dqaz27dvD1NQUp06dgpeXF06cOIHmzZuLOi/ehI6uDtpYq+fsHEVaNDFGiyaa2S4b162HxnU1c/f+ujqGaKXT/vUTqoCOlh5M9TXzV26JRAv6uu0AXfWcKVmalk5zaOloJpO1tEwBBZcSqoNEywDQsoYqu+skEgkmtWqNVdFR6GNqKjxKVSLRLT4TQ4GCoiKsio7GpFatVdaBrqPTCijxqFZWNY2aN0Sj5po7lmSMsdd55y8hYYwxJpacnAx3d3e4u7sDAHx9feHu7o7ExER4enoK061fvx4HDhyAq6srfvjhB6xbt666SmaM1QBjWrSAnpYW/EL/QsH/nyZUnoKiIviF/oW6WtoYXUvPeGCMMaZ6fAYGY4wxkUaNGuHs2bMKXzt9+rTwf5lMhqNHj2qqLMZYDaerpYVj9g4YcvcO+t/6HQvatUO/ho1Ej1YtIkJwcjJWP4xGXS1t/GBvD10t/nsaY4wx5XAHBmOMMcYYU4l6Ojo49b4j9sbHY3Z4OLILC+FuZob6OsVPKTmblARDbW1MatUao1u04M4LxhhjlcIdGIwxxhhjTGV0tbQwvlUrjGvZEtdfvsSttJfIKChEY4M6+Na2E5yMjdV602jGGGO1F3dgMMYYY4wxlZNIJOhuYoLuJup7ohZjjLF3C5+3xxhjjDHGGGOMsRqPOzAYY4wxxhhjjDFW43EHBmOMMcYYY4wxxmq8WncPjMLCQgBAQkJCNVfCGGMVk+eUPLdqI85kxtjbgPOYMcZqjooyudZ1YCQlJQEAhg8fXs2VMMaYcpKSktC6devqLkMtOJMZY28TzmPGGKs5FGWyhIiomupRi1evXiE0NBRmZmbQ1tau7nIYY6xchYWFSEpKgrW1NfT19au7HLXgTGaMvQ04jxljrOaoKJNrXQcGY4wxxhhjjDHGah++iSdjjDHGGGOMMcZqPO7AYIwxxhhjjDHGWI1X627iWZH8/Hx88cUX2Lt3Ly5fvoymTZsqnC48PBxLly5FamoqTExMsHTpUlhYWAAAAgMDsX37duTn50MqlWL16tUwMjKqVB3Xr19HQEAAsrOz8d5772HNmjVlarl79y7mzZsnGhcXF4fjx4/j77//xqpVq2BmZia8NmLECIwYMULldQCATCaDubm5MNykSRPs378fgObaAwBCQkLg7++PzMxM1K1bF/PmzUPnzp1x8+ZNjB8/Hs2aNROmHTBgAGbNmqWy5at7m1C2DnW1QWXrUPc2oUwdmthHAOVyQxPbR23EmVz5OoDancmcx5Wv413KY4AzWZ2q2raarKOi7V5VLly4gE2bNiEvLw/GxsZYtmwZpFKpaBpNtIcydWiiPc6dO4dt27YhNzcXJiYm1dYeytShifaQu3TpEiZMmIALFy6gRYsWoteU/T5VZx3x8fFwc3NDy5YthXG2trYICAhQ2bKVXYbKtg96h/j5+dFXX31FUqmUnj17Vu507u7uFBQUREREwcHB5OnpSURET548oa5du9KTJ0+IiGjNmjW0bNmyStWQlZVFTk5OFBoaSkRE+/fvp/Hjx7/2fX/88QcNHjyYioqK6Mcff6Q5c+ZUarlVqUMqlSocr8n2yM3NpS5dutD169eJiOjSpUvUo0cPIiK6ceMGjRgxolLLrezy1blNKFuHutqgsnUQqXebqEwdJal6H5FTJjfUvX3UVpzJb1ZHbc1kzuPK10H0buUxEWeyOlWlbTVdR3nbvaokJCSQo6MjPXjwgIiIDh48SN7e3mWmU3d7KFuHuttDvu/Ex8cTEdG+ffto8ODBZaZTd3soW4e620MuOzubPD09qUuXLhQXFyd67U2PL1RdR1xcHPXt21cty63sMlS1fbxTHRh37twhIqowEMPDw8nZ2Vk0rlu3bhQVFUXffvstTZ8+XRj/4MED6tatW6VquHDhAg0dOlQYzszMJCsrK8rIyKjwfUOHDqVbt24REankYKAydZQXAppsj8zMTDp37pwwnJGRQVKplNLS0qp0sKjM8tW9TShbh7raoLJ1EKl3m6hMHSWpeh+Re11uaGL7qK04k9+sjtqayZzHla+D6N3KYyLOZHWqSttqsg75a+qUnJxMly9fFobDwsLo/fffF02jifZQpg4i9bdHYmIiXblyRRiOiIggBwcH0TSaaA9l6iDSXAfG2rVraceOHdS3b98yHQdvenyh6jpqSgeGKrePd+oeGPb29q+dJiYmpszpPy1btsTDhw8RExODVq1aCeNbtWqFFy9eIC0tTekaYmJiRKfXGBoawtjYGLGxseW+59KlS9DT04Ojo6MwLiwsDL6+vnBzc8P8+fORkZGhdA1vUsfs2bPh4eGB4cOH486dO8I8NNUehoaGcHV1FYZ//fVXtGnTBvXr1wcAPH36FGPHjoWbmxumTp2KxMRElS1f3duEsnWoqw0qW4ecuraJytYBqGcfkXtdbmhi+6itOJPfvI7amMmcx5WvQ+5dyWOAM1mdqtK2mqxDTtF2ryoNGzZEr169hOFff/0VnTp1Ek2jifZQpg45dbZH48aN4ezsDAAoKCjAiRMn0K9fP9E0mmgPZeqQU2d7AEBERASuXbuGTz/9VOHrb3J8oY46ACAzMxOTJk2Cu7s7xo4di+joaJXWoMwyVLl9vFMdGMrIycmBnp6eaJyenh6ys7ORk5ODOnXqCOPr1KkDiUSCnJwclcy/PLt27cLYsWOF4TZt2qBfv37Yvn07Tp48iczMTKxevVrpGipbx7Bhw+Dn54czZ85g+PDhmDhxItLT06utPcLDw7F69WosX74cAGBmZgZXV1esW7cOp0+fRuPGjfGf//xHZctX9zahbB0lqbIN3qQOdW4TlalDTh37iCpqVVV7vMs4k9+dTOY8frM6OI+Vr5czuWreJB/UpbztXh2uX7+O/fv3l7nPi6bbo7w6AM21x/79++Hs7Izbt29j9uzZotc02R4V1QGovz2ICEuWLMHChQuhq6urcBpNtIcydRgaGsLT0xPz58/HmTNn4OzsjEmTJqGgoEBldSizDFW2R627ief58+exbt26MuPHjx+PoUOHvvb9BgYGyM3NFY179eoVDA0NYWBggLy8PGF8bm4uiAgGBgZK1/HJJ5+UO39FEhIS8ODBA/Ts2VMY5+DgAAcHB2F4woQJ8PPzU/h+VdSxYsUK4f8eHh7Yvn077t69Wy3tcefOHUyfPh2rVq1C165dAQBt27bFnDlzhGmmTJkCJycnZGdnK6ylpIo+b2WmqUwbVLUOOVW3wZvUoYptQhV1AFXfR6pKE9vH24wzWfV11NZM5jx+szo4j5WvlzO5aiqzLahbedt97969Vbqc4OBgrFixAl9//TXat28vek2T7VFRHYDm2mPUqFEYOXIkAgMD4ePjgzNnzkBfXx+AZtujojoA9bfHkSNH0L59e9GZZqVpoj2UqcPExASLFy8WhkePHo2tW7ciJiZG4bb0JpRZhirbo9Z1YLi6uopO66ystm3bIi4uThgmIjx+/Bjt2rVDYmIibt26JbwWExMDMzMz4ZRRZeq4fPky/vvf/wrDGRkZSEtLQ+vWrRXWc+nSJXTv3h3a2trCuGfPnkFPTw+mpqYAgMLCQujoKP4oq1pHVlYWEhMT0bZtW2GcfHnm5uYabY/w8HBMmzYNGzZsEO2oycnJKCwsRJMmTYT6JBJJuW1SUtu2bXHmzJkKl6+qbaKqdQDqaYPK1qGqbaKqdchVdR+pKk1sH28zzmTV1lGbM5nzuPJ1cB4rrpczWT0qaltNqmi7V6Vr165h1apV2LNnj8J11FR7vK4OTbRHdHQ0EhMT0b17d0gkEnh6emLFihV49OgRLC0tAWimPZSpQxPtceHCBYSGhuLixYsAgJSUFAwZMgQbN26Ek5MTgMplpzrrSEtLQ3p6uuhylqKiIpW2hzLLUOX2wZeQlNK+fXuYmpri1KlTAIATJ06gefPmMDc3R//+/XH9+nXhWp19+/bB09OzUvPv2rUrnj59itu3bwvz6Nu3b7m9/+Hh4WU+2O+//x4LFy5Efn4+CgsLceDAAfTp00ctdSQkJMDHxwePHz8GAFy5cgWpqano1KmTRtuDiDB37lwsWbKkTC/jhQsXMGXKFGRlZQEAvv32W3Tr1k10mmhVlq/ubULZOtTVBpWtQ93bhLJ1yKlrH1GWJraPdxln8ruTyZzHla+D87gszmT1qahtNami7V5VcnJyMG/ePGzevLncH1iaaA9l6tBEe6SkpODzzz8X7ucTEhKC/Px80Q9WTbSHMnVooj127tyJ69ev4+rVq7h69SqaNWuGY8eOCZ0GQOWPL9RVx19//YVRo0YhJSUFAHD06FE0a9ZM1GZVpcwyVLp9VPq2n2+ppKQkcnNzIzc3N5JKpdS/f39yc3OjhIQESkhIoA8//FCYNjw8nIYOHUoDBgwgHx8f0d1RAwMDyd3dnQYMGEDTpk2jzMzMStdy48YN8vLyov79+9OYMWPo+fPnRERl6iAimjBhAn3//feicdnZ2TRnzhwaMGAAubq60ty5cyk9PV1tdZw4cYI++OADcnNzo2HDhgl3hybSXHvcuXOHLCwshM9Q/i80NJQKCwvJ39+f+vfvT66urjRp0iRKSEio0vI1vU0oU4c626Cy7aHubULZOojUu49UlBua3j5qG87kN6+jNmcy53Hl2+NdyWMizmR1UkXbarKOirZ7VTh16hRZW1uX2cfj4+M12h7K1qHu9iAqfoSrfBleXl506dIljW8fytahifYoSf70jz///JPGjBkjjC/v+1TTdezcuZNcXV3Jzc2NRo4cqZbPRdEy1LV9SIiIVNL1whhjjDHGGGOMMaYmfAkJY4wxxhhjjDHGajzuwGCMMcYYY4wxxliNxx0YjDHGGGOMMcYYq/G4A4MxxhhjjDHGGGM1HndgMMYYY4wxxhhjrMbjDgzGGGOMMcYYY4zVeNyBUYP5+vpCJpPh9u3bZV6Lj4+HTCZDfHx8NVQmdvz4cchkMiQkJFR3KdVi8+bN6NixozDs4uKCBQsWVGNFjDF14Ex+O3AmM1a71ITsLZ0rNcH9+/fh5uYGKysrnD59urrLeSOczxWTb98//fRTdZdSo3AHRg2nra2NVatWoaioqLpLEezYsQNz584Vhj08PHDlyhU0bty4GquqOY4dO4Z58+ZVdxm1UlJSEmQyWXWXwd5hnMlvH85k9eFMZppSE7O3uu3evRtZWVn46aef0KdPn+ou57UKCwthb28v6mzifGZvgjswariBAwciOjoaP/74Y3WXIvjjjz9Ew/r6+jAzM4OWFm9OAGBqaop69epVdxm10p9//lndJbB3HGfy24czWX04k5mm1MTsrW5paWkwNzdH+/bt34qMi4yMRHZ2tmgc5zN7E3x0U8O99957GDNmDDZs2IDMzMwKp71w4QK8vb3h4OAAJycnLFy4EBkZGcLreXl5WLRoETp37gxHR0csWbIEgYGBolON09PTsXDhQnTr1g3W1tbo168ftmzZAiICUHwa34ULF3DixAnIZDLcvHlTdLry7Nmz4e7uXqa2pUuXolevXigqKkJRURF27NiBDz/8ELa2tnBxccGOHTuEZSjy6tUrrFy5Er1794a1tTX69OmDtWvXoqCgQJjm6NGj8PT0hLW1NZycnDB58mRRL+/mzZvRq1cv3LhxAx4eHrC1tYWvry9SUlLw3XffoXfv3nB0dMTnn3+OvLw8AP+cin3v3j0MGTIENjY26N27N44cOVJurSVPh7t58yZkMhn++OMPTJkyBQ4ODujRowfWrFkjWt/AwEAMGDAANjY28Pb2RkREBBwdHbFt27Zyl5OXl4eAgAC4ubnBxsYG7u7uOHbsmPD6jh074ODggMTERGHc6dOn0bFjR9y7d084Le306dOYOnUq7Ozs0LlzZyxfvlzUrmFhYRg7dizs7e1ha2uLYcOG4bfffhNel7fRw4cP8emnn8Le3h59+vTBrl27RPWGhIRg5MiR6NKlCxwdHTFt2jRRbfLP56+//sKwYcPQqVMnDBgwACdPnhSWM3nyZACATCYT/cWZMU3hTC7GmVwWZzJnMlMfZbN37ty5GDBggGicPFfl+TN37lx88sknOHPmDFxcXNCpUydMmTIF2dnZ+Oqrr9CtWzd07doVa9asKTP/u3fvYuDAgbC2toarqyuCg4NFrx87dgwfffQR7Ozs0KNHDwQEBAj5BRRn9uzZs7F06VLY2dnh2rVrCtfj1atXWLVqFXr27Alra2u4uLhgw4YNQha4uLjgt99+w++//w6ZTIbjx48rnI9MJsPhw4exbt06dO/eHe+//z4mTZqElJQUYZqMjAwsWrQILi4usLW1xUcffYRffvlFNJ/r16/Dy8sLNjY28PLyws2bNzFw4EDR5R9BQUEYPHgwbGxs0LlzZ3z66acIDw8HUJy9H330EQCgX79+8PX1FdZjwYIFePz4MWQyGQIDA0XLTUpKgqWlJQ4fPgwAiIqKwoQJE9C9e3fY29tj7NixiI6OVrjuclXNZgCIiIjA+PHj4eDggE6dOmHgwIE4d+6cML08v8+dO4fJkyfDzs4OLi4uCA4ORmRkJHx8fGBnZ4dBgwYhLCxM9Pns27cPixcvxvvvvw97e3vMmjULWVlZ5a7P644t4uLiMHnyZHTr1g22trbw9PQUrW+tQKzGGjFiBG3atImysrKoR48e5O/vL7wWFxdHUqmU4uLiiIjoxo0bZGFhQUuWLKHo6Gi6evUq9evXj/z8/IT3BAQEkLW1NR07doyio6PJ39+fBgwYQFKplJ49e0ZERLNnz6bevXtTSEgIPXnyhM6ePUs2NjZ06NAhIiJKTU2lAQMG0LRp0+j58+eUm5tLP/74ozCPCxcukFQqpaioKGG5hYWF5OzsTGvWrCEios2bN5OVlRUdPHiQYmJi6NixY2Rra0s7duwoty2++OIL6tmzJ928eZOePHlCly5dom7dutHWrVuJiOjq1asklUpp37599OTJE/r777/J29ubhg4dKsxj06ZN5OjoSBMmTKCwsDC6fv062dnZ0bBhw2jevHkUHR1NZ8+eJZlMRj/++CMRkbBuQ4YMoStXrlBUVBQtWrSIZDIZ/fnnn8J8LS0theX07duX5s+fL3wuUqmUPv74Y/r5558pNjaWduzYQVKplP773/8SEVFERARZWlrSrFmzKDIyks6dO0cff/wxdezYUVg/RebOnUuOjo70888/06NHj2j37t1kYWFBgYGBRESUn59PgwYNohkzZhARUWZmJvXo0YPWrl0r2ob69u1LR48epZiYGPruu+/IwsKCdu3aRUREiYmJ5OjoSJMmTaL79+9TVFQUzZ8/nzp27Ej3798XtdGIESPo4sWL9PjxY1q5ciVJpVK6d+8eERFFRUWRra0tTZkyhSIiIujOnTs0aNAg8vT0pIKCAtHnM2LECLp58yY9evSIpk6dSlZWVpSQkEA5OTn05ZdfklQqpefPn1N6enq5bcOYOnAm/4MzuSzOZM5kph6Vyd45c+ZQ//79Re8/ffp0mWl69+5NU6dOpQcPHtC5c+dIJpPRsGHD6Msvv6SYmBj69ttvSSqV0o0bN4ioeH+wsLCg4cOH082bNykyMpImTpxI1tbWlJCQQEREx48fJ6lUSps3b6ZHjx7R+fPnqWvXrrRkyRLRuri4uNCiRYsoPj6esrOzFa7z9OnTydnZmS5evEixsbF04sQJsrOzE9b9xYsX5OvrS97e3vT8+XPKyclROB+pVEqurq5CTZcuXSJbW1tatmyZMI2vry/16tWLfvnlF4qOjqaAgACytLSkkJAQYVl2dnb06aefUlhYGF27do0GDhxITk5OQrZGR0eTpaUlBQQEUGxsLD148IAmTpxIffr0odzcXMrNzaXvv/+epFIp/fnnn5SamkpE4nweNGgQTZs2TVT/wYMHycrKilJTU+nFixfk5OREw4cPp3v37tH9+/dpzJgx5OzsXGH+VDWbCwsLqU+fPjRmzBiKjIyk2NhY2rRpE3Xs2JEiIiKI6J/t0MvLiwIDA+nRo0c0evRocnZ2plGjRtGtW7coIiKCvLy8aMSIEaLPp2fPnvT111/To0ePKDAwUPT5yOd78uRJIlLu2MLb25tGjhxJYWFhFB8fTwcPHiSZTEa3bt0qt43eNtyBUYPJA5uI6MSJE2RlZUUxMTFEVDawx44dS//6179E7//ll19IKpUKO1ePHj1owYIFoml8fX1FB8sJCQkUHx9fpo7JkycLw25ubjRnzhxhuOTBcm5uLjk6OtL27duF13///XeSSqX0119/UV5eHtnb29PKlStFy/D39ycnJycqLCxU2BZ+fn40btw40bioqChh/TMzM4X1LF2XPNQ2bdok1CE3YcIEcnBwEAW/p6enUJ98HkeOHBFez83NFX2JKHOwvG3bNuH1wsJCsrOzE4Lxyy+/JHt7e3r16pUwzYkTJ0gqlZZ7sJyQkEAWFha0d+9e0fjPPvuMBg0aJAyHhYWRlZUVXb9+nfz9/cnNzU1Yjnwbmjlzpmgeo0ePpo8//piIiLZv3042NjaUkZEhqr9Xr160aNEiURv99NNPwjTJyckklUrp4MGDRES0ePFi6tatG+Xm5grTREREkFQqpV9++UVoR6lUSrdv3xam+fPPP0kqldKlS5eIiOibb74hqVSqsE0YUzfO5H9wJotxJjOmPpXJXmU7MCwsLCg5OVmYxtPTk1xdXYXhoqIisre3F/Zp+f5w5coVYZpnz56RTCYT9it3d3eaOHGiaNkHDhygjh07UlpamrAudnZ2onwpTT7fkjlHVNxxbG9vT3l5eURENGbMGNGPYUWkUil9+umnonF+fn40ePBgIiL6448/SCqVUlBQkGiaQYMG0WeffUZEREeOHCGpVCp01BD98z0iz9ZXr15RVFSUKFPkeRsWFkZEZT8HInE+79ixg+zt7UXz8PX1pQkTJhBRcd5YWVmJPrcXL16QjY0NHThwQOH6qyKbCwsL6fHjx5SSkiJMn5+fT5aWlrR//34i+mc7XL58uTDN2bNnSSqV0s8//yyM2717N73//vvCsFQqJW9vb1FtCxcupK5du4rmK+/AUObYwtbWlnbu3CmapmSnUW3Al5C8JQYOHIiOHTsqPJ0NAO7duwcnJyfRuM6dOwMoPt00NzcXz58/h1QqFU3Ts2dP0bBEIsHu3bvh6uoqnMoUEhKCtLQ0peqsU6cO+vfvj6CgIGHcuXPnYG5uDmtra0RHRyMrK6tMrV26dEFKSgqeP3+ucL59+vTB5cuXMXPmTAQFBSE9PR3t2rVDixYtAAAGBgYICQnBkCFD4OTkBHt7eyxZsgQAytResg0aNGgAc3Nz6Ovri8aVPj2xU6dOonXs0KEDnj59qlSbAICNjY3wfy0tLRgbGyM9PR0AEBsbi9atW0NPT0+YpvTnUlpoaCiKiooUtmNkZKRwKrSFhQX8/PywfurzbAAADyFJREFUYMECHDx4EKtXrxYtBwDs7OxEwx07dhTWLTQ0tMy1lVpaWrCyssL9+/fLXUdTU1MAENbx3r17cHBwQJ06dYRppFIpjI2NRafSlZ6PiYkJgLKfIWPVjTOZM7kkzmTGNON12ausRo0aoWHDhsJwgwYNYGFhIQxLJBKF2VNy/2zatCnMzMzw6NEjZGZm4uHDhwozoKCgAJGRkcK49u3bl9nvS/r7779BRGWywNbWFllZWXj8+HGl1rXkPgwU54E8C+T3sSldd9euXYUsiI2NRYMGDdCkSRPhdUdHRxgYGAjDenp6iIiIwOjRo4XLO8aPHw9A+bz44IMPkJWVJVxWk5KSgtu3b8PT0xNAcW516NBB9LmZmpqiffv2ZXJLThXZrKWlhbS0NCxatAh9+vSBvb09OnfujMLCwjLrVvKmxg0aNAAAWFpaisaVvNwDUJz5qampyMnJKbM+rzu2AIq/n7ds2YK1a9fixo0byMvLg62tLYyNjRW20dtIp7oLYMqRSCRYsGABvL29cfXqVbRu3Vr0emZmJr777jscPXq0zHuTk5Px8uVLAICRkZHotZIbMxFh7NixePnyJebNmwepVApdXV3Mnz+/UrV+8MEHOH78OJ49e4amTZsiKCgIQ4cOFeoEgBkzZkBbW1t4j/yu0klJSWjatGmZeQ4fPhwmJiY4fPgwZsyYASKCq6srlixZAmNjY+zZswcBAQGYMGEC3NzcUK9ePVy6dAmrV68WzUdbW1t0wCaRSFC3bl3RNBKJpMy136XbzcDAoEwAVaTkwXjpZbx8+bLMDYzkoVceeTv6+PhAIpEI4wsKCpCfn4/U1FThgNXb2xtff/01OnToAHt7+zLzqmjdMjMzFd5cydDQsMyXesl2lNckX8fMzExcvHixzPJzcnKQnJwsDCv6fErOh7GagjOZM7kkzmTGNON12assRRnwuuyRSCQwNDQUTWNgYICcnBxh/1u3bh02bNggvC5/f8n9qvQ8SpPPq/S+Ln/f6+6/VFpFeSefV+lO2vz8fOjq6gJQnIkSiUSUVWfPnsWMGTMwZMgQfP7550Jn6LRp05Sus0WLFujUqRPOnz+PPn36IDg4GHp6eujXr59Qa3h4eJncys3NhZmZmcJ5qiKbnzx5Al9fX1haWmL16tVo1qwZtLS08OGHH5ZZXsmOKfnySrZ/yRrkSretvGNI3slUen0qOrYAgLVr1+LAgQM4deoU9uzZg3r16mHkyJH47LPPas3NvbkD4y3SqVMneHl5Yc2aNdi6davoNSMjI7i5uWHs2LFl3tegQQMhqHJzc0WvpaamCv+PjIxEZGQk1q9fDw8PD2F8RkbGaw/eSurevTuMjY0RFBQEW1tbJCQkCDu5POyWLFkCR0fHMu8t2btbmoeHBzw8PJCZmYmgoCD4+/tj2bJl2LBhAwIDA+Hs7IyZM2cK06vyAKt0L2hWVhaaN2+uknnr6emVuSuz/MdNeeTtuGXLFrRs2bLM6/Xr1xf+v2bNGtjZ2SE6OhpHjhyBj4+PaNrSy87KyhLeb2RkhCdPnpSZf0ZGRpmD7NfV26NHD4U/vPju0+xtxZnMmSzHmcyY5lSUvYo6PEvvU2+KiPDq1SvRD9KsrCwYGBgI+82///1v4YyBkkqeNfA68n25dKesfLgy+7qyyzp69Kios7IkPT090Y1IgeJO7pL1BQYGok2bNli5cqXwI73kWSfK8vDwwNdff43CwkKcO3cOLi4uQseSkZERZDIZvvrqqzLvK91JU3r9qpLNv/zyC3JycrBx40bhOzEtLQ35+fmVXj9FFH2fAcXHCiU7vuTrU9GxBVDcFuPGjcO4ceOQmJiII0eOYPv27WjcuDE++eQTldRc3WpHN8w7ZPbs2Xjy5IlwN145GxsbxMXFoXXr1sK/Fi1aoKCgAMbGxjAxMUGDBg3w999/i95X8rRi+Y4oP0UUAMLDw0WnWMlVdCCqo6MDV1dXXL58GcHBwbCysoK5uTkAoG3btqhXrx6eP38uqrV+/fowMDBQGEBFRUUICgrCs2fPABQfXA0aNAheXl6IiooSai9ZNxHh1KlTr61VWbdv3xb+n5eXh6ioKGGdqqp169Z48OCB6IfM+fPnK3yPtbU1tLS0kJKSImpHfX19GBsbQ0enuG8yODgYFy9exJo1azBz5kysW7dOaEe5kJAQ0fDff/8trJu1tTUePHgg6gUuKChAaGhomVMSK2JjY4OYmBi0atVKVG9+fr7Q810Z/Nc/VlNwJnMmA5zJnMlM08rLXkNDwzJ/uVbl435LZs/z58+RnJwsXNbVtm1bPHv2TLRPmZmZQVtbu1Idg1ZWVtDS0sKdO3dE4+/evQsjI6M3PutEEVtbWwDFf+0vWbeOjg4aNWoEoDgTk5OThadjAcBvv/0m6hiSZ37JMwzKy/yK8sLd3R0vX77E1atXcfPmTVFnkI2NDeLj42FmZiaqtaCgoNwOIlVks6LvYlV+nynK/GbNmin8/n3dsUVaWhp++uknFBYWAij+I8TUqVPRoUMH4fu5NuAOjLdMkyZN4OfnhwMHDojGjxkzBjdu3MBXX32F6OhoREREYOHChfDx8REeleTu7o7Tp0/jzJkzePToEfz9/UWP6TE3N4eRkREOHTqE2NhY/Pbbb5g/fz5cXFwQGxsrXHPXoEED3L9/H2FhYWV6BuU8PDxw69YtBAUFwcvLSxivq6uLkSNHYufOnTh58iTi4uIQEhKCCRMmYOrUqQrnpaWlhV27dmH27Nm4e/cunj17ht9//x0XL14UrvuytbXFlStXEBISgqioKMyYMUO4ljEkJKTKve9HjhzBr7/+iocPH2LlypV49eqVwh72N+Hm5oasrCysWrUKDx8+xPnz54VgLE/jxo3h5eWFgIAABAcHIz4+HlevXsXIkSOxYsUKAMVfRkuXLsW4ceNgbm4Ob29vtGvXDosXLxbN686dO/j+++/x+PFjHDp0SHg0FgAMGTIEBgYGmDVrlvDDad68eUhPT8fw4cOVXkdfX188ffoUixcvRkREBB4+fIj169fjo48+qlSgynvKg4OD8fDhQ6Xfx5i6cCZzJgOcyZzJTNPKy15ra2ukpaXh22+/RVxcHA4fPqyyDgxtbW188803uH37Nh48eIBFixahbt266N+/PwBg7NixOHnyJPbv34/Hjx/jr7/+wowZMzBq1KgyZzC8bt08PT2xefNmXLhwAXFxcfjhhx9w6NAhjBo1SvjRrQqdOnVC586dsXDhQly7dg3x8fE4f/48hg4dih07dgAofuypjo4Oli9fjqioKFy7dg2bN28WXfJoa2uL0NBQXLp0CTExMVi5cqVw9sMff/yBzMxMIS8uX76MiIgIhfU0bdoUDg4O+OKLL2BoaIgePXoIrw0ePBja2tqYNWsWQkNDERsbiz179uBf//oXbty4oXB+qshmeSfPzp07ER8fj8OHD+Py5cto2bIl7t+/X+73rrLi4+Oxbds2xMTE4MyZMzh16pSQ+aW97tiCiLB06VIsW7YMDx48wNOnT/HTTz/h0aNHwvdzbcCXkLyFxo4dix9//FF0Gmn37t2xZcsWbN26FTt37oSuri4cHR1x4MAB4a8p//nPf/Dy5UvMnz8fBgYG+PjjjzFq1CgsXrwYderUgaGhIQICAuDv7w8vLy9YWFhgxYoVyMnJweTJk+Hj44Pr169jzJgxWLx4MT755JNyb6LUpUsXGBkZITY2VnTqMwBMnToVdevWxebNm5GQkAAjIyP0798fs2bNKnedN23ahDVr1mDixInIzMxE48aN0a9fP+H05OnTpyMxMRF+fn6oX78+Ro8ejeHDhyM6OhqLFi167fWGrzNz5kxs3boVoaGhaNiwIVavXo127dpVaZ5yDg4OWLRoEb755hv8/PPPsLe3x+rVq+Hu7l7hTZ5WrlyJDRs2YPny5UhOToapqSm8vLyE6w39/f1hYGCACRMmACj+0bF8+XIMHjwYJ06cEILMz88Pt2/fRkBAAHR0dDBy5EgMGTIEQPEpj/v378fatWvh4+MDIoKNjQ327t1bqfVv37499u7diw0bNmDYsGHCfHbv3o0OHTooPR9XV1ccPXoUM2bMEG5SxFh140zmTAY4kzmTmaYpyl5PT0/cuXMHW7duxcaNG+Hi4oKZM2fi3//+d5WXV69ePcyYMQPLli1DdHQ0WrRogU2bNgl//R8yZAiICHv37sW6deugr68PZ2dn7Nu3r9zLM8qzcuVKrF+/HkuWLEFqaiqaNWuGyZMnY9y4cVVej9K2bduGgIAAzJo1C+np6WjSpAl8fX2FrGrevDnWr1+P9evX4+OPP4alpSWWLl2KyZMnC5k4atQoREVFYdasWdDT08PgwYMxf/58pKenY8uWLTAwMICPjw+cnJzg7+8PqVSK48ePK6zHw8MDK1asgLe3t3AfDqA4/w4ePIiAgAD4+voiPz8fUqkUX375paijo7SqZvOgQYMwdepUHDp0CLt374azszMCAgJw8uRJbNy4EcuXL8fnn3/+xu0/dOhQJCcnY9iwYcjPz4e7u3u526syxxa7du3Cxo0b4ePjg4KCArRq1Qpz5syBu7v7G9dY00iIz/t7Z+Tl5SEzM1N0euj69etx6NChMqepsWLHjx/HvHnzcPnyZYU3slMFIsKLFy9gamoq3FwnOjoaHh4e2LRpE9zc3NSy3Pj4ePTr1w8BAQHl9vQyxtSHM7nyOJMZY0zzUlNTYWhoKHTEZGdnw9HREbNnz8aYMWOqubq3l0wmw7Rp0zBp0qTqLuWtwpeQvEM2btwIV1dXBAcH48mTJwgKCsKRI0cwePDg6i7tnRYZGYmePXvC398fjx8/xv3797FixQo0adLktY/uY4y9vTiTaybOZMYY+0dKSgr69u2LefPmITo6GlFRUVi4cCH09PQUPomDMXXjS0jeIdOnTwdQfCrVixcv0KRJE3h7e2PKlCnVXNm7TSaTYcuWLdi2bRt++OEH6Ovrw8bGBnv27BE9Y5sxVrtwJtdMnMmMMfYPU1NT4bKEoUOHQltbGzKZDLt3767wSVWMqQtfQsIYY4wxxhhjjLEajy8hYYwxxhhjjDHGWI3HHRiMMcYYY4wxxhir8bgDgzHGGGOMMcYYYzUed2AwxhhjjDHGGGOsxuMODMYYY4wxxhhjjNV4/wNen8EiG0q3+wAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{"tags":[]},"output_type":"display_data"}],"source":["fig = plt.figure(figsize=(15,5))\n","\n","ax = fig.add_subplot(131)\n","negative, ns_exp, recall = aggregate_z(\"negative\", \"ns_exponent\")\n","cm = sns.scatterplot(x=ns_exp, y=negative, hue=recall, palette=color_palette, legend=None)\n","ax.set_xlabel(\"Negative sampling exponent\", fontsize=16)\n","ax.set_ylabel(\"Number of negative samples\", fontsize=16)\n","plt.xticks(fontsize=12)\n","plt.yticks(fontsize=12)\n","ax.plot(0.75, 5, \n"," marker='*', \n"," color=cldr_colors[1],\n"," markersize=10)\n","ax.plot(best_config['ns_exponent'], \n"," best_config['negative'], \n"," marker=\"o\", \n"," fillstyle='none', \n"," color=cldr_colors[0],\n"," markersize=15)\n","ax = fig.add_subplot(132)\n","\n","window, ns_exp, recall = aggregate_z(\"window\", \"ns_exponent\")\n","cm = sns.scatterplot(x=ns_exp, y=window, hue=recall, palette=color_palette, legend=None)\n","ax.set_xlabel(\"Negative sampling exponent\", fontsize=16)\n","ax.set_ylabel(\"Context window size\", fontsize=16)\n","plt.xticks(fontsize=12)\n","plt.yticks(fontsize=12)\n","ax.plot(0.75, 5, \n"," marker='*', \n"," color=cldr_colors[1],\n"," markersize=10)\n","ax.plot(best_config['ns_exponent'], \n"," best_config['window'], \n"," marker=\"o\", \n"," fillstyle='none', \n"," color=cldr_colors[0],\n"," markersize=15)\n","\n","ax = fig.add_subplot(133)\n","window, negative, recall = aggregate_z(\"window\", \"negative\")\n","cm = sns.scatterplot(x=window, y=negative, hue=recall, palette=color_palette, legend=None)\n","ax.set_xlabel(\"Number of negative examples\", fontsize=16)\n","ax.set_ylabel(\"Context window size\", fontsize=16)\n","plt.xticks(fontsize=12)\n","plt.yticks(fontsize=12)\n","ax.plot(5, 5, \n"," marker='*',\n"," color=cldr_colors[1],\n"," markersize=10)\n","ax.plot(best_config['window'], \n"," best_config['negative'], \n"," marker=\"o\", \n"," fillstyle='none', \n"," color=cldr_colors[0],\n"," markersize=15);\n","\n","plt.tight_layout()\n","plt.savefig(\"hpsweep_results.png\", transparent=True, dpi=150)"]},{"cell_type":"markdown","metadata":{"id":"-5KwIKIl36gA"},"source":["And there we have it! Each panel shows the Recall@10 scores (where yellow is a high score and purple is a low score) associated with a unique configuration of hyperparameters. The best hyperparameter values for the Online Retail Data Set are denoted by the light blue circle. Word2vec’s default values are shown by the orange star. In all cases, the orange star is nowhere near the light blue circle, indicating that the default values are not optimal for this dataset."]},{"cell_type":"markdown","source":["## Online Evaluation\n","\n","Offline evaluations rarely inform us about the quality of recommendations as perceived by the users. In one study of e-commerce session-based recommenders, it was observed that offline evaluation metrics often fall short because they tend to reward an algorithm when they predict the exact item that the user clicked or purchased. In real life, though, there are identical products that could potentially make equally good recommendations. To overcome these limitations, we suggest incorporating human feedback on the recommendations from offline evaluations before conducting A/B tests.\n","\n"],"metadata":{"id":"wYL9GUoPwSh0"}},{"cell_type":"markdown","source":["## Summary\n","\n","- We experimented with an NLP-based algorithm—word2vec—which is known for learning low-dimensional word representations that are contextual in nature.\n","- We applied it to an e-commerce dataset containing historical purchase transactions, to learn the structure induced by both the user’s behavior and the product’s nature to recommend the next item to be purchased.\n","- While doing so, we learned that hyperparameter choices are data- and task-dependent, and especially, that they differ from linguistic tasks; what works for language models does not necessarily work for recommendation tasks.\n","- That said, our experiments indicate that in addition to specific parameters (like negative sampling exponent, the number of negative samples, and context window size), the number of training epochs greatly influences model performance. We recommend that word2vec be trained for as many epochs as computational resources allow, or until performance on a downstream recommendation metric has plateaued.\n","- We also realized during our experimentation that performing hyperparameter search over just a handful of parameters can be time consuming and computationally expensive; hence, it could be a bottleneck to developing a real-life recommendation solution (with word2vec). While scaling hyperparameter optimization is possible through tools like Ray Tune, we envision additional research into algorithmic approaches to solve this problem would pave the way in developing more scalable (and less complex) solutions."],"metadata":{"id":"p4qrxHCzwJxR"}},{"cell_type":"markdown","source":["---"],"metadata":{"id":"chZ1lJ_guGQQ"}},{"cell_type":"code","source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p gensim"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"l9S61GlZuGQR","executionInfo":{"status":"ok","timestamp":1639397783258,"user_tz":-330,"elapsed":3838,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"77933f06-a621-4e44-b690-7f1da6a4dbec"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2021-12-13 12:16:31\n","\n","gensim: 3.6.0\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.104+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","ray : 1.9.0\n","argparse : 1.1\n","pandas : 1.1.5\n","matplotlib: 3.2.2\n","seaborn : 0.11.2\n","IPython : 5.5.0\n","numpy : 1.19.5\n","\n"]}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"z-Ixq7XguGQR"}},{"cell_type":"markdown","source":["**END**"],"metadata":{"id":"Nn1dX3S7uGQS"}}],"metadata":{"colab":{"collapsed_sections":[],"name":"2022-01-12-sess-word2vec.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P205596%20%7C%20Training%20Session-based%20Product%20Recommender%20using%20Word2vec%20on%20Retail%20data.ipynb","timestamp":1644603087371},{"file_id":"https://github.com/recohut/reco-nb/blob/dev/_notebooks/2021-06-11-recostep-session-based-recommender-using-word2vec.ipynb","timestamp":1639397411886}],"authorship_tag":"ABX9TyO0pLP+n+7OIG+az0XeHhNU"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":0} \ No newline at end of file diff --git a/_notebooks/2022-01-12-slist-yoochoose.ipynb b/_notebooks/2022-01-12-slist-yoochoose.ipynb new file mode 100644 index 0000000..faf1361 --- /dev/null +++ b/_notebooks/2022-01-12-slist-yoochoose.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-12-slist-yoochoose.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/P293191%20%7C%20SLIST%20on%20Yoochoose%20Preprocessed%20Sample%20Dataset.ipynb","timestamp":1644607241308}],"collapsed_sections":[],"mount_file_id":"1jAKIE2Lz_IL3da8Weo1SbuO7aE9pZHjH","authorship_tag":"ABX9TyO/bUC4ZiV1mWxoO6hl51K4"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# SLIST on Yoochoose Preprocessed Sample Dataset"],"metadata":{"id":"QfsL1SQZVzGt"}},{"cell_type":"markdown","source":["# SLIST on Yoochoose Preprocessed Sample Dataset"],"metadata":{"id":"TthNcibSQiab"}},{"cell_type":"markdown","source":["## Executive summary\n","\n","| | |\n","| --- | --- |\n","| Prblm Stmnt | The goal of session-based recommendation is to predict the next item(s) a user would likely choose to consume, given a sequence of previously consumed items in a session. Formally, we build a session-based model M(𝑠) that takes a session ⁍ for ⁍ as input and returns a list of top-𝑁 candidate items to be consumed as the next one ⁍. |\n","| Solution | Firstly, we devise two linear models focusing on different properties of sessions: (i) Session-aware Linear Item Similarity (SLIS) model aims at better handling session consistency, and (ii) Session-aware Linear Item Transition (SLIT) model focuses more on sequential dependency. With both SLIS and SLIT, we relax the constraint to incorporate repeated items and introduce a weighting scheme to take the timeliness of sessions into account. Combining these two types of models, we then suggest a unified model, namely Session-aware Item Similarity/Transition (SLIST) model, which is a generalized solution to holistically cover various properties of sessions. |\n","| Dataset | Yoochoose |\n","| Preprocessing | We discard the sessions having only one interaction and items appearing less than five times following the convention. We hold-out the sessions from the last 𝑁-days for test purposes and used the last 𝑁 days in the training set for the validation set. To evaluate session-based recommender models, we adopt the iterative revealing scheme, which iteratively exposes the item of a session to the model. Each item in the session is sequentially appended to the input of the model. Therefore, this scheme is useful for reflecting the sequential user behavior throughout a session |\n","| Metrics | HR, MRR, Coverage, Popularity |\n","| Models | SLIST\n","| Cluster | Python 3.x |\n","| Tags | LinearRecommender, SessionBasedRecommender |"],"metadata":{"id":"YxLc0jRFV3S-"}},{"cell_type":"markdown","source":["## Process flow\n","\n","![](https://github.com/RecoHut-Stanzas/S181315/raw/main/images/process_flow_prototype_1.svg)"],"metadata":{"id":"UYrwO2hmWLjr"}},{"cell_type":"markdown","source":["## Setup"],"metadata":{"id":"B_x3nAE9QZ_7"}},{"cell_type":"markdown","source":["### Imports"],"metadata":{"id":"I1mMP5dnQbML"}},{"cell_type":"code","source":["import os.path\n","import numpy as np\n","import pandas as pd\n","from _datetime import datetime, timezone, timedelta\n","\n","from tqdm import tqdm\n","import collections as col\n","import scipy\n","import os\n","import pickle\n","\n","from scipy import sparse\n","from scipy.sparse.linalg import inv\n","from scipy.sparse import csr_matrix, csc_matrix, vstack\n","from sklearn.preprocessing import normalize"],"metadata":{"id":"BZTJO2V8E6IJ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Dataset"],"metadata":{"id":"PP2SgjqSQhxB"}},{"cell_type":"markdown","source":["### Load data\n","\n","Preprocessed Yoochoose clicks 100k"],"metadata":{"id":"5SpMxtzFQcXy"}},{"cell_type":"code","source":["!mkdir -p prepared\n","!wget -O prepared/events_test.txt -q --show-progress https://github.com/RecoHut-Stanzas/S181315/raw/main/data/rsc15/prepared/yoochoose-clicks-100k_test.txt\n","!wget -O prepared/events_train_full.txt -q --show-progress https://github.com/RecoHut-Stanzas/S181315/raw/main/data/rsc15/prepared/yoochoose-clicks-100k_train_full.txt\n","!wget -O prepared/events_train_tr.txt -q --show-progress https://github.com/RecoHut-Stanzas/S181315/raw/main/data/rsc15/prepared/yoochoose-clicks-100k_train_tr.txt\n","!wget -O prepared/events_train_valid.txt -q --show-progress https://github.com/RecoHut-Stanzas/S181315/raw/main/data/rsc15/prepared/yoochoose-clicks-100k_train_valid.txt"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0iiHncDp6gvs","executionInfo":{"status":"ok","timestamp":1639118974794,"user_tz":-330,"elapsed":2453,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"a6a1504e-a5bd-4c45-e1e2-7671e4535311"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["prepared/events_tes 100%[===================>] 404.64K --.-KB/s in 0.007s \n","prepared/events_tra 100%[===================>] 2.05M --.-KB/s in 0.02s \n","prepared/events_tra 100%[===================>] 1.55M --.-KB/s in 0.02s \n","prepared/events_tra 100%[===================>] 493.18K --.-KB/s in 0.008s \n"]}]},{"cell_type":"code","source":["def load_data_session( path, file, sessions_train=None, sessions_test=None, slice_num=None, train_eval=False ):\n"," '''\n"," Loads a tuple of training and test set with the given parameters. \n"," Parameters\n"," --------\n"," path : string\n"," Base path to look in for the prepared data files\n"," file : string\n"," Prefix of the dataset you want to use.\n"," \"yoochoose-clicks-full\" loads yoochoose-clicks-full_train_full.txt and yoochoose-clicks-full_test.txt\n"," rows_train : int or None\n"," Number of rows to load from the training set file. \n"," This option will automatically filter the test set to only retain items included in the training set. \n"," rows_test : int or None\n"," Number of rows to load from the test set file. \n"," slice_num : \n"," Adds a slice index to the constructed file_path\n"," yoochoose-clicks-full_train_full.0.txt\n"," density : float\n"," Percentage of the sessions to randomly retain from the original data (0-1). \n"," The result is cached for the execution of multiple experiments. \n"," Returns\n"," --------\n"," out : tuple of pandas.DataFrame\n"," (train, test)\n"," \n"," '''\n"," \n"," print('START load data') \n"," import time\n"," st = time.time()\n"," sc = time.perf_counter()\n"," \n"," split = ''\n"," if( slice_num != None and isinstance(slice_num, int ) ):\n"," split = '.'+str(slice_num)\n"," \n"," train_appendix = '_train_full'\n"," test_appendix = '_test'\n"," if train_eval:\n"," train_appendix = '_train_tr'\n"," test_appendix = '_train_valid'\n"," \n"," train = pd.read_csv(path + file + train_appendix +split+'.txt', sep='\\t' )\n"," test = pd.read_csv(path + file + test_appendix +split+'.txt', sep='\\t' )\n"," \n"," if( sessions_train != None ):\n"," keep = train.sort_values('Time', ascending=False).SessionId.unique()[:(sessions_train-1)]\n"," train = train[ np.in1d( train.SessionId, keep ) ]\n"," test = test[np.in1d(test.ItemId, train.ItemId)]\n"," \n"," if( sessions_test != None ):\n"," keep = test.SessionId.unique()[:(sessions_test)]\n"," test = test[ np.in1d( test.SessionId, keep ) ]\n"," \n"," session_lengths = test.groupby('SessionId').size()\n"," test = test[np.in1d(test.SessionId, session_lengths[ session_lengths>1 ].index)]\n"," \n"," #output\n"," data_start = datetime.fromtimestamp( train.Time.min(), timezone.utc )\n"," data_end = datetime.fromtimestamp( train.Time.max(), timezone.utc )\n"," \n"," print('Loaded train set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}\\n'.\n"," format( len(train), train.SessionId.nunique(), train.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat() ) )\n"," \n"," data_start = datetime.fromtimestamp( test.Time.min(), timezone.utc )\n"," data_end = datetime.fromtimestamp( test.Time.max(), timezone.utc )\n"," \n"," print('Loaded test set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}\\n'.\n"," format( len(test), test.SessionId.nunique(), test.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat() ) )\n"," \n"," check_data(train, test)\n"," \n"," print( 'END load data ', (time.perf_counter()-sc), 'c / ', (time.time()-st), 's' ) \n"," \n"," return (train, test)"],"metadata":{"id":"tkzyj627E9F2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def check_data( train, test ):\n"," \n"," if 'ItemId' in train.columns and 'SessionId' in train.columns:\n"," \n"," new_in_test = set( test.ItemId.unique() ) - set( train.ItemId.unique() )\n"," if len( new_in_test ) > 0:\n"," print( 'WAAAAAARRRNIIIIING: new items in test set' )\n"," \n"," session_min_train = train.groupby( 'SessionId' ).size().min()\n"," if session_min_train == 0:\n"," print( 'WAAAAAARRRNIIIIING: session length 1 in train set' )\n"," \n"," session_min_test = test.groupby( 'SessionId' ).size().min()\n"," if session_min_test == 0:\n"," print( 'WAAAAAARRRNIIIIING: session length 1 in train set' )\n"," \n"," else: \n"," print( 'data check not possible due to individual column names' )"],"metadata":{"id":"SioAVZHXFCP1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def evaluate_sessions(pr, metrics, test_data, train_data, items=None, cut_off=20, session_key='SessionId', item_key='ItemId', time_key='Time'):\n"," '''\n"," Evaluates the baselines wrt. recommendation accuracy measured by recall@N and MRR@N. Has no batch evaluation capabilities. Breaks up ties.\n"," Parameters\n"," --------\n"," pr : baseline predictor\n"," A trained instance of a baseline predictor.\n"," metrics : list\n"," A list of metric classes providing the proper methods\n"," test_data : pandas.DataFrame\n"," Test data. It contains the transactions of the test set.It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).\n"," It must have a header. Column names are arbitrary, but must correspond to the keys you use in this function.\n"," train_data : pandas.DataFrame\n"," Training data. Only required for selecting the set of item IDs of the training set.\n"," items : 1D list or None\n"," The list of item ID that you want to compare the score of the relevant item to. If None, all items of the training set are used. Default value is None.\n"," cut-off : int\n"," Cut-off value (i.e. the length of the recommendation list; N for recall@N and MRR@N). Defauld value is 20.\n"," session_key : string\n"," Header of the session ID column in the input file (default: 'SessionId')\n"," item_key : string\n"," Header of the item ID column in the input file (default: 'ItemId')\n"," time_key : string\n"," Header of the timestamp column in the input file (default: 'Time')\n"," Returns\n"," --------\n"," out : list of tuples\n"," (metric_name, value)\n"," '''\n","\n"," actions = len(test_data)\n"," sessions = len(test_data[session_key].unique())\n"," count = 0\n"," print('START evaluation of ', actions, ' actions in ', sessions, ' sessions')\n","\n"," import time\n"," sc = time.perf_counter()\n"," st = time.time()\n","\n"," time_sum = 0\n"," time_sum_clock = 0\n"," time_count = 0\n","\n"," for m in metrics:\n"," m.reset()\n","\n"," test_data.sort_values([session_key, time_key], inplace=True)\n"," items_to_predict = train_data[item_key].unique()\n"," prev_iid, prev_sid = -1, -1\n"," pos = 0\n","\n"," for i in tqdm(range(len(test_data))):\n","\n"," # if count % 1000 == 0:\n"," # print(f'eval process: {count} of {actions} actions: {(count / actions * 100.0):.2f} % in {(time.time()-st):.2f} s')\n","\n"," sid = test_data[session_key].values[i]\n"," iid = test_data[item_key].values[i]\n"," ts = test_data[time_key].values[i]\n"," if prev_sid != sid:\n"," prev_sid = sid\n"," pos = 0\n"," else:\n"," if items is not None:\n"," if np.in1d(iid, items):\n"," items_to_predict = items\n"," else:\n"," items_to_predict = np.hstack(([iid], items))\n","\n"," crs = time.perf_counter()\n"," trs = time.time()\n","\n"," for m in metrics:\n"," if hasattr(m, 'start_predict'):\n"," m.start_predict(pr)\n","\n"," preds = pr.predict_next(sid, prev_iid, items_to_predict, timestamp=ts)\n","\n"," for m in metrics:\n"," if hasattr(m, 'stop_predict'):\n"," m.stop_predict(pr)\n","\n"," preds[np.isnan(preds)] = 0\n","# preds += 1e-8 * np.random.rand(len(preds)) #Breaking up ties\n"," preds.sort_values(ascending=False, inplace=True)\n","\n"," time_sum_clock += time.perf_counter()-crs\n"," time_sum += time.time()-trs\n"," time_count += 1\n","\n"," for m in metrics:\n"," if hasattr(m, 'add'):\n"," m.add(preds, iid, for_item=prev_iid, session=sid, position=pos)\n","\n"," pos += 1\n","\n"," prev_iid = iid\n","\n"," count += 1\n","\n"," print('\\nEND evaluation in ', (time.perf_counter()-sc), 'c / ', (time.time()-st), 's')\n"," print(' avg rt ', (time_sum/time_count), 's / ', (time_sum_clock/time_count), 'c')\n"," print(' time count ', (time_count), 'count/', (time_sum), ' sum')\n","\n"," res = []\n"," for m in metrics:\n"," if type(m).__name__ == 'Time_usage_testing':\n"," res.append(m.result_second(time_sum_clock/time_count))\n"," res.append(m.result_cpu(time_sum_clock / time_count))\n"," else:\n"," res.append(m.result())\n","\n"," return res"],"metadata":{"id":"D-KD8eRwFQFj"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Metrics"],"metadata":{"id":"QalAVbeRRMCr"}},{"cell_type":"code","source":["class MRR: \n"," '''\n"," MRR( length=20 )\n"," Used to iteratively calculate the average mean reciprocal rank for a result list with the defined length. \n"," Parameters\n"," -----------\n"," length : int\n"," MRR@length\n"," '''\n"," def __init__(self, length=20):\n"," self.length = length;\n"," \n"," def init(self, train):\n"," '''\n"," Do initialization work here.\n"," \n"," Parameters\n"," --------\n"," train: pandas.DataFrame\n"," Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).\n"," It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties).\n"," '''\n"," return\n"," \n"," def reset(self):\n"," '''\n"," Reset for usage in multiple evaluations\n"," '''\n"," self.test=0;\n"," self.pos=0\n"," \n"," self.test_popbin = {}\n"," self.pos_popbin = {}\n"," \n"," self.test_position = {}\n"," self.pos_position = {}\n"," \n"," def skip(self, for_item = 0, session = -1 ):\n"," pass\n"," \n"," def add(self, result, next_item, for_item=0, session=0, pop_bin=None, position=None ):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," Result must be sorted correctly.\n"," \n"," Parameters\n"," --------\n"," result: pandas.Series\n"," Series of scores with the item id as the index\n"," '''\n"," res = result[:self.length]\n"," \n"," self.test += 1\n"," \n"," if pop_bin is not None:\n"," if pop_bin not in self.test_popbin:\n"," self.test_popbin[pop_bin] = 0\n"," self.pos_popbin[pop_bin] = 0\n"," self.test_popbin[pop_bin] += 1\n"," \n"," if position is not None:\n"," if position not in self.test_position:\n"," self.test_position[position] = 0\n"," self.pos_position[position] = 0\n"," self.test_position[position] += 1\n"," \n"," if next_item in res.index:\n"," rank = res.index.get_loc( next_item )+1\n"," self.pos += ( 1.0/rank )\n"," \n"," if pop_bin is not None:\n"," self.pos_popbin[pop_bin] += ( 1.0/rank )\n"," \n"," if position is not None:\n"," self.pos_position[position] += ( 1.0/rank )\n"," \n"," \n"," \n"," def add_batch(self, result, next_item):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," \n"," Parameters\n"," --------\n"," result: pandas.DataFrame\n"," Prediction scores for selected items for every event of the batch.\n"," Columns: events of the batch; rows: items. Rows are indexed by the item IDs.\n"," next_item: Array of correct next items\n"," '''\n"," i=0\n"," for part, series in result.iteritems(): \n"," result.sort_values( part, ascending=False, inplace=True )\n"," self.add( series, next_item[i] )\n"," i += 1\n"," \n"," def result(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," return (\"MRR@\" + str(self.length) + \": \"), (self.pos/self.test), self.result_pop_bin(), self.result_position()\n"," \n"," def result_pop_bin(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," csv = ''\n"," csv += 'Bin: ;'\n"," for key in self.test_popbin:\n"," csv += str(key) + ';'\n"," csv += '\\nPrecision@' + str(self.length) + ': ;'\n"," for key in self.test_popbin:\n"," csv += str( self.pos_popbin[key] / self.test_popbin[key] ) + ';'\n"," \n"," return csv\n"," \n"," def result_position(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," csv = ''\n"," csv += 'Pos: ;'\n"," for key in self.test_position:\n"," csv += str(key) + ';'\n"," csv += '\\nPrecision@' + str(self.length) + ': ;'\n"," for key in self.test_position:\n"," csv += str( self.pos_position[key] / self.test_position[key] ) + ';'\n"," \n"," return csv"],"metadata":{"id":"fUVIGWZz2xPu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class HitRate: \n"," '''\n"," MRR( length=20 )\n"," Used to iteratively calculate the average hit rate for a result list with the defined length. \n"," Parameters\n"," -----------\n"," length : int\n"," HitRate@length\n"," '''\n"," \n"," def __init__(self, length=20):\n"," self.length = length;\n"," \n"," def init(self, train):\n"," '''\n"," Do initialization work here.\n"," \n"," Parameters\n"," --------\n"," train: pandas.DataFrame\n"," Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).\n"," It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties).\n"," '''\n"," return\n"," \n"," def reset(self):\n"," '''\n"," Reset for usage in multiple evaluations\n"," '''\n"," self.test=0;\n"," self.hit=0\n"," \n"," self.test_popbin = {}\n"," self.hit_popbin = {}\n"," \n"," self.test_position = {}\n"," self.hit_position = {}\n"," \n"," def add(self, result, next_item, for_item=0, session=0, pop_bin=None, position=None):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," Result must be sorted correctly.\n"," \n"," Parameters\n"," --------\n"," result: pandas.Series\n"," Series of scores with the item id as the index\n"," '''\n"," \n"," self.test += 1\n"," \n"," if pop_bin is not None:\n"," if pop_bin not in self.test_popbin:\n"," self.test_popbin[pop_bin] = 0\n"," self.hit_popbin[pop_bin] = 0\n"," self.test_popbin[pop_bin] += 1\n"," \n"," if position is not None:\n"," if position not in self.test_position:\n"," self.test_position[position] = 0\n"," self.hit_position[position] = 0\n"," self.test_position[position] += 1\n"," \n"," if next_item in result[:self.length].index:\n"," self.hit += 1\n"," \n"," if pop_bin is not None:\n"," self.hit_popbin[pop_bin] += 1\n"," \n"," if position is not None:\n"," self.hit_position[position] += 1\n"," \n"," \n"," \n"," def add_batch(self, result, next_item):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," \n"," Parameters\n"," --------\n"," result: pandas.DataFrame\n"," Prediction scores for selected items for every event of the batch.\n"," Columns: events of the batch; rows: items. Rows are indexed by the item IDs.\n"," next_item: Array of correct next items\n"," '''\n"," i=0\n"," for part, series in result.iteritems(): \n"," result.sort_values( part, ascending=False, inplace=True )\n"," self.add( series, next_item[i] )\n"," i += 1\n"," \n"," def result(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," return (\"HitRate@\" + str(self.length) + \": \"), (self.hit/self.test), self.result_pop_bin(), self.result_position()\n","\n"," \n"," def result_pop_bin(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," csv = ''\n"," csv += 'Bin: ;'\n"," for key in self.test_popbin:\n"," csv += str(key) + ';'\n"," csv += '\\nHitRate@' + str(self.length) + ': ;'\n"," for key in self.test_popbin:\n"," csv += str( self.hit_popbin[key] / self.test_popbin[key] ) + ';'\n"," \n"," return csv\n"," \n"," def result_position(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," csv = ''\n"," csv += 'Pos: ;'\n"," for key in self.test_position:\n"," csv += str(key) + ';'\n"," csv += '\\nHitRate@' + str(self.length) + ': ;'\n"," for key in self.test_position:\n"," csv += str( self.hit_position[key] / self.test_position[key] ) + ';'\n"," \n"," return csv"],"metadata":{"id":"NL1uEgE6FUg1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class Coverage:\n"," '''\n"," Coverage( length=20 )\n"," Used to iteratively calculate the coverage of an algorithm regarding the item space. \n"," Parameters\n"," -----------\n"," length : int\n"," Coverage@length\n"," '''\n"," \n"," item_key = 'ItemId'\n"," \n"," def __init__(self, length=20):\n"," self.num_items = 0\n"," self.length = length\n"," self.time = 0;\n"," \n"," def init(self, train):\n"," '''\n"," Do initialization work here.\n"," \n"," Parameters\n"," --------\n"," train: pandas.DataFrame\n"," Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).\n"," It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties).\n"," ''' \n"," self.coverage_set = set()\n"," self.items = set(train[self.item_key].unique()) # keep track of full item list\n"," self.num_items = len( train[self.item_key].unique() )\n"," \n"," def reset(self):\n"," '''\n"," Reset for usage in multiple evaluations\n"," '''\n"," self.coverage_set = set()\n"," return\n"," \n"," def skip(self, for_item = 0, session = -1 ):\n"," pass\n"," \n"," def add(self, result, next_item, for_item=0, session=0, pop_bin=None, position=None):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," Result must be sorted correctly.\n"," \n"," Parameters\n"," --------\n"," result: pandas.Series\n"," Series of scores with the item id as the index\n"," '''\n"," recs = result[:self.length]\n"," items = recs.index.unique()\n"," self.coverage_set.update( items )\n"," self.items.update( items ) # update items\n"," self.num_items = len( self.items )\n"," \n"," def add_multiple(self, result, next_items, for_item=0, session=0, position=None): \n"," self.add(result, next_items[0], for_item, session)\n"," \n"," def add_batch(self, result, next_item):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," \n"," Parameters\n"," --------\n"," result: pandas.DataFrame\n"," Prediction scores for selected items for every event of the batch.\n"," Columns: events of the batch; rows: items. Rows are indexed by the item IDs.\n"," next_item: Array of correct next items\n"," '''\n"," i=0\n"," for part, series in result.iteritems(): \n"," result.sort_values( part, ascending=False, inplace=True )\n"," self.add( series, next_item[i] )\n"," i += 1\n"," \n"," \n"," def result(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," return (\"Coverage@\" + str(self.length) + \": \"), ( len(self.coverage_set) / self.num_items )"],"metadata":{"id":"QDlDFjyE28eB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class Popularity:\n"," '''\n"," Popularity( length=20 )\n"," Used to iteratively calculate the average overall popularity of an algorithm's recommendations. \n"," Parameters\n"," -----------\n"," length : int\n"," Coverage@length\n"," '''\n"," \n"," session_key = 'SessionId'\n"," item_key = 'ItemId'\n"," \n"," def __init__(self, length=20):\n"," self.length = length;\n"," self.sum = 0\n"," self.tests = 0\n"," \n"," def init(self, train):\n"," '''\n"," Do initialization work here.\n"," \n"," Parameters\n"," --------\n"," train: pandas.DataFrame\n"," Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).\n"," It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties).\n"," '''\n"," self.train_actions = len( train.index )\n"," #group the data by the itemIds\n"," grp = train.groupby(self.item_key)\n"," #count the occurence of every itemid in the trainingdataset\n"," self.pop_scores = grp.size()\n"," #sort it according to the score\n"," self.pop_scores.sort_values(ascending=False, inplace=True)\n"," #normalize\n"," self.pop_scores = self.pop_scores / self.pop_scores[:1].values[0]\n"," \n"," def reset(self):\n"," '''\n"," Reset for usage in multiple evaluations\n"," '''\n"," self.tests=0;\n"," self.sum=0\n"," \n"," def skip(self, for_item = 0, session = -1 ):\n"," pass \n"," \n"," def add(self, result, next_item, for_item=0, session=0, pop_bin=None, position=None):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," Result must be sorted correctly.\n"," \n"," Parameters\n"," --------\n"," result: pandas.Series\n"," Series of scores with the item id as the index\n"," '''\n"," #only keep the k- first predictions\n"," recs = result[:self.length]\n"," #take the unique values out of those top scorers\n"," items = recs.index.unique()\n"," \n"," self.sum += ( self.pop_scores[ items ].sum() / len( items ) )\n"," self.tests += 1\n"," \n"," def add_multiple(self, result, next_items, for_item=0, session=0, position=None): \n"," self.add(result, next_items[0], for_item, session)\n"," \n"," def add_batch(self, result, next_item):\n"," '''\n"," Update the metric with a result set and the correct next item.\n"," \n"," Parameters\n"," --------\n"," result: pandas.DataFrame\n"," Prediction scores for selected items for every event of the batch.\n"," Columns: events of the batch; rows: items. Rows are indexed by the item IDs.\n"," next_item: Array of correct next items\n"," '''\n"," i=0\n"," for part, series in result.iteritems(): \n"," result.sort_values( part, ascending=False, inplace=True )\n"," self.add( series, next_item[i] )\n"," i += 1\n"," \n"," def result(self):\n"," '''\n"," Return a tuple of a description string and the current averaged value\n"," '''\n"," return (\"Popularity@\" + str( self.length ) + \": \"), ( self.sum / self.tests )"],"metadata":{"id":"_cvJCNlb3BUx"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## SLIST model"],"metadata":{"id":"F5cC3PEPRPpE"}},{"cell_type":"code","source":["class SLIST:\n"," '''\n"," SLIST(reg=10)\n"," Parameters\n"," --------\n"," Will be added\n"," --------\n"," '''\n","\n"," # Must need\n"," def __init__(self, reg=10, alpha=0.5, session_weight=-1, train_weight=-1, predict_weight=-1,\n"," direction='part', normalize='l1', epsilon=10.0, session_key='SessionId', item_key='ItemId',\n"," verbose=False):\n"," self.reg = reg\n"," self.normalize = normalize\n"," self.epsilon = epsilon\n"," self.alpha = alpha\n"," self.direction = direction \n"," self.train_weight = float(train_weight)\n"," self.predict_weight = float(predict_weight)\n"," self.session_weight = session_weight*24*3600\n","\n"," self.session_key = session_key\n"," self.item_key = item_key\n","\n"," # updated while recommending\n"," self.session = -1\n"," self.session_items = []\n"," \n"," self.verbose = verbose\n","\n"," # Must need\n"," def fit(self, data, test=None):\n"," '''\n"," Trains the predictor.\n"," Parameters\n"," --------\n"," data: pandas.DataFrame\n"," Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).\n"," It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties).\n"," '''\n"," # make new session ids(1 ~ #sessions)\n"," sessionids = data[self.session_key].unique()\n"," self.n_sessions = len(sessionids)\n"," self.sessionidmap = pd.Series(data=np.arange(self.n_sessions), index=sessionids)\n"," data = pd.merge(data, pd.DataFrame({self.session_key: sessionids, 'SessionIdx': self.sessionidmap[sessionids].values}), on=self.session_key, how='inner')\n","\n"," # make new item ids(1 ~ #items)\n"," itemids = data[self.item_key].unique()\n"," self.n_items = len(itemids)\n"," self.itemidmap = pd.Series(data=np.arange(self.n_items), index=itemids)\n"," data = pd.merge(data, pd.DataFrame({self.item_key: itemids, 'ItemIdx': self.itemidmap[itemids].values}), on=self.item_key, how='inner')\n","\n"," # ||X - XB||\n"," input1, target1, row_weight1 = self.make_train_matrix(data, weight_by='SLIS')\n"," # ||Y - ZB||\n"," input2, target2, row_weight2 = self.make_train_matrix(data, weight_by='SLIT')\n"," # alpha * ||X - XB|| + (1-alpha) * ||Y - ZB||\n"," input1.data = np.sqrt(self.alpha) * input1.data\n"," target1.data = np.sqrt(self.alpha) * target1.data\n"," input2.data = np.sqrt(1-self.alpha) * input2.data\n"," target2.data = np.sqrt(1-self.alpha) * target2.data\n","\n"," input_matrix = vstack([input1, input2])\n"," target_matrix = vstack([target1, target2])\n"," w2 = row_weight1 + row_weight2 # list\n","\n"," # P = (X^T * X + λI)^−1 = (G + λI)^−1\n"," # (A+B)^-1 = A^-1 - A^-1 * B * (A+B)^-1\n"," # P = G\n"," W2 = sparse.diags(w2, dtype=np.float32)\n"," G = input_matrix.transpose().dot(W2).dot(input_matrix).toarray()\n"," if self.verbose:\n"," print(f\"G is made. Sparsity:{(1 - np.count_nonzero(G)/(self.n_items**2))*100}%\")\n","\n"," P = np.linalg.inv(G + np.identity(self.n_items, dtype=np.float32) * self.reg)\n"," if self.verbose:\n"," print(\"P is made\")\n"," del G\n","\n"," if self.alpha == 1:\n"," C = -P @ (input_matrix.transpose().dot(W2).dot(input_matrix-target_matrix).toarray())\n","\n"," mu = np.zeros(self.n_items)\n"," mu += self.reg\n"," mu_nonzero_idx = np.where(1 - np.diag(P)*self.reg + np.diag(C) >= self.epsilon)\n"," mu[mu_nonzero_idx] = (np.diag(1 - self.epsilon + C) / np.diag(P))[mu_nonzero_idx]\n","\n"," # B = I - Pλ + C\n"," self.enc_w = np.identity(self.n_items, dtype=np.float32) - P @ np.diag(mu) + C\n"," if self.verbose:\n"," print(\"weight matrix is made\")\n"," else:\n"," self.enc_w = P @ input_matrix.transpose().dot(W2).dot(target_matrix).toarray()\n","\n","\n"," def make_train_matrix(self, data, weight_by='SLIT'):\n"," input_row = []\n"," target_row = []\n"," input_col = []\n"," target_col = []\n"," input_data = []\n"," target_data = []\n","\n"," maxtime = data.Time.max()\n"," w2 = []\n"," sessionlengthmap = data['SessionIdx'].value_counts(sort=False)\n"," rowid = -1\n"," \n"," directory = os.path.dirname('./data_ckpt/')\n"," if not os.path.exists(directory):\n"," os.makedirs(directory)\n","\n"," if weight_by == 'SLIT':\n"," if os.path.exists(f'./data_ckpt/{self.n_sessions}_{self.n_items}_{self.direction}_SLIT.p'):\n"," with open(f'./data_ckpt/{self.n_sessions}_{self.n_items}_{self.direction}_SLIT.p','rb') as f:\n"," input_row, input_col, input_data, target_row, target_col, target_data, w2 = pickle.load(f)\n"," else:\n"," for sid, session in tqdm(data.groupby(['SessionIdx']), desc=weight_by):\n"," slen = sessionlengthmap[sid]\n"," # sessionitems = session['ItemIdx'].tolist() # sorted by itemid\n"," sessionitems = session.sort_values(['Time'])['ItemIdx'].tolist() # sorted by time\n"," slen = len(sessionitems)\n"," if slen <= 1:\n"," continue\n"," stime = session['Time'].max()\n"," w2 += [stime-maxtime] * (slen-1)\n"," for t in range(slen-1):\n"," rowid += 1\n"," # input matrix\n"," if self.direction == 'part':\n"," input_row += [rowid] * (t+1)\n"," input_col += sessionitems[:t+1]\n"," for s in range(t+1):\n"," input_data.append(-abs(t-s))\n"," target_row += [rowid] * (slen - (t+1))\n"," target_col += sessionitems[t+1:]\n"," for s in range(t+1, slen):\n"," target_data.append(-abs((t+1)-s))\n"," elif self.direction == 'all':\n"," input_row += [rowid] * slen\n"," input_col += sessionitems\n"," for s in range(slen):\n"," input_data.append(-abs(t-s))\n"," target_row += [rowid] * slen\n"," target_col += sessionitems\n"," for s in range(slen):\n"," target_data.append(-abs((t+1)-s))\n"," elif self.direction == 'sr':\n"," input_row += [rowid]\n"," input_col += [sessionitems[t]]\n"," input_data.append(0)\n"," target_row += [rowid] * (slen - (t+1))\n"," target_col += sessionitems[t+1:]\n"," for s in range(t+1, slen):\n"," target_data.append(-abs((t+1)-s))\n"," else:\n"," raise (\"You have to choose right 'direction'!\")\n"," with open(f'./data_ckpt/{self.n_sessions}_{self.n_items}_{self.direction}_SLIT.p','wb') as f:\n"," pickle.dump([input_row, input_col, input_data, target_row, target_col, target_data, w2], f, protocol=4)\n"," input_data = list(np.exp(np.array(input_data) / self.train_weight))\n"," target_data = list(np.exp(np.array(target_data) / self.train_weight))\n"," elif weight_by == 'SLIS':\n"," if os.path.exists(f'./data_ckpt/{self.n_sessions}_{self.n_items}_SLIS.p'):\n"," with open(f'./data_ckpt/{self.n_sessions}_{self.n_items}_SLIS.p','rb') as f:\n"," input_row, input_col, input_data, target_row, target_col, target_data, w2 = pickle.load(f)\n"," else:\n"," for sid, session in tqdm(data.groupby(['SessionIdx']), desc=weight_by):\n"," rowid += 1\n"," slen = sessionlengthmap[sid]\n"," sessionitems = session['ItemIdx'].tolist()\n"," stime = session['Time'].max()\n"," w2.append(stime-maxtime)\n"," input_row += [rowid] * slen\n"," input_col += sessionitems\n","\n"," target_row = input_row\n"," target_col = input_col\n"," input_data = np.ones_like(input_row)\n"," target_data = np.ones_like(target_row)\n"," \n"," with open(f'./data_ckpt/{self.n_sessions}_{self.n_items}_SLIS.p','wb') as f:\n"," pickle.dump([input_row, input_col, input_data, target_row, target_col, target_data, w2], f, protocol=4)\n"," else:\n"," raise (\"You have to choose right 'weight_by'!\")\n","\n"," # Use train_weight or not\n"," input_data = input_data if self.train_weight > 0 else list(np.ones_like(input_data))\n"," target_data = target_data if self.train_weight > 0 else list(np.ones_like(target_data))\n","\n"," # Use session_weight or not\n"," w2 = list(np.exp(np.array(w2) / self.session_weight))\n"," w2 = w2 if self.session_weight > 0 else list(np.ones_like(w2))\n","\n"," # Make sparse_matrix\n"," input_matrix = csr_matrix((input_data, (input_row, input_col)), shape=(max(input_row)+1, self.n_items), dtype=np.float32)\n"," target_matrix = csr_matrix((target_data, (target_row, target_col)), shape=input_matrix.shape, dtype=np.float32)\n"," if self.verbose:\n"," print(f\"[{weight_by}]sparse matrix {input_matrix.shape} is made. Sparsity:{(1 - input_matrix.count_nonzero()/(self.n_items*input_matrix.shape[0]))*100}%\")\n","\n","\n"," if weight_by == 'SLIT':\n"," pass\n"," elif weight_by == 'SLIS':\n"," # Value of repeated items --> 1\n"," input_matrix.data = np.ones_like(input_matrix.data)\n"," target_matrix.data = np.ones_like(target_matrix.data)\n","\n"," # Normalization\n"," if self.normalize == 'l1':\n"," input_matrix = normalize(input_matrix, 'l1')\n"," elif self.normalize == 'l2':\n"," input_matrix = normalize(input_matrix, 'l2')\n"," else:\n"," pass\n","\n"," return input_matrix, target_matrix, w2\n","\n"," # 필수\n","\n"," def predict_next(self, session_id, input_item_id, predict_for_item_ids, input_user_id=None, skip=False, type='view', timestamp=0):\n"," '''\n"," Gives predicton scores for a selected set of items on how likely they be the next item in the session.\n"," Parameters\n"," --------\n"," session_id : int or string\n"," The session IDs of the event.\n"," input_item_id : int or string\n"," The item ID of the event.\n"," predict_for_item_ids : 1D array\n"," IDs of items for which the network should give prediction scores. Every ID must be in the set of item IDs of the training set.\n"," Returns\n"," --------\n"," out : pandas.Series\n"," Prediction scores for selected items on how likely to be the next item of this session. Indexed by the item IDs.\n"," '''\n"," # new session\n"," if session_id != self.session:\n"," self.session_items = []\n"," self.session = session_id\n"," self.session_times = []\n","\n"," if type == 'view':\n"," if input_item_id in self.itemidmap.index:\n"," self.session_items.append(input_item_id)\n"," self.session_times.append(timestamp)\n","\n"," # item id transfomration\n"," session_items_new_id = self.itemidmap[self.session_items].values\n"," predict_for_item_ids_new_id = self.itemidmap[predict_for_item_ids].values\n"," \n"," if session_items_new_id.shape[0] == 0:\n"," skip = True\n","\n"," if skip:\n"," return pd.Series(data=0, index=predict_for_item_ids)\n","\n"," W_test = np.ones_like(self.session_items, dtype=np.float32)\n"," W_test = self.enc_w[session_items_new_id[-1], session_items_new_id]\n"," for i in range(len(W_test)):\n"," W_test[i] = np.exp(-abs(i+1-len(W_test))/self.predict_weight)\n","\n"," W_test = W_test if self.predict_weight > 0 else np.ones_like(W_test)\n"," W_test = W_test.reshape(-1, 1)\n","\n"," # [session_items, num_items]\n"," preds = self.enc_w[session_items_new_id] * W_test\n"," # [num_items]\n"," preds = np.sum(preds, axis=0)\n"," preds = preds[predict_for_item_ids_new_id]\n","\n"," series = pd.Series(data=preds, index=predict_for_item_ids)\n","\n"," series = series / series.max()\n"," \n"," # remove current item from series of prediction\n"," # series.drop(labels=[input_item_id])\n"," \n"," return series\n","\n"," # 필수\n"," def clear(self):\n"," self.enc_w = {}"],"metadata":{"id":"FnPMwD7_32BW"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Main"],"metadata":{"id":"qtKntJplRUu9"}},{"cell_type":"code","source":["'''\n","FILE PARAMETERS\n","'''\n","PATH_PROCESSED = './prepared/'\n","FILE = 'events'"],"metadata":{"id":"yOMQysGb3Iwp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["'''\n","MODEL HYPERPARAMETER TUNING\n","'''\n","alpha = 0.2 #[0.2, 0.4, 0.6, 0.8] \n","direction = 'all' # sr / part / all\n","reg = 10\n","train_weight = 1 #0.5 #[0.125, 0.25, 0.5, 1, 2, 4, 8]\n","predict_weight = 1 #4 #[0.125, 0.25, 0.5, 1, 2, 4, 8]\n","session_weight = 1 #256 #[1, 2, 4, 8, 16, 32, 64, 128, 256]"],"metadata":{"id":"RYIZjliz3LUh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Training\n","train, val = load_data_session(PATH_PROCESSED, FILE, train_eval=True)\n","model = SLIST(alpha=alpha, direction=direction, reg=reg, train_weight=train_weight, \n"," predict_weight=predict_weight, session_weight=session_weight)\n","model.fit(train, val)\n","\n","mrr = MRR(length=100)\n","hr = HitRate()\n","pop = Popularity()\n","pop.init(train)\n","cov = Coverage()\n","cov.init(train)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4a4mvCbi3TAy","outputId":"0d221f82-da35-42aa-e0b1-91fa3f8051cd","executionInfo":{"status":"ok","timestamp":1639119017665,"user_tz":-330,"elapsed":17591,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}}},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["START load data\n","Loaded train set\n","\tEvents: 53254\n","\tSessions: 13629\n","\tItems: 2873\n","\tSpan: 2014-04-01 / 2014-04-06\n","\n","Loaded test set\n","\tEvents: 16539\n","\tSessions: 4084\n","\tItems: 2029\n","\tSpan: 2014-04-06 / 2014-04-07\n","\n","END load data 0.08983836199999473 c / 0.0898427963256836 s\n"]},{"output_type":"stream","name":"stderr","text":["SLIS: 100%|██████████| 13629/13629 [00:03<00:00, 3569.65it/s]\n","SLIT: 100%|██████████| 13629/13629 [00:09<00:00, 1411.28it/s]\n"]}]},{"cell_type":"code","source":["result = evaluate_sessions(model, [mrr, hr, pop, cov], val, train)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"n94vXWR67O36","executionInfo":{"status":"ok","timestamp":1639119130459,"user_tz":-330,"elapsed":62776,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"5d2f7bbe-e29f-46f5-9929-16a28702ef9d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["START evaluation of 16539 actions in 4084 sessions\n"]},{"output_type":"stream","name":"stderr","text":["100%|██████████| 16539/16539 [01:02<00:00, 263.56it/s]"]},{"output_type":"stream","name":"stdout","text":["\n","END evaluation in 62.77697779699997 c / 62.77697801589966 s\n"," avg rt 0.003989196120376618 s / 0.003987816305178503 c\n"," time count 12455 count/ 49.68543767929077 sum\n"]},{"output_type":"stream","name":"stderr","text":["\n"]}]},{"cell_type":"code","source":["result"],"metadata":{"id":"SpsUr_403z0U","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1639119133944,"user_tz":-330,"elapsed":555,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"b51ed462-d70f-41df-d262-3bc6927f15ff"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('MRR@100: ',\n"," 0.3521687942062188,\n"," 'Bin: ;\\nPrecision@100: ;',\n"," 'Pos: ;0;1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16;17;18;19;20;21;22;23;24;25;26;27;28;29;30;31;32;33;34;35;36;37;38;39;40;41;42;43;44;45;46;47;48;49;50;51;\\nPrecision@100: ;0.39043544699954863;0.3325325335562308;0.358431025984701;0.31195065232051633;0.3372114712929858;0.32268523251291464;0.3555237695229066;0.3166781959167088;0.34238487011710844;0.3221189158972095;0.31515860275983537;0.2895189958634645;0.3960849891052557;0.3853882229014253;0.3197574713916099;0.3025236016461091;0.3004164800841002;0.36369765591924186;0.3808260628265034;0.29341860165801975;0.2725921570091597;0.34991145218417946;0.24232666413393963;0.4091435185185185;0.2363186813186813;0.22504370629370626;0.18930041152263372;0.11895735686058267;0.14806547619047616;0.3020408163265306;0.13849206349206347;0.3333333333333333;0.3482142857142857;0.10606060606060605;0.125;0.6;0.2;0.0;0.03333333333333333;0.0;0.06666666666666667;0.3333333333333333;0.14285714285714285;0.1111111111111111;0.058823529411764705;0.14285714285714285;0.0;0.06666666666666667;0.0;0.058823529411764705;0.047619047619047616;0.3333333333333333;'),\n"," ('HitRate@20: ',\n"," 0.637173825772782,\n"," 'Bin: ;\\nHitRate@20: ;',\n"," 'Pos: ;0;1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16;17;18;19;20;21;22;23;24;25;26;27;28;29;30;31;32;33;34;35;36;37;38;39;40;41;42;43;44;45;46;47;48;49;50;51;\\nHitRate@20: ;0.6339373163565132;0.6320914479254869;0.6607958251793868;0.627134724857685;0.6466575716234653;0.626641651031895;0.6666666666666666;0.6081504702194357;0.6629213483146067;0.6051282051282051;0.6329113924050633;0.59375;0.7247706422018348;0.6630434782608695;0.5833333333333334;0.639344262295082;0.6;0.6122448979591837;0.7297297297297297;0.6774193548387096;0.4583333333333333;0.6363636363636364;0.6842105263157895;0.7222222222222222;0.5333333333333333;0.7272727272727273;0.4444444444444444;0.4444444444444444;0.625;0.42857142857142855;0.5;0.5;0.75;0.5;0.5;1.0;1.0;0.0;0.0;0.0;1.0;1.0;1.0;1.0;1.0;1.0;0.0;1.0;0.0;1.0;0.0;1.0;'),\n"," ('Popularity@20: ', 0.12476653149405403),\n"," ('Coverage@20: ', 0.9557953358858337)]"]},"metadata":{},"execution_count":27}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"ogypTsIWHEyR"}},{"cell_type":"code","source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HY5KNqOJHEyT","executionInfo":{"status":"ok","timestamp":1639119146044,"user_tz":-330,"elapsed":3786,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"ed8e9ba0-c8c1-4072-8e22-f40f8efb340e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2021-12-10 06:52:33\n","\n","Compiler : GCC 7.5.0\n","OS : Linux\n","Release : 5.4.104+\n","Machine : x86_64\n","Processor : x86_64\n","CPU cores : 2\n","Architecture: 64bit\n","\n","numpy : 1.19.5\n","IPython: 5.5.0\n","scipy : 1.4.1\n","pandas : 1.1.5\n","\n"]}]},{"cell_type":"markdown","source":["---"],"metadata":{"id":"3RXL1ys5HEyU"}},{"cell_type":"markdown","source":["**END**"],"metadata":{"id":"E8qwaCPdHEyU"}}]} \ No newline at end of file