graph_unet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import copy\n",
    "import time\n",
    "import math\n",
    "import torch\n",
    "import torch.utils\n",
    "import torch.utils.data\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.optim.lr_scheduler as lr_scheduler\n",
    "from torch.nn.parameter import Parameter\n",
    "from os.path import join as pjoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch 0.4.1\n"
     ]
    }
   ],
   "source": [
    "# Experiment parameters\n",
    "batch_size = 32\n",
    "threads = 0\n",
    "lr = 0.005\n",
    "epochs = 40\n",
    "log_interval = 10\n",
    "wdecay = 1e-4\n",
    "dataset = 'proteins'\n",
    "model_name = 'unet'  # 'gcn', 'unet'\n",
    "device = 'cpu'  # 'cuda', 'cpu'\n",
    "visualize = True\n",
    "shuffle_nodes = False\n",
    "n_folds = 10  # 10-fold cross validation\n",
    "seed = 111\n",
    "print('torch', torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data loader and reader\n",
    "class GraphData(torch.utils.data.Dataset):\n",
    "    def __init__(self,\n",
    "                 datareader,\n",
    "                 fold_id,\n",
    "                 split):\n",
    "        self.fold_id = fold_id\n",
    "        self.split = split\n",
    "        self.rnd_state = datareader.rnd_state\n",
    "        self.set_fold(datareader.data, fold_id)\n",
    "\n",
    "    def set_fold(self, data, fold_id):\n",
    "        self.total = len(data['targets'])\n",
    "        self.N_nodes_max = data['N_nodes_max']\n",
    "        self.n_classes = data['n_classes']\n",
    "        self.features_dim = data['features_dim']\n",
    "        self.idx = data['splits'][fold_id][self.split]\n",
    "         # use deepcopy to make sure we don't alter objects in folds\n",
    "        self.labels = copy.deepcopy([data['targets'][i] for i in self.idx])\n",
    "        self.adj_list = copy.deepcopy([data['adj_list'][i] for i in self.idx])\n",
    "        self.features_onehot = copy.deepcopy([data['features_onehot'][i] for i in self.idx])\n",
    "        print('%s: %d/%d' % (self.split.upper(), len(self.labels), len(data['targets'])))\n",
    "        self.indices = np.arange(len(self.idx))  # sample indices for this epoch\n",
    "        \n",
    "    def pad(self, mtx, desired_dim1, desired_dim2=None, value=0):\n",
    "        sz = mtx.shape\n",
    "        assert len(sz) == 2, ('only 2d arrays are supported', sz)\n",
    "        # if np.all(np.array(sz) < desired_dim1 / 3): print('matrix shape is suspiciously small', sz, desired_dim1)\n",
    "        if desired_dim2 is not None:\n",
    "            mtx = np.pad(mtx, ((0, desired_dim1 - sz[0]), (0, desired_dim2 - sz[1])), 'constant', constant_values=value)\n",
    "        else:\n",
    "            mtx = np.pad(mtx, ((0, desired_dim1 - sz[0]), (0, 0)), 'constant', constant_values=value)\n",
    "        return mtx\n",
    "    \n",
    "    def nested_list_to_torch(self, data):\n",
    "        if isinstance(data, dict):\n",
    "            keys = list(data.keys())           \n",
    "        for i in range(len(data)):\n",
    "            if isinstance(data, dict):\n",
    "                i = keys[i]\n",
    "            if isinstance(data[i], np.ndarray):\n",
    "                data[i] = torch.from_numpy(data[i]).float()\n",
    "            elif isinstance(data[i], list):\n",
    "                data[i] = list_to_torch(data[i])\n",
    "        return data\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        index = self.indices[index]\n",
    "        N_nodes_max = self.N_nodes_max\n",
    "        N_nodes = self.adj_list[index].shape[0]\n",
    "        graph_support = np.zeros(self.N_nodes_max)\n",
    "        graph_support[:N_nodes] = 1\n",
    "        return self.nested_list_to_torch([self.pad(self.features_onehot[index].copy(), self.N_nodes_max),  # node_features\n",
    "                                          self.pad(self.adj_list[index], self.N_nodes_max, self.N_nodes_max),  # adjacency matrix\n",
    "                                          graph_support,  # mask with values of 0 for dummy (zero padded) nodes, otherwise 1 \n",
    "                                          N_nodes,\n",
    "                                          int(self.labels[index])])  # convert to torch\n",
    "\n",
    "\n",
    "class DataReader():\n",
    "    '''\n",
    "    Class to read the txt files containing all data of the dataset\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                 data_dir,  # folder with txt files\n",
    "                 rnd_state=None,\n",
    "                 use_cont_node_attr=False,  # use or not additional float valued node attributes available in some datasets\n",
    "                 folds=10):\n",
    "\n",
    "        self.data_dir = data_dir\n",
    "        self.rnd_state = np.random.RandomState() if rnd_state is None else rnd_state\n",
    "        self.use_cont_node_attr = use_cont_node_attr\n",
    "        files = os.listdir(self.data_dir)\n",
    "        data = {}\n",
    "        nodes, graphs = self.read_graph_nodes_relations(list(filter(lambda f: f.find('graph_indicator') >= 0, files))[0])\n",
    "        data['features'] = self.read_node_features(list(filter(lambda f: f.find('node_labels') >= 0, files))[0], \n",
    "                                                 nodes, graphs, fn=lambda s: int(s.strip()))  \n",
    "        data['adj_list'] = self.read_graph_adj(list(filter(lambda f: f.find('_A') >= 0, files))[0], nodes, graphs)                      \n",
    "        data['targets'] = np.array(self.parse_txt_file(list(filter(lambda f: f.find('graph_labels') >= 0, files))[0],\n",
    "                                                       line_parse_fn=lambda s: int(float(s.strip()))))\n",
    "        \n",
    "        if self.use_cont_node_attr:\n",
    "            data['attr'] = self.read_node_features(list(filter(lambda f: f.find('node_attributes') >= 0, files))[0], \n",
    "                                                   nodes, graphs, fn=lambda s: np.array(list(map(float, s.strip().split(',')))))\n",
    "        \n",
    "        features, n_edges, degrees = [], [], []\n",
    "        for sample_id, adj in enumerate(data['adj_list']):\n",
    "            N = len(adj)  # number of nodes\n",
    "            if data['features'] is not None:\n",
    "                assert N == len(data['features'][sample_id]), (N, len(data['features'][sample_id]))\n",
    "            n = np.sum(adj)  # total sum of edges\n",
    "            assert n % 2 == 0, n\n",
    "            n_edges.append( int(n / 2) )  # undirected edges, so need to divide by 2\n",
    "            if not np.allclose(adj, adj.T):\n",
    "                print(sample_id, 'not symmetric')\n",
    "            degrees.extend(list(np.sum(adj, 1)))\n",
    "            features.append(np.array(data['features'][sample_id]))\n",
    "                        \n",
    "        # Create features over graphs as one-hot vectors for each node\n",
    "        features_all = np.concatenate(features)\n",
    "        features_min = features_all.min()\n",
    "        features_dim = int(features_all.max() - features_min + 1)  # number of possible values\n",
    "        \n",
    "        features_onehot = []\n",
    "        for i, x in enumerate(features):\n",
    "            feature_onehot = np.zeros((len(x), features_dim))\n",
    "            for node, value in enumerate(x):\n",
    "                feature_onehot[node, value - features_min] = 1\n",
    "            if self.use_cont_node_attr:\n",
    "                feature_onehot = np.concatenate((feature_onehot, np.array(data['attr'][i])), axis=1)\n",
    "            features_onehot.append(feature_onehot)\n",
    "\n",
    "        if self.use_cont_node_attr:\n",
    "            features_dim = features_onehot[0].shape[1]\n",
    "            \n",
    "        shapes = [len(adj) for adj in data['adj_list']]\n",
    "        labels = data['targets']        # graph class labels\n",
    "        labels -= np.min(labels)        # to start from 0\n",
    "        N_nodes_max = np.max(shapes)    \n",
    "\n",
    "        classes = np.unique(labels)\n",
    "        n_classes = len(classes)\n",
    "\n",
    "        if not np.all(np.diff(classes) == 1):\n",
    "            print('making labels sequential, otherwise pytorch might crash')\n",
    "            labels_new = np.zeros(labels.shape, dtype=labels.dtype) - 1\n",
    "            for lbl in range(n_classes):\n",
    "                labels_new[labels == classes[lbl]] = lbl\n",
    "            labels = labels_new\n",
    "            classes = np.unique(labels)\n",
    "            assert len(np.unique(labels)) == n_classes, np.unique(labels)\n",
    "\n",
    "        print('N nodes avg/std/min/max: \\t%.2f/%.2f/%d/%d' % (np.mean(shapes), np.std(shapes), np.min(shapes), np.max(shapes)))\n",
    "        print('N edges avg/std/min/max: \\t%.2f/%.2f/%d/%d' % (np.mean(n_edges), np.std(n_edges), np.min(n_edges), np.max(n_edges)))\n",
    "        print('Node degree avg/std/min/max: \\t%.2f/%.2f/%d/%d' % (np.mean(degrees), np.std(degrees), np.min(degrees), np.max(degrees)))\n",
    "        print('Node features dim: \\t\\t%d' % features_dim)\n",
    "        print('N classes: \\t\\t\\t%d' % n_classes)\n",
    "        print('Classes: \\t\\t\\t%s' % str(classes))\n",
    "        for lbl in classes:\n",
    "            print('Class %d: \\t\\t\\t%d samples' % (lbl, np.sum(labels == lbl)))\n",
    "\n",
    "        for u in np.unique(features_all):\n",
    "            print('feature {}, count {}/{}'.format(u, np.count_nonzero(features_all == u), len(features_all)))\n",
    "        \n",
    "        N_graphs = len(labels)  # number of samples (graphs) in data\n",
    "        assert N_graphs == len(data['adj_list']) == len(features_onehot), 'invalid data'\n",
    "\n",
    "        # Create test sets first\n",
    "        train_ids, test_ids = self.split_ids(np.arange(N_graphs), rnd_state=self.rnd_state, folds=folds)\n",
    "\n",
    "        # Create train sets\n",
    "        splits = []\n",
    "        for fold in range(folds):\n",
    "            splits.append({'train': train_ids[fold],\n",
    "                           'test': test_ids[fold]})\n",
    "\n",
    "        data['features_onehot'] = features_onehot\n",
    "        data['targets'] = labels\n",
    "        data['splits'] = splits \n",
    "        data['N_nodes_max'] = np.max(shapes)  # max number of nodes\n",
    "        data['features_dim'] = features_dim\n",
    "        data['n_classes'] = n_classes\n",
    "        \n",
    "        self.data = data\n",
    "\n",
    "    def split_ids(self, ids_all, rnd_state=None, folds=10):\n",
    "        n = len(ids_all)\n",
    "        ids = ids_all[rnd_state.permutation(n)]\n",
    "        stride = int(np.ceil(n / float(folds)))\n",
    "        test_ids = [ids[i: i + stride] for i in range(0, n, stride)]\n",
    "        assert np.all(np.unique(np.concatenate(test_ids)) == sorted(ids_all)), 'some graphs are missing in the test sets'\n",
    "        assert len(test_ids) == folds, 'invalid test sets'\n",
    "        train_ids = []\n",
    "        for fold in range(folds):\n",
    "            train_ids.append(np.array([e for e in ids if e not in test_ids[fold]]))\n",
    "            assert len(train_ids[fold]) + len(test_ids[fold]) == len(np.unique(list(train_ids[fold]) + list(test_ids[fold]))) == n, 'invalid splits'\n",
    "\n",
    "        return train_ids, test_ids\n",
    "\n",
    "    def parse_txt_file(self, fpath, line_parse_fn=None):\n",
    "        with open(pjoin(self.data_dir, fpath), 'r') as f:\n",
    "            lines = f.readlines()\n",
    "        data = [line_parse_fn(s) if line_parse_fn is not None else s for s in lines]\n",
    "        return data\n",
    "    \n",
    "    def read_graph_adj(self, fpath, nodes, graphs):\n",
    "        edges = self.parse_txt_file(fpath, line_parse_fn=lambda s: s.split(','))\n",
    "        adj_dict = {}\n",
    "        for edge in edges:\n",
    "            node1 = int(edge[0].strip()) - 1  # -1 because of zero-indexing in our code\n",
    "            node2 = int(edge[1].strip()) - 1\n",
    "            graph_id = nodes[node1]\n",
    "            assert graph_id == nodes[node2], ('invalid data', graph_id, nodes[node2])\n",
    "            if graph_id not in adj_dict:\n",
    "                n = len(graphs[graph_id])\n",
    "                adj_dict[graph_id] = np.zeros((n, n))\n",
    "            ind1 = np.where(graphs[graph_id] == node1)[0]\n",
    "            ind2 = np.where(graphs[graph_id] == node2)[0]\n",
    "            assert len(ind1) == len(ind2) == 1, (ind1, ind2)\n",
    "            adj_dict[graph_id][ind1, ind2] = 1\n",
    "            \n",
    "        adj_list = [adj_dict[graph_id] for graph_id in sorted(list(graphs.keys()))]\n",
    "        \n",
    "        return adj_list\n",
    "        \n",
    "    def read_graph_nodes_relations(self, fpath):\n",
    "        graph_ids = self.parse_txt_file(fpath, line_parse_fn=lambda s: int(s.rstrip()))\n",
    "        nodes, graphs = {}, {}\n",
    "        for node_id, graph_id in enumerate(graph_ids):\n",
    "            if graph_id not in graphs:\n",
    "                graphs[graph_id] = []\n",
    "            graphs[graph_id].append(node_id)\n",
    "            nodes[node_id] = graph_id\n",
    "        graph_ids = np.unique(list(graphs.keys()))\n",
    "        for graph_id in graphs:\n",
    "            graphs[graph_id] = np.array(graphs[graph_id])\n",
    "        return nodes, graphs\n",
    "\n",
    "    def read_node_features(self, fpath, nodes, graphs, fn):\n",
    "        node_features_all = self.parse_txt_file(fpath, line_parse_fn=fn)\n",
    "        node_features = {}\n",
    "        for node_id, x in enumerate(node_features_all):\n",
    "            graph_id = nodes[node_id]\n",
    "            if graph_id not in node_features:\n",
    "                node_features[graph_id] = [ None ] * len(graphs[graph_id])\n",
    "            ind = np.where(graphs[graph_id] == node_id)[0]\n",
    "            assert len(ind) == 1, ind\n",
    "            assert node_features[graph_id][ind[0]] is None, node_features[graph_id][ind[0]]\n",
    "            node_features[graph_id][ind[0]] = x\n",
    "        node_features_lst = [node_features[graph_id] for graph_id in sorted(list(graphs.keys()))]\n",
    "        return node_features_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NN layers and models\n",
    "class GraphConv(nn.Module):\n",
    "    '''\n",
    "    Graph Convolution Layer according to (T. Kipf and M. Welling, ICLR 2017)\n",
    "    Additional tricks (power of adjacency matrix and weight self connections) as in the Graph U-Net paper\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                in_features,\n",
    "                out_features,\n",
    "                activation=None,\n",
    "                adj_sq=False,\n",
    "                scale_identity=False):\n",
    "        super(GraphConv, self).__init__()\n",
    "        self.fc = nn.Linear(in_features=in_features, out_features=out_features)\n",
    "        self.adj_sq = adj_sq\n",
    "        self.activation = activation\n",
    "        self.scale_identity = scale_identity\n",
    "            \n",
    "    def laplacian_batch(self, A):\n",
    "        batch, N = A.shape[:2]\n",
    "        if self.adj_sq:\n",
    "            A = torch.bmm(A, A)  # use A^2 to increase graph connectivity\n",
    "        I = torch.eye(N).unsqueeze(0).to(device)\n",
    "        if self.scale_identity:\n",
    "            I = 2 * I  # increase weight of self connections\n",
    "        A_hat = A + I\n",
    "        D_hat = (torch.sum(A_hat, 1) + 1e-5) ** (-0.5)\n",
    "        L = D_hat.view(batch, N, 1) * A_hat * D_hat.view(batch, 1, N)\n",
    "        return L\n",
    "\n",
    "    def forward(self, data):\n",
    "        x, A = data[:2]\n",
    "        x = self.fc(torch.bmm(self.laplacian_batch(A), x))\n",
    "        if self.activation is not None:\n",
    "            x = self.activation(x)\n",
    "        return (x, A)\n",
    "        \n",
    "class GCN(nn.Module):\n",
    "    '''\n",
    "    Baseline Graph Convolutional Network with a stack of Graph Convolution Layers and global pooling over nodes.\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                 in_features,\n",
    "                 out_features,\n",
    "                 filters=[64,64,64],\n",
    "                 n_hidden=0,\n",
    "                 dropout=0.2,\n",
    "                 adj_sq=False,\n",
    "                 scale_identity=False):\n",
    "        super(GCN, self).__init__()\n",
    "\n",
    "        # Graph convolution layers\n",
    "        self.gconv = nn.Sequential(*([GraphConv(in_features=in_features if layer == 0 else filters[layer - 1], \n",
    "                                                out_features=f, \n",
    "                                                activation=nn.ReLU(inplace=True),\n",
    "                                                adj_sq=adj_sq,\n",
    "                                                scale_identity=scale_identity) for layer, f in enumerate(filters)]))\n",
    "        \n",
    "        # Fully connected layers\n",
    "        fc = []\n",
    "        if dropout > 0:\n",
    "            fc.append(nn.Dropout(p=dropout))\n",
    "        if n_hidden > 0:\n",
    "            fc.append(nn.Linear(filters[-1], n_hidden))\n",
    "            if dropout > 0:\n",
    "                fc.append(nn.Dropout(p=dropout))\n",
    "            n_last = n_hidden\n",
    "        else:\n",
    "            n_last = filters[-1]\n",
    "        fc.append(nn.Linear(n_last, out_features))       \n",
    "        self.fc = nn.Sequential(*fc)\n",
    "        \n",
    "    def forward(self, data):\n",
    "        x = self.gconv(data)[0]\n",
    "        x = torch.max(x, dim=1)[0].squeeze()  # max pooling over nodes\n",
    "        x = self.fc(x)\n",
    "        return x  \n",
    "    \n",
    "class GraphUnet(nn.Module):\n",
    "    def __init__(self,\n",
    "                 in_features,\n",
    "                 out_features,\n",
    "                 filters=[64,64,64],\n",
    "                 n_hidden=0,\n",
    "                 dropout=0.2,\n",
    "                 adj_sq=False,\n",
    "                 scale_identity=False,\n",
    "                 shuffle_nodes=False,\n",
    "                 visualize=False,\n",
    "                 pooling_ratios=[0.8, 0.8]):\n",
    "        super(GraphUnet, self).__init__()\n",
    "\n",
    "        self.shuffle_nodes = shuffle_nodes\n",
    "        self.visualize = visualize\n",
    "        self.pooling_ratios = pooling_ratios\n",
    "        # Graph convolution layers\n",
    "        self.gconv = nn.ModuleList([GraphConv(in_features=in_features if layer == 0 else filters[layer - 1], \n",
    "                                                out_features=f, \n",
    "                                                activation=nn.ReLU(inplace=True),\n",
    "                                               adj_sq=adj_sq,\n",
    "                                               scale_identity=scale_identity) for layer, f in enumerate(filters)])\n",
    "        # Pooling layers\n",
    "        self.proj = []\n",
    "        for layer, f in enumerate(filters[:-1]):\n",
    "            # Initialize projection vectors similar to weight/bias initialization in nn.Linear\n",
    "            fan_in = filters[layer]\n",
    "            p = Parameter(torch.Tensor(fan_in, 1))\n",
    "            bound = 1 / math.sqrt(fan_in)\n",
    "            torch.nn.init.uniform_(p, -bound, bound)\n",
    "            self.proj.append(p)\n",
    "        \n",
    "        # Fully connected layers\n",
    "        fc = []\n",
    "        if dropout > 0:\n",
    "            fc.append(nn.Dropout(p=dropout))\n",
    "        if n_hidden > 0:\n",
    "            fc.append(nn.Linear(filters[-1], n_hidden))\n",
    "            if dropout > 0:\n",
    "                fc.append(nn.Dropout(p=dropout))\n",
    "            n_last = n_hidden\n",
    "        else:\n",
    "            n_last = filters[-1]\n",
    "        fc.append(nn.Linear(n_last, out_features))       \n",
    "        self.fc = nn.Sequential(*fc)\n",
    "        \n",
    "    def forward(self, data):\n",
    "        # [signal, W, signal_support, N_nodes, int(label)]\n",
    "        if self.shuffle_nodes:\n",
    "            N = data[0].shape[1]\n",
    "            idx = torch.randperm(N)\n",
    "            data = (data[0][:, idx], data[1][:, idx][idx, :], data[2][:, idx], data[3])\n",
    "        plot = -1\n",
    "        N_nodes_tmp = -1\n",
    "        for layer, gconv in enumerate(self.gconv):\n",
    "            N_nodes = data[3]\n",
    "            N_nodes_max = N_nodes.max()\n",
    "            #print('layer', layer, N_nodes_max)\n",
    "            #data = (data[0][:, :N_nodes_max], data[1][:, :N_nodes_max, :N_nodes_max], data[2][:, :N_nodes_max], data[3])      \n",
    "            B, N, _ = data[0].shape\n",
    "            if layer < len(self.gconv) - 1 and self.visualize:      \n",
    "                x, W = data[:2]\n",
    "                for b in range(B):\n",
    "                    if (layer == 0 and N_nodes[b] < 20 and N_nodes[b] > 10) or plot > -1:\n",
    "                        if plot > -1 and plot != b:\n",
    "                            continue\n",
    "                        if N_nodes_tmp < 0:\n",
    "                            N_nodes_tmp = N_nodes[b]\n",
    "                        plt.figure(figsize=(18,5))\n",
    "                        plt.subplot(141)\n",
    "                        plt.title('layer %d, Input adjacency matrix' % (layer))\n",
    "                        plt.imshow(W[b][:N_nodes_tmp, :N_nodes_tmp].data.cpu().numpy())\n",
    "                        plot = b                        \n",
    "                        break\n",
    "            mask = data[2].clone()\n",
    "            data = gconv(data)\n",
    "            x, W = data\n",
    "            if layer < len(self.gconv) - 1:\n",
    "                B, N, C = x.shape\n",
    "                y = torch.mm(x.view(B * N, C), self.proj[layer]).view(B, N)\n",
    "                y = y / (torch.sum(self.proj[layer] ** 2).view(1, 1) ** 0.5)  # node scores used for ranking below\n",
    "                idx = torch.sort(y, dim=1)[1]  # B,N                \n",
    "                N_remove = (N_nodes.float() * (1 - self.pooling_ratios[layer])).long()\n",
    "                assert torch.all(N_nodes > N_remove), 'the number of removed nodes must be large than the number of nodes'\n",
    "                for b in range(B):\n",
    "                    assert torch.sum(mask[b]) == float(N_nodes[b]), (torch.sum(mask[b]), N_nodes[b])\n",
    "                N_nodes_prev = N_nodes\n",
    "                N_nodes = N_nodes - N_remove\n",
    "                                \n",
    "                for b in range(B):\n",
    "                    idx_b = idx[b, mask[b, idx[b]] == 1]\n",
    "                    assert len(idx_b) >= N_nodes[b], (len(idx_b), N_nodes[b])\n",
    "                    mask[b, idx_b[:N_remove[b]]] = 0\n",
    "                for b in range(B):\n",
    "                    assert torch.sum(mask[b]) == float(N_nodes[b]), (b, torch.sum(mask[b]), N_nodes[b], N_remove[b], N_nodes_prev[b])\n",
    "                    s = torch.sum(y[b] >= torch.min((y * mask.float())[b]))\n",
    "                    assert s >= float(N_nodes[b]), (s, N_nodes[b], (y * mask.float())[b])\n",
    "                \n",
    "                mask = mask.unsqueeze(2)\n",
    "                x = x * torch.tanh(y).unsqueeze(2) * mask\n",
    "                W = mask * W * mask.view(B, 1, N)\n",
    "                mask = mask.squeeze()\n",
    "                data = (x, W, mask, N_nodes)\n",
    "                \n",
    "                if self.visualize and plot > -1:\n",
    "                    b = plot\n",
    "                    plt.subplot(142)\n",
    "                    plt.title('layer %d, Ranking' % (layer))\n",
    "                    plt.imshow(y[b].view(N, 1).expand(N, 2)[:N_nodes_tmp].data.cpu().numpy())\n",
    "                    plt.colorbar()\n",
    "                    plt.subplot(143)\n",
    "                    plt.title('layer %d, Pooled nodes (%d/%d)' % (layer, mask[b].sum(), N_nodes_prev[b]))\n",
    "                    plt.imshow(mask[b].view(N, 1).expand(N, 2)[:N_nodes_tmp].data.cpu().numpy())\n",
    "                    plt.subplot(144)\n",
    "                    plt.title('layer %d, Pooled adjacency matrix' % (layer))\n",
    "                    plt.imshow(W[b][:N_nodes_tmp, :N_nodes_tmp].data.cpu().numpy())\n",
    "                    plt.show()\n",
    "                        \n",
    "        if self.visualize and plot > -1:\n",
    "            self.visualize = False\n",
    "        x = torch.max(x, dim=1)[0].squeeze()  # max pooling over nodes\n",
    "        x = self.fc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data\n",
      "N nodes avg/std/min/max: \t39.06/45.76/4/620\n",
      "N edges avg/std/min/max: \t72.82/84.60/5/1049\n",
      "Node degree avg/std/min/max: \t3.73/1.15/0/25\n",
      "Node features dim: \t\t3\n",
      "N classes: \t\t\t2\n",
      "Classes: \t\t\t[0 1]\n",
      "Class 0: \t\t\t663 samples\n",
      "Class 1: \t\t\t450 samples\n",
      "feature 0, count 21151/43471\n",
      "feature 1, count 20931/43471\n",
      "feature 2, count 1389/43471\n",
      "\n",
      "FOLD 0\n",
      "TRAIN: 1001/1113\n",
      "TEST: 112/1113\n",
      "\n",
      "Initialize model\n",
      "GraphUnet(\n",
      "  (gconv): ModuleList(\n",
      "    (0): GraphConv(\n",
      "      (fc): Linear(in_features=3, out_features=64, bias=True)\n",
      "      (activation): ReLU(inplace)\n",
      "    )\n",
      "    (1): GraphConv(\n",
      "      (fc): Linear(in_features=64, out_features=64, bias=True)\n",
      "      (activation): ReLU(inplace)\n",
      "    )\n",
      "    (2): GraphConv(\n",
      "      (fc): Linear(in_features=64, out_features=64, bias=True)\n",
      "      (activation): ReLU(inplace)\n",
      "    )\n",
      "  )\n",
      "  (fc): Sequential(\n",
      "    (0): Dropout(p=0.2)\n",
      "    (1): Linear(in_features=64, out_features=2, bias=True)\n",
      "  )\n",
      ")\n",
      "N trainable parameters: 8706\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBIAAAE/CAYAAAD7UuCJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3XmYZGV5///3h4EB2QRm0DCAYBJI\nRL9K4ogxMUIkCq6oGIW4gMYfJtG4xA0St6BGE5doItEQRUAURdxQMSMSCNGgMuiAIrKIC8OAbIIs\nCjJz//44p6Gm6a26u6q6qt+v6zpXV53lOfeprtNdz13PkqpCkiRJkiRpJjYZdACSJEmSJGl4mEiQ\nJEmSJEkzZiJBkiRJkiTNmIkESZIkSZI0YyYSJEmSJEnSjJlIkCRJkiRJM2YiQZI0I0l+nORPBx3H\nQpVk9ySVZNMJtj0gya1JlgwiNjV8D09uLq9Nv17XJG9OclIPyt0ryer5LneukrwsyTsGHYckTcRE\ngiRpqKTxT0luaJd/TpIZHvvmJL9uK/U3Jfm/JI/qdcxV9dOq2rqq1vf6XFr4hvE9POLeArxr7EmS\nlyZZneSOJMd37pjkD5KckeTGJNcl+VSSncYXmOTSJHsm+ZMkZyW5OcmPJwsgyb5tIvKtHauPBZ6b\n5H5zvkJJmmcmEiRJC9ZE3+4DRwBPAx4GPBR4MvDiLor9ZFVtDSwHzgI+Ndc4pcn0+D28I/A14DMz\nTURoY20S4E+Az3WsXge8FThugkO2p6ng7w7sBtwCfGRcmb8FbFJVlwK3teW8ZooYNgPeB3yzc31V\n/Qr4MvD8bq5JkvrBRIIkqWtJ9klybvuN6NVJ3p9kabvtmCTvHrf/F5K8on28Ismn22/zfpTkZR37\nvTnJqUlOSvIL4PAJTn8Y8O6qWltVVwHvnmS/KVXVXcDHgJ2T7Nief/skX2xj+3n7eJeO+M5O8pYk\nX09yS5KvJFk+yWt0cNvk+yHjuz1MV06S5yf5Sftt9Rtskj//RuQ9/GvgBOA3gGVJNkny+va9c22S\nE5PctyO2pya5qL3ms5M8aJLXZpMkRyb5YfsePCXJDh3bn9fx/vz7qWJMcnz7en6pfa9/s61oj23/\nwyTntd/Yn5fkDzu2PTDJ/7THnUGT/Oss+w/StMi4KckFSfbr2HZ4kivaY3+U5DmThPg44NttpX3s\ndf1MVX0OuGH8zlX15ar6VFX9oqpuB94P/NG43Z4EnN7u/62q+ihwxRQv06uArwA/mGDb2W15krSg\nmEiQJM3GeuCVNB/sHwXsD/x1u+0E4NAkmwC0FeT9gZPbdV8ALgB2bte/IskBHWUfBJwKbEdT0R/v\nwe3xYy5o13WlrTQ+n6ay8PN29SY03y7uBjwA+CVNRaHTnwMvAO4HLAVePUHZLwD+CfjTqvreJCFM\nWE6SvYB/B54D7ATcl+a10vwahffw5jQJiLVVdX37+HCab9h/E9ia9v2bZE/gZOAVNC0ZTge+MJY8\nGedlNC0m9gVW0Nwfx7Tl7AV8AHheu20ZsMsEZXQ6FPgHmm/zLwfe1pa1A/Al4F/bct4DfCnJsva4\njwPn0/yO3kKTgBm79p3bY98K7EBz/3w6yY5JtmrLfEJVbQP8IbBmktj+H3DJNPFP5THARePWPbGN\nbVpJdgNeCBw9yS4X07RckaQFxUSCJKlrVXV+VX2jqu6qqh8D/0FT6aCqvgXcTFPBAjgEOLuqfgY8\nAtixqo6uqjur6grgP9t9xpxbVZ+rqg1V9csJTr91W/6Ym4Gtkxk37X5WkptokgT/H/DMtnUCVXVD\nVX26qm6vqltoKjz7jjv+I1V1aRvbKcDe47a/gqYZ835VdfkUcUxWzjOBL1TV16rqTuCNQM3w2jRD\nI/IevhJ4OE2lH5rk03uq6oqquhU4CjikbQnzbOBLVXVG25LhXcB9aCrZ470Y+Pu2xcQdwJuBZ7bl\nPBP4YlWd0257A7Bhmng/034zP9YKaOy9/iTgsqr6aPt7OJnmW/mnJHkAzWv9hqq6o6rOoUngjHku\ncHpVnd6+zmcAq2kq8bQxPSTJfarq6qoaX9kfsx1N94SuJXkozf35mo51W7Zx/88Mi/lXmmu8dZLt\nt9AkEyVpQTGRIEnqWppBxL6Y5Jq2+fY/snGz4xNoPujT/vxo+3g3YEXbFPmmtjL0d8D9O469cprT\n3wps2/F8W+DWqpppZfuUqtquPef3aCpiY9e1ZZL/aJtt/wI4B9guG8+2cE3H49tpKoWdXgMcU1Vr\np4ljsnJW0PEatM2n79XEWnMzCu/hqrpfVT22qs5v168AftKx30+ATdvYNtpWVRvaOCdq7bIb8NmO\n67uYpgXHWDmd78/bmP79OdV7/Sfj9v1JG9MK4Odt+Z3bOmP8s3G/h0cDO7XHPBv4S+DqtlvF704S\n28+BbaaJ/16S/DbN+AUvr6r/7di0P/B/nV0lpijjKcA2VfXJKXbbho2TTpK0IJhIkCTNxgdovjnc\no6q2palIdX6behJwUJKHAQ/inoHMrgR+1FaCxpZtquqJHcdOV5m6iI2b+j6MezctnlbbFPzFwJtz\nz6jrrwJ+B3hke12Padd3M5Dd44HXJzm425haV9PRVDzJfWiafWt+Df17eALraCrYYx4A3AX8bPy2\ntvXDrsBVE5RzJU23gM5r3KIdz+Hq9rixcrZk9u/P8fGOxTx2nu3bbgqd2zpj/Oi4GLeqqncAVNWq\nqnocTfegH9C0GpnIhcCe3QTddkf4KvCWdvyDTjPu1kCTdFjZJrOuoUl+vCLJ5zv2eRAbd4ORpAXB\nRIIkaTa2AX4B3Np+0/dXnRvbb+PPo/kW99Mdzbu/BfwiyeuS3CfJkjSDET6ii3OfCPxtkp2TrKCp\n/B8/tjHNwISHz6SgqvoBsAp4bcd1/RK4qe2//aYu4hpzEXAgcEySp87i+FNpmnb/Ydt//R/oLpGh\nmRmJ9/A4JwOvbAcp3JqmlcUn2y4FpwBPSrJ/mlkCXgXcAfzfBOV8EHhbW2GmHXfgoHbbqcCTkzy6\nfX8ezew/T54O7Jnkz5NsmuTZwF40XSd+QtNV4R+SLE3yaOApHceeRHOfHND+DrZIsl+SXZLcP83A\nklu113grTYuKiZwB/H6SLcZWtLFsASwBxsoeGyh1Z+C/aVodfXCC8p7QXtdYWZu0ZW3WPM0WHeNS\nvIEmibF3u5xGk/B4QUd5+9K0fJCkBcVEgiRpNl5NM1jgLTQffCdqmnsCzUBmd39jV1XraSoDewM/\nAq4HPkR3fYD/g6av9HdpuiZ8qV03NoDiMuAbXZT3TuCINHO1v5em3/j1bRn/1UU5d6uqC2im9PvP\nJE/o8tiLgL8BPkHzrewtwLU0FSLNn1F6D485ro31nDa2X9G8l6iqS2i6aPxbG/NTgKe043CM9z6a\nSu1XktzSxvLItpyLgJfQDIR4NU3XgOm68Uyoqm6guU9eRdM94rXAk9vWQtD8fh4J3EiT1Dux49gr\naQa1/DvgOpoWCq+h+Wy7SVvmuvbYfblnIM3xMfyMJjFwUMfq19MkFI+kec1+2a4DeBHNQJZvSnLr\n2AKQ5CE0XVR+2lHWY9rjT+eeAVy/0p77lqq6Zmxpt91WVTe25W1B08LhhOleS0nqt8y8O54kSTOX\n5DE03xru3vbH7sc5Hw28pKoO7cf5+qH9Zvkmmib4Pxp0PIuJ7+HFIc1MFCcA+3QxTsVE5bwWWF5V\nr51255mV9zfArvNVniTNJxMJkqR51zad/gRwQVVNNq2ZJtEOwnYmTZeGd9N8K/v7c6nkqDu+h9Wt\nJM8CvltVFw86FknqNbs2SJLmVZIH0XyDvhNNVwF17yCaZtnrgD2AQ0wi9I/vYc1GVZ1iEkHSYmGL\nBEmSJEmSNGO2SJAkSZIkSTNmIkGSJEmSJM3YpoMOQJKkyWy6xVa1+dY7DDqMCd1+w9rrq2rHmex7\nwJ9sVTfcONk09hM7/8I7VlXVgbMKTpqB5Tssqd133WzQYUzo/AvvmPH9JUnqPxMJkqQFa/Otd2Cv\nJ71y0GFMaPWJr/rJTPe94cb1fGvVA7oqf8lOly3vOiipC7vvuhnfWrXroMOY0JKdLp/x/SVJ6j8T\nCZIk9VgBG9gw6DAkSZLmhYkESZJ6rlhfJhIkSdJoMJEgSVKPNS0SnG5ZkiSNBhMJkiT1gV0bJEnS\nqHD6R0lS3yQ5MMklSS5PcuSg4+mXolhf3S1Stxbr/SVJ6j8TCZKkvkiyBDgGeAKwF3Bokr0GG1X/\nbKC6WqRuLPb7S5LUXyYSJEn9sg9weVVdUVV3Ap8ADhpwTH1RwHqqq0Xq0qK9vyRJ/WciQZLULzsD\nV3Y8X9uuWxRskaAeW9T3lySpvxxsUZLUL5lg3b1qzEmOAI4AWLrV9r2OqS8KHPdAvdb1/fWAnf0Y\nKEmanaFqkZDkx0n+dNBxLFZJzk7yovbxc5J8ZdAxDaMkf5fkQ4OOQxqAtcCuHc93AdaN36mqjq2q\nlVW1ctMttupbcL22octF6lLX99eOy5b0LTipn6wzTG4ur818vq6dZfnZePaSfDDJGwZx7qFKJCwU\nafxTkhva5Z+TTPRNwETHvjnJSX2IcfcklaQnXzdU1ceq6vG9KHtYJdkvydrp9quqf6yqF/UjJmmB\nOQ/YI8kDkywFDgFOG3BMfVFdjo/gGAmahUV7f0kL1TzUGX6d5NYkNyX5vySP6nXMg+Bn43tLcniS\nr023X1X9ZVW9pR8xjWciYRqTVMSPAJ4GPAx4KPBk4MX9jEvDqVeJHWkYVNVdwEuBVcDFwClVddFg\no+qTgvVdLlI3FvX9JS0APaozfLKqtgZ2BL4GfGamiQiNvna2noEZ2kRCkn2SnNtm6K5O8v42A0+S\nY5K8e9z+X0jyivbxiiSfTnJdkh8leVnHfm9OcmqSk5L8Ajh8gtMfBry7qtZW1VXAuyfZbybXUUn+\nMsllSX7exp522+FJvp7k35LcnOQHSfbvOHaj5kXjWjuc0/68qc1k3iuDOdVr2G5/XHvOm5O8n47+\nl+OzZEnel+TKJL9Icn6SP+7YtqRtsvTDJLe023dtt/1ukjOS3Jhm7utndRx3fPt6fKk97ptJfqtj\n+4M7jv1Ze47fSHJ7kmUd+z28/V1vNsFr8OYkn2p/37ck+W6SPZMcleTa9poe37H/C5Jc3O57RZIX\nt+u3Ar4MrGhf71vb99m93k+dv6ckz27L2bZ9/oQk1yTZ8d7vFmn4VdXpVbVnVf1WVb1t0PH0S2HX\nBvXeYr2/pKmMQp2hqn4NnAD8BrAsySZJXp/kJ+3n1ROT3Lcjtqcmuai95rOTPGiS12aTJEe2n9Fv\nSHJKkh06tj+vPccNSf5+qhiTPCnJd9q6wJVJ3jxu+6RlZVyL7faz+TVtHeScJA/u2HafJO9uy7o5\nydeS3Kfd9gdpWm7clOSCJPt1HHd2krekqVvdkuQrSZZ3bH90x7FXpqnrPKKtY2zasd/BSdZM8hoc\nn+Tfk3y5rQt8PU3d5L1p6nk/SPJ7HfuPvfa3JPl+kqe36x8EfBB4VFvOTR3lfyDJ6UluA/6kXffW\ndvvrknxjLN4kf9W+D7aY6nc3W0ObSADWA68ElgOPAvYH/rrddgLN/MmbALRvkv2Bk9t1XwAuoBnN\neH/gFUkO6Cj7IOBUYDvgYxOc+8Ht8WMuaNfN1pOBR9BkK58FdMbySOAKmut8E00mcod7lXBvj2l/\nbldVW1fVuRPsM+lr2L5mnwZe327/IfBHU5zvPGBvYAfg48CnOt60fwscCjwR2BZ4IXB7msr3Ge3+\n92v3+ffOPxbtun8AtgcuB97WxrcN8FXgv4AVwG8DZ1bVNcDZNK/jmOcCn2j/CE/kKcBH23N8h+bb\nnE1o3h9HA//Rse+1NL+vbYEXAP+S5Per6jaaubvXta/31lU11jd10vdTVX0SOBf41zTJjw8DL6qq\n6yaJVdJQCuu7XCRJ82Lo6wxJNqdJQKytquvbx4cDfwL8JrA18P523z2Bk4FX0LRkOB34Qjq+LOzw\nMpoWE/vSfJ7+OXBMW85ewAeA57XbltGMvTKZ24Dn07wWTwL+KsnTZlnWl4E9aOoH32bj1/ZdwMOB\nP6Spd7wW2JBkZ+BLwFvb9a8GPp2Nv5z7c5rP7/cDlrb7kOQB7Tn/jeY12xtYU1XnATcAj+so47k0\n9YbJPIt76k930HzO/3b7/FTgPR37/hD4Y+C+NPWdk5LsVFUXA38JnNvWKbYbdw1vA7ahaaXS6Z3A\nncDrk+wB/CPw3Kr61RTxztrQJhKq6vyq+kZV3VVVP6ap7O3bbvsWcDPNDQ9NP8Gzq+pnNBX2Havq\n6Kq6s6quAP6z3WfMuVX1uaraUFW/nOD0W7flj7kZ2DqZdVOjd1TVTVX1U+AsmjfvmGuB91bVr9tK\n5yU0N+ecTfUa0lT6v19Vp7YV8PcC10xR1klVdUNb1ruBzYHfaTe/CHh9VV1SjQuq6gaaCvmPq+oj\n7XHfpklePLOj6M9U1bfaJpsf457X5snANVX17qr6VVXdUlXfbLedQHOTjzX5OZSpb/j/rapV7Tk+\nRfMH5B3tdX8C2D3Jdu11fqmqfthex/8AX6H5AzCV6d5PLwEeS5MA+UJVfXGa8iQNmQI2VHeLJGnu\nhrzO8Kz22+graSrPT2vXPwd4T1VdUVW3AkcBh7TfRD8b+FJVndF+ln0XcB+aivd4Lwb+vm0xcQfw\nZuCZbTnPBL5YVee0297AFA3mqursqvpu+1pcSJPMGKtXdFvWce1n+7GYHpbkvm1y54XAy6vqqqpa\nX1X/1+73XOD0tmXWhqo6A1hNU6cZ85GqurT9XZ3CPfWK5wBfraqT2zrXDVU11uqgs16xA80Xvh+f\nLHbgs+177lfAZ4FfVdWJVbUe+CRwd4uEqvpUVa1r4/0kcBmwzxRlA3y+qr7eHrNRgqCqNtAkc15G\nM0bOP1fVd6Ypb9aGtr92m217D7AS2JLmWs7v2GXsl35G+/N97frdaJqf39Sx7xLgfzued87DPJFb\nab6RHrMtcGvVrOf26qyg307zR2fMVePK/QlNJm/OpnkNV9DxOlRVJZn0dUnyKpqEwQqaz8zb0mTe\noBlF+ocTHLYb8Mhxv4tN2bjSP9lrM1mZAJ8HPpjkN4E9gZvbfxST+VnH418C17c3+9hz2vPelOQJ\nNC1D9qRJxG0JfHeKsmGa91NV3ZTkUzQtNw6epixJQ8pWBpLUf0NeZzilqp47wfoVNHWCMT+hua77\nj99WVRvaz/A7T1DObsBnk3RW6td3lNNZF7gtyQ2TBZrkkcA7gIfQfNu/Oc0XdGPxzqis9kvAtwF/\nRvPl3lhsy9syt2DyesWfJXlKx7rNaL6kHTObesVJwMVJtqZpbfC/VXX1JPvCvesV45/fXc9L8nya\nz/+7t6u25p7602Smq1f8OMlZNAmUY6Ypa06GtkUCTfOYHwB7VNW2wN+x8RzKJwEHJXkY8CDgc+36\nK4EfVdV2Hcs2VdWZrZru5r6IphvCmIe163ph53FZywdwz3ROt9H8QRzzGx2PZ/IHaqrX8Go6ppFq\nY9j1XiU02/4YeB3NzbV9Nc1vbu4o60rgtyY49Ergf8b9Lrauqr+aQeyTlUmbnTuFJrv4PKZujTBj\nbbOyT9Nkdu/fXufp3HOdk73mU/4ukuxNk109GfjX+YhVkiRJwGjWGdbRVJzHPAC4i6bSutG2js/w\nV01QzpXAE8Zd4xbVjOcwvi6wJU2XhMl8nOZb8F2r6r40ffwnq1dMVdaf03QZ+VOaJv+7jx0GXA/8\nisnrFR8ddy1bVdU7poi589jJ6hVX0XRPeDrzW6/YjaaFy0uBZW294nvMvV7xRJouPGfSdHXomWFO\nJGwD/AK4NcnvAhtVPqtqLU2//Y8Cn+5obvQt4BftYBT3STMQ4EOSPKKLc58I/G2SnZOsAF4FHD+2\nMc0giIfP9sLGuR/wsiSbJfkzmj9wp7fb1tA0Y9osyUo27hJwHU0G7zenKHuq1/BLwIOTPKNt3vQy\nNk5UjC/nrvacmyZ5IxtnXz8EvCXJHmk8tB0P4IvAnmkGX9msXR6RSQaEGeeLwG8keUWSzZNs02ZC\nx5xI03fsqTT/IObDWHb1OuCutnVC5xSYP6MZAOe+Ex08kXYciZNo/qm9gCZx9NdTHyVp2BQ4RoIk\nDcYo1hlOBl6ZZrrXrWn6wn+y7aZ7CvCkJPunGWj8VTR99f9vgnI+CLytrdSSZMckB7XbTgWenGYQ\nwqU044ZNVXfcBrixqn6VZB+ahMCYbsrapo33BpovTP9xbEPbdP844D1pBsJckuRR7Zd9JwFPSXJA\nu36LNFOzTzUWw5iPAX+a5FlJNk2yrP2ib8yJNGMx/D+a7grzYSuajwfXQTOgO01rjjE/A3bJxGNb\nTCjNGB8fpmklfhjN6/HEqY+avWFOJLya5g16C00255MT7HMCzS/87sxR22T9KTR9Yn5Ek9n6EE3G\na6b+g2bwle/SZI6+1K6j/WUvA77R1dVM7ps0g41cT9PM55nVjC8ATf+i36IZGOUf6OivU1W3t/t/\nPc3oo38wQdmTvobVDOTyZzRNlG5oY/j6JDGuohmg5FKaplS/YuNmN++h+aP2FZo/5B8G7lNVt9BU\nxA+hyZ5eA/wTTWV9Su2xj6P5XV5D06foTzq2f50mkfLttj/cnLXnfFl7LT+nee1O69j+A5o/7Fe0\nr/lMuqC8nWbgnA909O96a5oBUiSNkA2VrhZJ0rwYxTrDcW2s57Sx/Qr4mzbuS2g+T/5bG/NTgKdU\n1Z0TlPM+ms+yX0lySxvLI9tyLqIZx+vjNC0Kfg6snSKmvwaObst5I83nZWZR1ok09YmrgO9z79fn\n1TSv53nAjTR1h02q6kqalgx/R1M5vxJ4DTOo71YzTt0TaZIuN9J8WdvZkuSztN1Aqhlgfc6q6vs0\ns3icS5M0+H9sXNf6b5rWK9ckuX6GxR5LM4bC6W198S+AD6VjNrv5lNl361/4kjyGJju1e5vB6sc5\nHw28pKoOnYeyDqcZwf/Rcw5sniV5Ic0ooI8ddCyTSfLfwMer6kODjkXS7Gy1fNfa60mvHHQYE1p9\n4qvOr6qVM9l3r4curZO+OFmjrok9fLcrZ1y+NBsrH7ZFfWvVhL0WB27JTpf7/lffDHudYdglORrY\npapeOOhYJpPkh8CLq+qrg45loRjawRan0zbleTnwoX79QQCoqq9x76k4RtGDaTKgC1Lb7Oz3aTKT\nkjRQRVg/1I0AJWk0WWcYrHYMh73YeJrMBSXJwTTdEP570LEsJCP5qabtY38TsBPNtIWaR0k+BxxI\n0xxnwUlyAvBV4BVtdwRJGrhedG1IcmCSS5JcnuTICbbvluTMJBcmOXusr2iSvZOcm+SidtuzO445\nPsmPkqxpl73HlytJo8A6w4LwbWAXmm4nC06Ss2kG7HxJPxNNw2AkWyRU1cU0A1gMtao6no4BWRaK\nqnra9HsNTlUdNugYJKnT2GCL8ynNFFnH0IwXsxY4L8lpbb/LMe8CTqyqE5I8lmZclufRTHv1/Kq6\nrB3P5fwkq6pqbJqz11TVqfMasCQtMKNSZxhmVfV7g45hKlW136BjWKhGMpEgSdLCEtbXvDcC3Ae4\nvKquAEjyCZruXJ2JhL2AsUEmzqKd1qyqLh3boarWJbmWZr7uzvnSJUmSJjSSXRskSVpICtjAJl0t\nM7AzG8+Qs7Zd1+kC4OD28dOBbcaP3txO07UU+GHH6re1XR7+pZ1WS5Ik6W59bZGwNJvXFnNoPbTn\nQ2+fx2j669ILtxx0CFokfsVt3Fl3OHectMDMomvD8iSrO54fW1XHdjyfqMDxUzG9Gnh/OwvQOTTT\nad11dwHJTjRTiB3W0ffzKJppdZfSTCX1Opo5vyVpYKxHSL3XTT2ir4mELdiKR2b/WR+/atWaeYym\nvw5Y4VhV6o9v1pmDDkHSOFWz6tpw/TTT360FOufu2wVYt/F5ax3wDIAkWwMHV9XN7fNtaeY0f31V\nfaPjmKvbh3ck+QhNMkKSBsp6hNR73dQj7NogSVIfbCBdLTNwHrBHkgcmWQocApzWuUOS5UnG/tcf\nBRzXrl8KfJZmIMZPjTtmp/ZngKcB35vDZUuSpBHkYIuSJPVYM2vD/Obuq+quJC8FVgFLgOOq6qIk\nRwOrq+o0YD/g7UmKpmvDS9rDnwU8BljWdnsAOLyq1gAfS7IjTdeJNcBfzmvgkiRp6M0pkZDkQOB9\nNB9gPlRV75iXqCRJGik9mbWBqjodOH3cujd2PD4VuNc0jlV1EnDSJGU+dp7DlKR7sR4hDbdZf6rp\nmL/6CTTTSx2aZK/5CkySpFHRo1kbJGkoWY+Qht9cPqncPX91Vd0JjM1fLUmSxllf6WqRpBFmPUIa\ncnNJJMxk/mpJkha9Iqxnk64WSRph1iOkITeXMRJmMn81SY4AjgDYAudAlSRJkhY56xHSkJtLImHa\n+asBqupY4FiAbbPDvf5ASJK0GGzowWCLkjSkrEdIQ24un2qmnb9akiTdM/2jXRskCbAeIQ29WbdI\nmGz+6nmLTJKkEVE4gKIkjbEeIQ2/uXRtmHD+akmSdG9O6ShJ97AeIQ23OSUSJEnS9KpgvWMkSJKk\nEWEiQZKkngsbJhykXJIkafj0NZGw50NvZ9WqNbM+/oAVe8/62FXrZn9eSZLmorBFgiTNhfUIaWGx\nRYIkSX3gTAySJGlUmEiQJKnHirDBWRskSdKI8OsRSVJfJNk1yVlJLk5yUZKXDzqmflrPJl0tUjcW\n+/0lSeovWyRIkvrlLuBVVfXtJNsA5yc5o6q+P+jAeq2ADY6RoN5atPeXJKn/TCRIkvqiqq4Grm4f\n35LkYmBnYBFUdMJ6Z21QDy3u+0uS1G8mEiRJfZdkd+D3gG8ONpL+sEWC+mmx3V+SpP4zkSBJ6qsk\nWwOfBl5RVb+YYPsRwBEAS7favs/R9Y4tEtQP3dxfD9jZj4GSpNnx6xFJUt8k2YymkvOxqvrMRPtU\n1bFVtbKqVm66xVb9DbBHqsKG2qSrRepWt/fXjsuW9DdASdLIMBUtSeqLJAE+DFxcVe8ZdDz9tt7k\ngHposd9fkqT+8lONJKlf/gh4HvDYJGva5YmDDkoaEd5fkqS+sUWCJKkvquprsDgHCihgw+K8dPXJ\nYr6/JEn9ZyJBkqSei10bJEnktnN7AAAgAElEQVTSyDCRIElSjzXTP/plsSRJGg0mEiRJ6oP1Dksk\nSZJGxFAlElatWzPrYw9YsffAzi1JWtyK2CJBkgbIeoQ0v4YqkSBJWly2vf+t7P+3Xx90GBNafWJ3\n+2+wRYIkSRoRJhIkSeqxKlhviwRJkjQiTCRIktQHdm2QJEmjwkSCJEk91oyRYNcGSZI0Gmb9qSbJ\nrknOSnJxkouSvHw+A5MkaZSsJ10tkjSqrEdIw28uLRLuAl5VVd9Osg1wfpIzqur78xSbJEkjobBr\ngyR1sB4hDblZJxKq6mrg6vbxLUkuBnYG/AMgSdJG7NogSWOsR0jDb14+1STZHfg94JvzUZ4kSaNm\nA+lqmYkkBya5JMnlSY6cYPtuSc5McmGSs5Ps0q7fO8m5bZPiC5M8u+OYByb5ZpLLknwyydJ5exEk\naRzrEdJwmnMiIcnWwKeBV1TVLybYfkSS1UlWX3fD+rmeTpKkoTM2/WM3y3SSLAGOAZ4A7AUcmmSv\ncbu9Czixqh4KHA28vV1/O/D8qnowcCDw3iTbtdv+CfiXqtoD+DnwF3O8fEmakPUIaXjNKZGQZDOa\nm/9jVfWZifapqmOramVVrdxx2ZK5nE6SpKG1oTbpapmBfYDLq+qKqroT+ARw0Lh99gLObB+fNba9\nqi6tqsvax+uAa4EdkwR4LHBqe8wJwNPmcNmSNCHrEdJwm8usDQE+DFxcVe+Zv5AkSRotzfSP3S0z\nsDNwZcfzte26ThcAB7ePnw5sk2RZ5w5J9gGWAj8ElgE3VdVdU5QpSXNiPUIafnNpkfBHwPOAxyZZ\n0y5PnKe4JEla7JaPNeltlyPGbZ8o21Djnr8a2DfJd4B9gatoRktvCkh2Aj4KvKCqNsywTEmaK+sR\n0pCby6wNX2PiDxySJGmcmQ6g2OH6qlo5xfa1wK4dz3cB1nXu0HZbeAbc3Rf54Kq6uX2+LfAl4PVV\n9Y2xcwLbJdm0bZVwrzIlaa6sR0jDz7moJEnqsYJedG04D9ijnWVhKXAIcFrnDkmWJxn7X38UcFy7\nfinwWZqBGD91d5xVRTOWwjPbVYcBn5/9lUuSpFFkIkGSpD6Y78EW2xYDLwVWARcDp1TVRUmOTvLU\ndrf9gEuSXArcH3hbu/5ZwGOAwzuaFe/dbnsd8LdJLqcZM+HD8/QSSJKkETHrrg3DZtW6NXM6/oAV\ne0+/U4/OP9dzS5IGbOatDLortup04PRx697Y8fhU7pmBoXOfk4CTJinzCpoZISRJWI+QJrJoEgmS\nJA1KMasxEiRJkhYkEwmSJPVBL1okSJIkDYKJBEmSemxssEVJkqRRYCJBkqQ+MJEgSZJGhYkESdKC\nddtPt+S8Vzx80GFM4jMz3rPozWCLkiRJg2AiQZKkPnCwRUmSNCpMJEiS1Gtl1wZJkjQ6TCRIktRj\nDrYoSZJGiYkESZL6wESCJEkaFSYSJEnqMQdblCRJo2STQQcgSVpckixJ8p0kXxx0LP1Ula4WaTYW\n6/0lSeovWyRIkvrt5cDFwLaDDqSfnLVBfbIo7y9JUn/ZIkGS1DdJdgGeBHxo0LH0U7WzNnSzSN1a\nrPeXJKn/TCRIkvrpvcBrgQ2T7ZDkiCSrk6y+89e39S8yafh1dX9dd8P6/kUmSRopJhIkSX2R5MnA\ntVV1/lT7VdWxVbWyqlYu3WyrPkXXe46RoF6azf2147IlfYpOkjRq+jpGwqUXbskBK/bu5ynnzap1\na+Z0/Fyue67nnoth/X1JWpD+CHhqkicCWwDbJjmpqp474Lj6wO4K6rlFfH9pMbAeMZhzz8Ww/r40\nM7ZIkCT1RVUdVVW7VNXuwCHAfy+mSo4tEtRLi/3+kiT1l7M2SJLUYwW2SJAkSSPDRIIkqe+q6mzg\n7AGH0T/VzNwg9cOiu78kSX03564NSZYk+U6SL85HQJIkjaINpKtFkkad9QhpeM1Hi4SXAxcD285D\nWZIkjZwCxz2QpHuzHiENqTm1SEiyC/Ak4EPzE44kSaOombWhm0WSRpn1CGm4zbVrw3uB1wIb5iEW\nSZJGVlV3iySNOOsR0hCbddeGJE8Grq2q85PsN8V+RwBHAGzBlrM9nSRpEVq/+Sb84gFbDDqMeWHX\nBklqWI+Qht9cWiT8EfDUJD8GPgE8NslJ43eqqmOramVVrdyMzedwOkmShlPTyiBdLZI0wqxHSENu\n1omEqjqqqnapqt2BQ4D/rqrnzltkkiSNEMdIkKSG9Qhp+M3HrA2SJGkajnsgSZJGxbwkEqrqbODs\n+ShLkqRRZHcFSbo36xHScLJFgiRJPVY47oEkSRodJhIkSeoDezZIkqRRYSJhhg5Ysfecjl+1bs1Q\nnnsux8LcY5ckSZKGmfWI2bEesbCZSJAkqdfKMRIkSdLoMJEgSVI/2LdBkiSNiE0GHYAkSYtBVbpa\nZiLJgUkuSXJ5kiMn2L5bkjOTXJjk7CS7dGz7ryQ3JfniuGOOT/KjJGvaxbalkiRpIyYSJEnqg6ru\nlukkWQIcAzwB2As4NMle43Z7F3BiVT0UOBp4e8e2dwLPm6T411TV3u0yt06ukiRp5JhIkCSpx4qe\ntEjYB7i8qq6oqjuBTwAHjdtnL+DM9vFZndur6kzgljlfnCRJWnRMJEiS1GsFVLpbprczcGXH87Xt\nuk4XAAe3j58ObJNk2QzKflvbHeJfkmw+k2AkSdLiYSJBkqQ+mEXXhuVJVncsR4wrcqJsw/hOEa8G\n9k3yHWBf4CrgrmlCPQr4XeARwA7A67q8VEmSNOKctUGSpH7oftaG66tq5RTb1wK7djzfBVi30Smr\n1gHPAEiyNXBwVd08ZZhVV7cP70jyEZpkhCRJ0t1skSBJUs91Nz7CDMdIOA/YI8kDkywFDgFO2+is\nyfIkY//rjwKOmzbSZKf2Z4CnAd/r4kIlSdIiYCJBkqR+qC6X6Yqrugt4KbAKuBg4paouSnJ0kqe2\nu+0HXJLkUuD+wNvGjk/yv8CngP2TrE1yQLvpY0m+C3wXWA68dfYXLUmSRpFdGyRJ6rVipq0Muiu2\n6nTg9HHr3tjx+FTg1EmO/eNJ1j92PmOUJEmjx0SCJEn90P0YCZIkSQuSiQRJ0oK1ZNmdbHv42kGH\nMbETuz1g/lskSJIkDYKJBEmS+sEWCZIkaUSYSJAkqR9MJEiSpBFhImEIrFq3Zk7HH7Bi74Gde5Cx\nS9KCUUAPBluUJGkq1iPUK07/KEmSJEmSZsxEgiSpb5Jsl+TUJD9IcnGSRw06pn6p6m6RurWY7y9J\nUn/ZtUGS1E/vA/6rqp6ZZCmw5aAD6huTA+q9xXt/SZL6ak4tEsx8S5JmKsm2wGOADwNU1Z1VddNg\no+qjSneL1IVFf39p6FiPkIbbXLs2jGW+fxd4GHDx3EOSJI2o3wSuAz6S5DtJPpRkq0EH1S+p7hap\nS4v6/tJQsh4hDbFZJxLMfEuSurQp8PvAB6rq94DbgCPH75TkiCSrk6y+8+Zf9jvG3qhZLFJ3ur6/\nrrthfb9jlADrEdIomEuLBDPfkqRurAXWVtU32+en0lR8NlJVx1bVyqpaufS+9+lrgL3TZbcGuzao\ne13fXzsuW9LXAKUO1iOkITeXRELXme9fc8ccTidJGmZVdQ1wZZLfaVftD3x/gCH1ly0S1EOL/v7S\nsLEeIQ25uczaMFHm+15/AKrqWOBYgG2zgx+NJGlx+xvgY+2I8lcALxhwPP3jf0D13uK9vzRsrEdI\nQ27WiYSquibJlUl+p6ouwcy3JGkaVbUGWDnoOAbCj8DqsUV9f2moWI+Qht9cWiSAmW9JkqZXOO6B\nJG3MeoQ0xOaUSDDzLUnSzDiloyTdw3qENNzm2iJBkiTNhIkESZI0IuYya4MkSZIkSVpkbJHQJwes\n2HvWx65at2ZO557L8XOJe67nlqRRYtcGSdJsWI/QQmQiQZK0cK3bFI5ePugo5oeDLUqSpBFhIkGS\npF4rHCNBkiSNDMdIkCRJkiRJM2aLBEmS+sEWCZIkaUSYSJAkqQ8cbFGSJI0KEwmSJPWDiQRJkjQi\nTCRIktQPJhIkSdKIMJEgSVKPpezaIEmSRoeJBEmS+qEy6AgkSZLmhYkESZL6wRYJkiRpRJhIkCSp\nD+zaIEmSRoWJBEmS+sFEgiRJGhEmEiRJ6jUHW5QkSSNkk0EHIEnSolBdLjOQ5MAklyS5PMmRE2zf\nLcmZSS5McnaSXTq2/VeSm5J8cdwxD0zyzSSXJflkkqWzul5JkjSyTCRIktQP85xISLIEOAZ4ArAX\ncGiSvcbt9i7gxKp6KHA08PaObe8EnjdB0f8E/EtV7QH8HPiLGV2fJElaNOzaMAQOWLH3nI5ftW7N\nQI6FwcY+V3ONXZI69aBrwz7A5VV1BUCSTwAHAd/v2Gcv4JXt47OAz41tqKozk+y3UYxJgMcCf96u\nOgF4M/CBeY9ektRzi7keMRfDHHu/2CJBkqSFaXmS1R3LEeO27wxc2fF8bbuu0wXAwe3jpwPbJFk2\nxTmXATdV1V1TlClJkhY5WyRIktQP3bdIuL6qVk6xPTM4y6uB9yc5HDgHuAq4a/xBXZYpSZIWORMJ\nkiT1Wm9mbVgL7NrxfBdg3UanrVoHPAMgydbAwVV18xRlXg9sl2TTtlXCvcqUJEmaUyIhySuBF9F8\nW/Fd4AVV9av5CEySpDt2CJc/e7NBhzGxswcdAOcBeyR5IE1Lg0O4Z2wDAJIsB26sqg3AUcBxUxVY\nVZXkLOCZwCeAw4DP9yB2SYuc9QhpuM16jIQkOwMvA1ZW1UOAJTQfYiRJ0njzPGtD22LgpcAq4GLg\nlKq6KMnRSZ7a7rYfcEmSS4H7A28bOz7J/wKfAvZPsjbJAe2m1wF/m+RymjETPjz7i5ake7MeIQ2/\nuXZt2BS4T5JfA1ti80dJkibWg5EGqup04PRx697Y8fhU4NRJjv3jSdZfQTMjhCT1kvUIaYjNukVC\nVV1FMz/1T4GrgZur6ivzFZgkSaMiNGMkdLNI0qiyHiENv7l0bdieZr7qBwIrgK2SPHeC/Y4Ym7rq\n19wx+0glSRpm89y1QZKGlfUIafjNOpEA/Cnwo6q6rqp+DXwG+MPxO1XVsVW1sqpWbsbmczidJElD\nqsvWCLZIkDTirEdIQ24uiYSfAn+QZMskAfanGexJkqQJJXllkouSfC/JyUm2GHRMfWOLBPXYor6/\nNGysR0hDbi5jJHyTZgCnb9NM2bIJcOw8xSVJGjGLfpRuEwnqoUV/f2moWI+Qht+cZm2oqjcBb5qn\nWCRJo2/RjtJtdwX1waK9vzR8rEdIw20uXRskSZqxRT9Kty0S1EOL/v6SJPWViQRJUl/MZpTu9bfe\n1u8we6PbJIKJBHVpNvfXdTes73eYkqQRYSJBktQvXY/SvWTrrfoeZK84a4N6rOv7a8dlS/oepCRp\nNMxpjAQNhwNW7D2wc69at2ZOx88l9rmee5CxSyPq7lG6gV/SjNK9erAh9ZHJAfXW4r6/JPXEYv08\nu1ivuxsmEiRJfVFV30wyNkr3XcB3WESjdNvKQL202O8vSVJ/mUiQJPXNoh6l20SCemxR31+SpL4y\nkSBJUq85gKIkSRohJhIkSeqxtIskSdIoMJEgSVI/2CJBkiSNCKd/lCRJkiRJM2aLBEmS+sBZGyRJ\n0qgwkSBJWrA2/+nt7PnX3xp0GBP6abcHmEjQAnPphVsu4LnSLx90AJKkKZhIkCSpH0wkSJKkEWEi\nQZKkXiu7NkiSpNFhIkGSpH4wkSBJkkaEiQRJkvrAFgmSJGlUmEiQJKkfTCRIkqQRYSJBkqQ+sEWC\nJEkaFSYStKCtWrdm1sfOdUqruZx7Po6frX0OuH0g55U0hcIWCZIkdWGun6UHOb3toOoB0L/rNpEg\nSVI/mEiQJEkjwkSCJEk9FuzaIEmSRoeJBEmS+sFEgiRJGhEmEiRJ6oOUmQRJkjQaNpluhyTHJbk2\nyfc61u2Q5Iwkl7U/t+9tmJIkDbGaxSJJI8C6hDSapk0kAMcDB45bdyRwZlXtAZzZPpckSZNIdbdI\n0og4HusS0siZNpFQVecAN45bfRBwQvv4BOBp8xyXJEmjxRYJkhYh6xLSaJpJi4SJ3L+qrgZof95v\nsh2THJFkdZLVv+aOWZ5OkiSNl+TAJJckuTzJvb7RS7JbkjOTXJjk7CS7dGw7rG1WfFmSwzrWn92W\nuaZdJv0fL0mzNKO6hPUIaeHq+WCLVXUscCzAttnB71gkSYvSfHdXSLIEOAZ4HLAWOC/JaVX1/Y7d\n3gWcWFUnJHks8HbgeUl2AN4ErKRp/3B+e+zP2+OeU1Wr5zdiSeqO9Qhp4Zpti4SfJdkJoP157fyF\nJEnSCJr/rg37AJdX1RVVdSfwCZrmwp32oul/DHBWx/YDgDOq6sY2eXAG9+7DLEm9Yl1CGnKzTSSc\nBow1gzwM+Pz8hCNJ0gjqcqDFGbZe2Bm4suP52nZdpwuAg9vHTwe2SbJsBsd+pO3W8IYk6eJKJWkm\nrEtIQ27arg1JTgb2A5YnWUvTFPIdwClJ/gL4KfBnvQxSkrQ47fnQ21m1as2gw5jQkp26PKD7RrnL\nk3R2Lzi2beY7ZqIK/vizvBp4f5LDgXOAq4C7pjn2OVV1VZJtgE8DzwNO7Dp6LXgjdX9pwbIuIY2m\naRMJVXXoJJv2n+dYJEkaSWFWYyRcX1Urp9i+Fti14/kuwLrOHapqHfAMgCRbAwdX1c3th/n9xh17\ndnvMVe3PW5J8nKYLhYkESbNiXUIaTbPt2iBJkrpR1d0yvfOAPZI8MMlS4BCa5sJ3S7I8ydj/+qOA\n49rHq4DHJ9k+yfbA44FVSTZNsrw9djPgycD35nztkiRppPR81gZJkjT/szZU1V1JXkqTFFgCHFdV\nFyU5GlhdVafRtDp4e5Ki6drwkvbYG5O8hSYZAXB0u24rmoTCZm2ZXwX+c34jlyRJw85EgnrqgBV7\nz+n4Vetm33dzLsfCYGOXNGJmPhNDd8VWnQ6cPm7dGzsenwqcOsmxx3FPC4WxdbcBD5//SCVJ6s5c\nP4vPhfWI6ZlIkCSpD7Jh0BFIkiTND8dIkCTNqyTHJbk2yfc61u2Q5Iwkl7U/tx9kjANRXS7SBLy/\nJEkLgYkESdJ8Ox44cNy6I4Ezq2oP4Mz2+aKS6m6RJnE83l+SpAEzkSBJmldVdQ5w47jVBwEntI9P\nAJ7W16AGrejFrA1ahLy/JEkLgWMkSJL64f5VdTVAVV2d5H6DDqjfbGWgHlr095ckqb9skSBJWlCS\nHJFkdZLV192wftDhzB/HSNACMLL3lySpr0wkSJL64WdJdgJof1472Y5VdWxVrayqlTsuW9K3AHsp\nOEaCempR31+SpP4zkSBJ6ofTgMPax4cBnx9gLP3X7fgIjpGg7izu+0uS1HcmEiRJ8yrJycC5wO8k\nWZvkL4B3AI9LchnwuPa5pC55f0mSFgIHW5QkzauqOnSSTfv3NZAFxu4Kmg/eX5KkhcBEgiRJ/WAi\nQZIkjQgTCZIk9YEtEiRJ0qgwkSBJUq8VsMFMgiRJGg0mEiRJ6gfzCJIkaUSYSJAkqQ/s2iBJkkaF\niQQtaAes2HvWx65at2ZO557r8YOMXRoVF922Aw8+9zmDDmMS/9Dd7mUmQZKkxWCQ9Yh+MZEgSVIf\n2CJBkiSNChMJkiT1WuEYCZIkaWSYSJAkqccCxK4NkiRpRGwy3Q5JjktybZLvdax7Z5IfJLkwyWeT\nbNfbMCVJGnIbulwkachZj5BG17SJBOB44MBx684AHlJVDwUuBY6a57gkSRopqepqkaQRcDzWI6SR\nNG0ioarOAW4ct+4rVXVX+/QbwC49iE2SpNFQs1gkachZj5BG10xaJEznhcCX56EcSZJGVDXTP3az\nSNLosx4hDak5DbaY5O+Bu4CPTbHPEcARAFuw5VxOJ0nS0HL6R0m6h/UIabjNOpGQ5DDgycD+VZN/\ndVJVxwLHAmybHfwYJUlanGxlIEmA9QhpFMwqkZDkQOB1wL5Vdfv8hiRJkiRpFFmPkEbDTKZ/PBk4\nF/idJGuT/AXwfmAb4Iwka5J8sMdxSpI0vAqyobtFkoad9QhpdE3bIqGqDp1g9Yd7EIskSaPLrg2S\nFhnrEdLomtNgi5IkaYbMI0iSpBFhIkGSpD6ILRIkSdKIMJEg9ciqdWtmfewBK/ae9bGX1g2zPlZS\nD5lIkCRpKMzls/hiYSJBkqReK8ABFCVJ0ogwkSBJUo+FsmuDJEkaGdNO/yhJkuZBVXfLDCQ5MMkl\nSS5PcuQE23dLcmaSC5OcnWSXjm2HJbmsXQ7rWP/wJN9ty/zXJJmX65ckSSPDFgmSpAVr6doN7Hbk\nLwcdxoR+0O0B89wiIckS4BjgccBa4Lwkp1XV9zt2exdwYlWdkOSxwNuB5yXZAXgTsJKm48X57bE/\nBz4AHAF8AzgdOBD48rwGL0mShpotEiRJ6rWxMRK6Waa3D3B5VV1RVXcCnwAOGrfPXsCZ7eOzOrYf\nAJxRVTe2yYMzgAOT7ARsW1XnVlUBJwJP6/p6JUnSSDORIElSH6Sqq2UGdgau7Hi+tl3X6QLg4Pbx\n04Ftkiyb4tid28dTlSlJkhY5EwmSJPVD92MkLE+yumM5YlyJE41dMD4D8Wpg3yTfAfYFrgLumuLY\nmZQpSZIWOcdIkCSp52Y+gGKH66tq5RTb1wK7djzfBVi30Vmr1gHPAEiyNXBwVd2cZC2w37hjz27L\n3GXc+o3KlCRJskWCJEm9VvRi1obzgD2SPDDJUuAQ4LTOHZIsTzL2v/4o4Lj28Srg8Um2T7I98Hhg\nVVVdDdyS5A/a2RqeD3x+ztcvSZJGiokESdK8SnJckmuTfK9j3TuT/KCdhvCzSbYbZIwDMc+DLVbV\nXcBLaZICFwOnVNVFSY5O8tR2t/2AS5JcCtwfeFt77I3AW2iSEecBR7frAP4K+BBwOfBDnLFhQfH+\nkiQtBCYSJEnz7XiaKQM7nQE8pKoeClxK8+34otKDwRapqtOras+q+q2qGksSvLGqTmsfn1pVe7T7\nvKiq7ug49riq+u12+UjH+tVV9ZC2zJe2szdo4Tge7y9J0oCZSJAkzauqOge4cdy6r7TfoAN8g437\n4UuaIe8vSdJCYCJBktRvL2QxNpef/zESpIkszvtLktRXztogSeqbJH9PM/3gx6bY5wjgCIAtNt22\nT5H1WAEbTA6ot7q9vx6wsx8DJUmzY4sESVJfJDkMeDLwnKn63VfVsVW1sqpWLl1yn/4F2FNdtkaw\nRYK6NJv7a8dlS/oXoCRppPQ1FX0LP7/+q3XqT6bYZTlwfb/i8dwDP3dPz79kp8GdewamOfflcyl7\nt7kcLPVCkgOB1wH7VtXtg45nIEwOqEe8v7QYWI/w3Avs/KN67hnXI/qaSKiqHafanmR1Va3sVzye\ne7DnHvT5F+u5pV5LcjLNtIPLk6wF3kQzivzmwBlJAL5RVX85sCAHwUSC5oH3lxYr6xGeeyGdf7Ge\nu5Od4yRJ86qqDp1g9Yf7HshC4hgJmifeX5KkhcBEgiRJPVdQGwYdhCRJ0rxYaImEYz33ojr3oM+/\nWM8taRDs2iBJvbRYP9ct1nMP+vyL9dx3W1CJhKoa2IviuRff+RfruSUNgF0bJKmnFuvnusV67kGf\nf7Geu9OCSiRIkjSybJEgSZJGxCaDDgCaaYuSXJLk8iRH9vncuyY5K8nFSS5K8vI+n39Jku8k+WI/\nz9uee7skpyb5QXv9j+rjuV/Zvt7fS3Jyki16fL7jklyb5Hsd63ZIckaSy9qf2/fx3O9sX/cLk3w2\nyXa9OLekBaSqu0WSNC3rEdYjrEcMph4x8BYJSZYAxwCPA9YC5yU5raq+36cQ7gJeVVXfTrINcH7y\n/7d3PyF2nWUcx7+PjkVTTaFY/zRT2gglkFUrpVoLLoxi1NK4cBFRCdht/bOSFPcSRUShopSgqRha\nQiyYjU2jLrqpUlukWoI1VNuMjSal/kOFGObn4hxhGDLNmeTe99x7+/3AZe493LzPO8N9J/P8eM85\ndaJh/S8AJ4Gtjeqt9S3g0SSfqKqrgC0tilbVNuDzwM4k/6mqI8Be4NAUyx4C7gd+sObYfuBnSQ70\n//Hsp7sPd4vaJ4D7klyoqq/S3bprGrUlzQTDAc2e557Zwoevv2XsaWzg1NgT0Bywj7CPsI8Yr4+Y\nhR0JtwOnkjyf5DzwMLCnVfEkZ5I83T//J91i3NaidlUtAx8DDraot672VuD99LeMSnI+yd8aTmEJ\neFNVLdH94nlpmsWSPA68su7wHuDB/vmDwMdb1U7yWJIL/ctfAMvTqC1pRgRYXd3cQ5J0KfYR9hH2\nESP1EbMQJGwDTq95vUKjBbheVd0E3Ar8slHJbwJfAsb4i/FdwDng+/2WqINVdXWLwkn+BHwdeBE4\nA/w9yWMtaq/z9iRn+jmdAd42whwAPgv8ZKTaklrx1AZJmjT7CPsI+4iR+ohZCBLqIsea/wVVVW8G\nfgR8Mck/GtS7Czib5Klp19rAEvBu4DtJbgX+RbclZ+r6c4j2ANuB64Grq+rTLWrPmqr6Mt22uMNj\nz0XSlBkkSNKk2UeMwz5iBozdR8xCkLAC3LDm9TJT3p6yXlW9gW7xH07ySKOydwJ3V9Uf6bZhfaCq\nftioNnQ/95Uk/09Nj9L9Qmjhg8AfkpxL8l/gEeB9jWqv9ZeqeidA//Vsy+JVtQ+4C/hUYtcgLbZ0\nt3/czEOSdCn2EfYR9hEj9RGzECQ8CdxcVdv7C3XsBY61Kl5VRXd+z8kk32hVN8l9SZaT3ET3Pf88\nSbM0LcmfgdNVtaM/tAtodWGYF4H3VtWW/ue/i+6cstaOAfv65/uAH7cqXFW76S6KcneSf7eqK2kk\ngWR1Uw9J0iXZR9hH2EeMZPQgob9QxL3AcboPwZEkzzacwp3AZ+iSvF/3j482rD+mzwGHq+oZ4Bbg\nKy2K9unlUeBp4Dd0n8MHplmzqh4CngB2VNVKVd0DHAA+VFW/p7va74GGte8H3gKc6D9z351GbUmS\npEVlHzEq+4jXeB9R7s0uiVcAAAKlSURBVKiWJM2qa974jtxx475Lv3EEx5/72lNJbhvy3muWrssd\nWzd3Qefjfz04eHzpcmyta/Oe2jX2NC7qpznq51+SZtjS2BOQJOk1weBekiQtCIMESZKmLYFVr3sg\nSZIWg0GCJEktuCNBkiQtCIMESZIaiDsSJEnSgjBIkCRp6uKOBEmStDAMEiRJmrYAqwYJkiRpMRgk\nSJLUQjy1QZIkLQaDBEmSpixA3JEgSZIWhEGCJEnTlrgjQZIkLQyDBEmSGnBHgiRJWhQGCZIkteCO\nBEmStCAq3o5KkjSjquoc8MKAt74VeHlCZYeOdWOS64YMWFWP9uNuxstJdm/y30iDDVxfk1xbmxlv\n8PqSJLVnkCBJmntV9askt83aWNK8m/R6cH1J0mJ43dgTkCRJkiRJ88MgQZIkSZIkDWaQIElaBA/M\n6FjSvJv0enB9SdIC8BoJkiRJkiRpMHckSJIkSZKkwQwSJElzq6p2V9XvqupUVe2/wrG+V1Vnq+q3\nk5qfNM9cX5KkjRgkSJLmUlW9Hvg28BFgJ/DJqtp5BUMeAnZPYGrS3HN9SZJejUGCJGle3Q6cSvJ8\nkvPAw8Ceyx0syePAK5OanDTnXF+SpA0ZJEiS5tU24PSa1yv9MUlXzvUlSdqQQYIkaV7VRY55KyJp\nMlxfkqQNGSRIkubVCnDDmtfLwEsjzUVaNK4vSdKGDBIkSfPqSeDmqtpeVVcBe4FjI89JWhSuL0nS\nhgwSJElzKckF4F7gOHASOJLk2csdr6oeAp4AdlTVSlXdM5mZSvPH9SVJejWVeLqbJEmSJEkaxh0J\nkiRJkiRpMIMESZIkSZI0mEGCJEmSJEkazCBBkiRJkiQNZpAgSZIkSZIGM0iQJEmSJEmDGSRIkiRJ\nkqTBDBIkSZIkSdJg/wP+etI/JWUBxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7bd5816b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBIAAAE/CAYAAAD7UuCJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3Xm8JHV97//XmwFENmFYhAGULJjE\nGMVkFDU3SkQWlQhZNBo1o4ZLlpurJjGIMQlEr0rMjVF/mmWCyLjhgkZxyR1HDDEmgiwiLqiDipkR\nZF/dkJnP74+qI82xz8zp06erT/d5PR+PepxTVd+q+lR31zn9/dT3+61UFZIkSZIkSfOxw7gDkCRJ\nkiRJk8NEgiRJkiRJmjcTCZIkSZIkad5MJEiSJEmSpHkzkSBJkiRJkubNRIIkSZIkSZo3EwmSpHlJ\ncnWSJ4w7jqUsSSX5yTnW3Znkx7uOSffwMzy3JBckOanrbQc8znOSfHIE+90vyZeT7LLY+x5Gkqck\neee445CkfkwkSJImSpJfTvJvSW5LcvWA2z4nyZa2Un97ks8mOX5Eod5LVe1eVV/r4lha2hb5M3x5\nV5/hKXYq8Oaq+h5Akqcl+a8k30lywezCSQ5Pcmm7/tIkh/cp85UkD9rWe51k/yTnJLmmXf+fSY6Y\nWV9V5wEPSfLQxT5hSRqWiQRJ0pKVZMc+i78NnAX86QJ3+6mq2h3YC/h74J1J9lrgvqRt6uAz/Cbg\n3UlWLnBfy1qS+wBrgLf1LL4ZeC1wRp/yOwMfaMvvDawDPtAunynzE8AOVfUVtv1e7w5cDPwCsLLd\n14eT7N5T5hzg5IWenySNiokESdLAkjwyyaeS3Jrk2iRvmPkineSNSf52VvkPJnlh+/uqJO9NckOS\nryd5fk+505Ocm+RtSW4HnjP72FX16ap6KzDU3f2q2gq8FdgNOKwnhvck+VZ7h/ATSX62Z93Z7fl9\nOMkdSS5qKw39XqP/kWRTkl9u53/Y7WF7+0lyTNvU+rYkf5/k37toOr6cTNFn+CzgvsCPt8f/n0mu\nSnJzkvOSrOqJ7TFJLm4/Vxcnecw2Xp/nJbkyyS1J1id5YM+6o5N8qd3PG4BsYz+nJ3l3kre0n/Uv\nJFnds/5n0nSNuLVd95Sedfu053B7kk8DPzFr3z+dZEN7rl9O8rSedU9K8sX2mN9M8qI5QjwCuLWq\nNve8rh+rqncD1/QpfySwI/Daqvp+Vb2+Pf/H95R5MvCRdl9zvtdV9bWqek1VXVtVW6pqLbAz8FM9\nxS5o9ydJS4qJBEnSQmwB/gjYF3g0cBTwB+26dcAzkuwAkGTfdv057bIPAp8FDmqXvzDJsT37PgE4\nl+Zu69tHdQJJVgDPBX4AfKNn1b/SJBb2By7rE8MzgL+iuRt5FfCKPvs+luZO4q9X1b/NEULf/bSv\n17nAS4B9gC8Dc1b4tGDT8BneETgJuBPYmOTxwKuApwEH0nyu39mWXQl8GHg9zefqNTR3v/fps98T\ngT8Dfg3YD/gPms/zzGvxXuDPaV67rwK/uJ1Qn9LGsRdwHvCGdl870byWH6W53v438PYkMxXpNwLf\na8/lee00E+NuwAbgHe22zwD+vifx9ybgd6tqD+AhwMfniO3naK6x+fpZ4Iqqqp5lV7TLZzyJ5rUe\nSJouEjvT/D2YcSVwaJI9B92fJI2SiQRJ0sCq6tKqurCq7q6qq4F/Ah7Xrvs0cBtNBQvg6cAFVXUd\n8Ahgv6p6WVXd1Y4Z8M9tmRmfqqr3V9XWqvruCMJ/VJJbaSoo/xd4VlVd33NuZ1XVHVX1feB04GFJ\n7tez/fvau4x301QSZ/ePfiqwFnhS+1rMZa79PAn4QlW9r133euBbCz5b9TUln+Fv0VSgf7WqbgOe\nCZxVVZe1n9+XAI9OcijNXe2NVfXW9pzPAb4E/Eqf/f8u8KqqurL9DL4SOLxtlfAk4ItVdW5V/YCm\nC8D2Pp+frKqPVNUWmlZAD5s5D5rm/We0r+XHgQ/RJHFWAL8O/GVVfbuqPk+T4JlxPHB1Vb25PZ/L\naBIcv9Gu/wHw4CR7VtUt7fp+9gLu2E78vXan+Wz0ug3YAyDJrjSfkX8fYJ+0iYK3An/VvpczZmKz\n+5WkJcVEgiRpYGkGEftQ2wXgdpqKxr49RdYBz2p/fxbNF2SABwKr2mbMt7aVoT8D7t+z7aYRh39h\nVe1F0xLgPOCXZlYkWZHkjCRfbc/r6nZV77n1Vpq+Q1Ox6PVC4N1V9bntxDHXflbR8xq0dz43o0U1\nDZ/hqtq3qh5VVR9rl6+ip3VNVd0J3ETTcuJe61rfaNfN9kDgdT3ndzNN8/2Z/cz+fG7vfGd/1ndp\nW1OsAja1XTRmx7QfTReCTbPW9cZ4xKz34ZnAAe36X6dJenyj7Rr06Dliu4U2CTBPdwKzWwfsyT0V\n/qOA/5oZuHE+ktyXpmXGhVX1qlmrZ2K7dYAYJWnkTCRIkhbiH2juZh5WVXvSVKR6+0m/DTghycOA\nnwHe3y7fBHy9rQTNTHtU1ZN6tu1tMjwybSXrD4BnJ3l4u/i3aJqlPwG4H3Bou3zOPuB9PBU4caY/\n/QJcCxw8M5MkvfNaNBP/Ge7jGpoKNvDD5v/7AN+cva71gHbdbJtougX0nuN9q+q/aD6fh/QcI73z\nC4j3kJkuJLNiugG4e9a+HzArxn+fFePuVfX7AFV1cVWdQNPt4f3Au+eI4QrgQQPE/AXgoe15z3ho\nuxwG7NaQZrDH99Oc8+/2KfIzNC0vbh8gRkkaORMJkqSF2AO4HbgzyU8Dv9+7sh247GKau7jv7Wne\n/Wng9iQvTnLftgXAQ5I8Yr4HTrJDmue979TMZpfce8T0C5KcPp99VdVNwJnAX/ac1/dp7uLuSnOX\nelDX0NyVfH6SP9he4T4+DPxckhPbu7b/i3vusmrxTMVneJZ3AM9N83jC+9B8fi9qu258BHhQkt9K\nsmOS3wQeTNOVYLZ/BF4yM95AkvsleWq77sPAzyb5tfbz+XwW/vm8iOapBqck2SnJkTRdLd7ZdoN4\nH3B6kl2TPJjm6QozPtSez7PbbXdK8og0gzfunOSZSe7Xdr+4nWZMjH4+DeyV5IctM9r3dBeaFhE7\ntO/PTu3qC9p9PT/JfZL8Ybt8ZgyGJ9IOtNjua873ut3nucB3gd+e1TJjxuNoxm2RpCXFRIIkaSFe\nRHP3/g6a/uHv6lNmHc1AZjNNwmkrB79CMx7A14EbaSry9+uz/VweS/PF+yM0dyi/SzNY24xDgP8c\nYH+vBZ6U5lntb6FpPv1N4IvAhQPs54eq6r9pkgkvzoBPW6iqG2laNbyaJqHxYOASmgSHFs80fYZn\nYjsf+AuasQKupXnKwdPbdTfRjCvwJzSfq1OA49vP2+z9/Avw1zSPRr0d+DxNBbn383lGu5/DFhJr\nu6+7aAZifCLN6/j3NBXqL7VF/pCmy8+3gLOBN/dsewdwTHt+17Rl/hq4T1vk2cDVbfy/xz3dVPrF\ncPas9c+meU/+gabr03dpPiMz5U8Efpumu8HzgBOr6q4kDwHubK//Gdt6rx9D854cA9ya5M52+qWe\n7Z9BM36HJC0pufegs5IkLY4kj6VpHn7oHHfaRnHMg4H3VNVc/aEnTtvsezPwzJr7CRAaAT/Dy0OS\nmSdTPHyYwTGTnALsW1WnLFJcvwI8u6qett3CktQxEwmSpEXXNtl9J/DZqnrZuOOZNGkeJXgRzd3L\nP6Xp3vDjI3oCgPrwM6xBJXka8LmqunLcsUjSqNm1QZK0qJL8DE2T3wNpug1ocI8GvkrT3PtXaJpO\nm0ToiJ9hLURVvdskgqTlwhYJkiRJkiRp3myRIEmSJEmS5s1EgiRJkiRJmrcdxx2AJElz2WWvXWr3\nA3cfdxh93fSlm26sqv3mU/bYX96tbrp5rsfY93fpFd9fX1XHLSg4aR72XbmiDj1kp3GH0delV3x/\n3teXJKl7JhIkSUvW7gfuzpPXPWXcYfT1liPe/I35lr3p5i18ev0DBtr/igM37jtwUNIADj1kJz69\n/pBxh9HXigOvmvf1JUnqnokESZJGrICtbB13GJIkSYvCRIIkSSNXbCkTCZIkaTqYSJAkacSaFgk+\nblmSJE0HEwmSJHXArg2SJGla+PhHSVJnkhyX5MtJrkpy6rjj6UpRbKnBJmlQy/X6kiR1z0SCJKkT\nSVYAbwSeCDwYeEaSB483qu5spQaapEEs9+tLktQtEwmSpK48Eriqqr5WVXcB7wROGHNMnShgCzXQ\nJA1o2V5fkqTumUiQJHXlIGBTz/zmdtmyYIsEjdiyvr4kSd1ysEVJUlfSZ9mP1JiTnAycDLDbAbuN\nOqZOFDjugUZt4OvrAQf5NVCStDAT1SIhydVJnjDuOJarJGcn+T/t77+U5MvjjmkSJXlmko+OOw5p\nDDYDh/TMHwxcM7tQVa2tqtVVtXqXvXbpLLhR2zrgJA1o4Otrv31WdBac1CXrDHNLckGSk7redlv7\n8rvxwiX5syRnjuPYE5VIWCqS/HKSf0tyW5KrB9z2OUk+OaLQZh+rkvzkKPZdVf9RVT81in1PqiSH\ntq/5Nm/xVNXbq+qYruKSlpCLgcOS/FiSnYGnA+eNOaZO1IDjIzhGghZg2V5f0lK1CHWGLUnuTHJ7\nksuTHD+iUMfK78Y/KsmRSTZvr1xVvbKqFiW5MygTCdsxR6Xw28BZwJ92HI4m3PaSDNI0q6q7gT8E\n1gNXAu+uqi+MN6qOFGwZcJIGsayvL2kJGFGd4VNVtTuwF/Am4N1JVi5wX5oy465XTGwiIckjk3wq\nya1Jrk3yhjYDT5I3JvnbWeU/mOSF7e+rkrw3yQ1Jvp7k+T3lTk9ybpK3JbkdeM7sY1fVp6vqrcDX\nFuE8rk7yoiRXtNnKdyXZpV13ZJLNbZOVG9uyz+zZ9l7Ni3pbOyT5RLv4s20m8zf7HPsnknw8yU3t\n/t+eZK+e9Q9PclmSO5K8C9ilZ929smRJTk3y1bbsF5P86qxj/c8kV/as//l2+fbei3cneUu73ReS\nrO5Zf0iS97Xb3tR+Bu6T5OYkP9dTbv8k302yX5/X4DlJ/jPJ37Wfpa8leUy7fFOS65Os6Sn/5CSf\naTPDm5Kc3rO7mdf81vY1f/Ss/d8MnD7rfXpM+9of0s4/rI3jp2fHKk2DqvpIVT2oqn6iql4x7ni6\nUti1QaO3XK8vaVumoc5QVVtpEhL3BX68Pf7/THJV+733vCSremJ7TJKL27rFxUkes43X53ntd/Rb\nkqxP8sCedUcn+VK7nzfQfyyWmbJzvs7b21dmtdhO8rr2e/btSS5N8ks961akqRvN1Dsu7fke/dNJ\nNrSvyZeTPK1nu7Pb9/vD7XYXJfmJnvU/27Ptde0xDkjynST79JT7hfbzsFOf1+D0JO9pPxN3JPlc\nkgcleUlbp9iU5Jie8s/NPfWjryX53Xb5bsC/AqvaOsWd7WfxRz5z7bK3tdv9ZrufPdv5Jyb5VvrU\ngRbDxCYSgC3AHwH7Ao8GjgL+oF23jub5yTsAJNm3XX9Ou+yDwGdpRjM+CnhhkmN79n0CcC5N9u/t\noz8VngYcB/wY8FDu/YfoAJpzPAhYA6xNst0uBVX12PbXh1XV7lX1rj7FArwKWAX8DE3fytMB2gv/\n/cBbgZXAe4Bf38Yhvwr8EnA/4K+AtyU5sN3XU9v9/jawJ/AU4KZ5vhdPoXmE1V40TTTf0O5zBfAh\n4BvAoe3276yq77fln9Wzj2cAH6uqG+aI/QjgCmAf4B3t9o8AfrLdzxuS7N6W/XZ7HnsBTwZ+P8mJ\n7bqZ13yv9jX/VM/+vwbsD9zri11V/RfwT8C6JPeleb3/vKq+NEeskiZS2DLgJElaFBNfZ0hz5/kk\n4E5gY5LH03yHfxpwIM334Xe2ZVcCHwZeT/Pd9jXAh3srwz37PRH4M+DXgP2A/wDOadftC7wX+HOa\n1+6rwC9uI8w5X+cF7Oti4HCaOsg7gPekvdEK/DHNd/sn0dQrngd8p618b2jL79+W+fskP9uz32fQ\n1FP2Bq6i/V6eZA/gY8D/o6kX/SRwflV9C7iA5nWe8SyaOscP5oj9V2i+z+8NfIamldgONJ+hl9F8\n759xPXB8ex7PBf4uyc9X1beBJwLXtHWK3atqZsybOT9zbX3vU8Dr2/f7TcBJ26gDDWViEwlVdWlV\nXVhVd1fV1TRvyuPadZ8GbqP5AEPTT/CCqrqOpoK4X1W9rKruqqqvAf/clpnxqap6f1VtrarvdnA6\nr6+qa6rqZpo/WIfPWv8XVfX9qvp3mj8MT/uRPSxAVV1VVRvafd9A84fmce3qRwE7Aa+tqh9U1bk0\nF/Vc+3pPew5b2w/xRppnWkPzh+/VVXVxNa6qqm8wv/fik+0dli00F+XD2uWPpLnQ/7Sqvl1V36uq\nmUzmOuC3Zv4pAM9ut53L16vqze0x3kWTUHlZ+7p8FLiL5g8KVXVBVX2uPc8raP7YPm7OPTeuqar/\nr/2s9vs8nU6TgPk0zcBYb9zO/iRNmAK21mCTJGl4E15neFSSW4Fv0VSCf7WqbgOeCZxVVZe1N9Fe\nAjw6yaE0N7o2VtVb23M+B/gSTQV3tt8FXlVVV7bdo14JHN62SngS8MWqOretNL+2jaOvbb3OC9jX\n26rqpnZffwvcB5i5kXoSzU23L7f1is9W1U00FfKr2+/0d1fVZTTJi9/o2fX72lYid9NUwmfqXMcD\n36qqv23rFHdU1UXtunW0NyjbG5nPYNv1iv+oqvXtMd5Dk6A5oz3vdwKHpm0BXlUfrqqvtufx78BH\naW7Mbsv2PnP/C3g8TQLkg1X1oe3sb8Emtr92kgfRVHxXA7vSnMulPUVm3vQN7c/XtcsfSNNM5Nae\nsitoMnAzep/D3IXeC+k7NBXkGbe0WakZ35i1fsGS7E+TrfwlYA+axNIt7epVwDer7vW8sm9sY1+/\nTZMhPLRdtDtNxhGaivlX+2w2n/di9muzS5uVPQT4RnuR3ktVXZTk28DjklxLkwTY1oBT1/X8/t12\nH7OX7Q6Q5AjgDOAhwM40f9jes419w3Y+T1X1gyRn07wXfzzrNZc0JWxlIEndm/A6w4VV9T/6LF8F\nXDYzU1V3JrmJ5q73Kn70O/s32nWzPRB4Xe7dvSM9+/nh+VVVJZnzfLfzOg+6rz+hSRisosnF78n8\n6hVHzHq/duTelf7Z9YqZFsdz7RPgA8A/Jvlx4EHAbW0Cai6z6xA3tjcrZ+Zpj3trkicCp7X73YHm\ndfvcNvYN269X3JrkPTT1sm21Jh/axLZIAP6BJrt2WFXtSdMsp/db2tuAE5I8jKbZ/vvb5Zto7kDv\n1TPtUVVP6tl2KVXk9m6b6sx4APc8zunbNB+4GQcMuO9X0ZzrQ9vX8Fnc8xpeCxyUpPc1fUC/nbRZ\ny3+mGeRpn6raC/h8z742AT/RZ9P5vBdz2QQ8IHMPMjLzT+HZwLlV9b157HM+3kGTlDikqu4H/CP3\nnOdcn5ttfp6SHETzR+TNwN8muc8ixSpJkrTcTWOd4RqaijPwwz71+wDfnL2u9YB23WybgN+ddY73\nrabr7bX0PFK2rRMc0mcfM7b1Os97X2nGQ3gxTQvsvdt6xW3Mr17x77POZfeq+v1txNy7bb990tYh\n3k3TCmR7rZznrf2+/17g/wL3b8/zIwxfrzicprvHOTQ3KUdmkhMJewC3A3emGZjuXh+SqtpM0xT/\nrcB7e5p+fBq4PcmLk9w3zYAdD0nyiPkeOMkObT+dnZrZ7JJ7DyZyQe49CN+w/irJzu2FdTz33AG/\nHPi1JLumeczj78za7jraAVnmsAdNX6tb28ps74iynwLuBp6fZMckv8Y9XRVm243mQ30DNAOH0Nyx\nn3Em8KI0g5MkyU+2yYdh3otP0/xROiPJbu170NvX6q3Ar9IkE94yj/3N1x7AzVX1vSSPBH6rZ90N\nNGOkbes1v5f2D+nZNH2YfofmnF6+aNFKWhIKHCNBksZjGusM7wCem+TwtkL6SuCitkvBR4AHJfmt\n9jv8bwIPphlbbLZ/BF4yM45AkvulGdsMmu7UP5vk19obd89n2zctt/U6D7KvPWjqIDcAOyb5S5oW\nCTPOBF6e5LC2XvHQdjyAD7Xn/ewkO7XTI5L8zDZinvEh4IAkL0wzcPsebSvkGW+hGcPuKTSJp8Uw\n07L5BuDutnVC7yMwrwP2SXK/+e6w/ay9jSaJ81yam8J/sO2tFm6SEwkvoqnE3UFzN7zfYILrgJ+j\nJ3PUNi35FZo+MV8HbqT5QM77TaIZVO+7NBfqA9rfP9qz/hDgPwfY37Z8i6a7wTU0fXl+r+4ZiO/v\naPrvX0dzrrMHeTmdZhC/W9MzammPvwJ+nibL92HgfTMrquoumoFXntMe/zd71/eqqi8Cf0uTfLiO\n5jX/z57176EZzOQdNO/X+4GVw7wXPdv+JPDfwOY2xpn1m2mafBX3boI2rD8AXpbkDuAvaTKUM8f8\nDs15/mf7mj9qHvt7PnB/mnEwiuaif256RqeVNB22VgaaJEmLYurqDFV1PvAXNHe0r6W5m/70dt3M\neAF/AtwEnAIcX1U39tnPvwB/DbwzzVMAPk8zyB9t+afSdOm9CThsO7HO+ToPuK/1NE8s+ApNl4zv\nce/m/K+h+f79UZrExZuA+1bVHTQV8afT1Ju+1Z7bdlv6ttseTfN+f4tmrLdf7ln/nzQ3Cy9rkzVD\na4/5/PZcbqF57c7rWf8lmlYFX2vrFfPp2v4qYHNV/UM1Y2c8C/g/SQ5bjJhnyzR3x07yWJqszKHV\nPDali2MeDLynqh69CPs6EnhbVR08dGCLLM1osWdW1bzvvnctyVk0Ax3++bhjkbQw+/7MvvXkdU8Z\ndxh9veWIN19aVau3XxIe/NCd620fGqz32S88cNO89y8txOqH7VKfXr+tlsrjs+LAq/z8qzOTXmeY\ndEmeBzyrqh4/7ljmkuTjwDuq6sxxx7JUTOxgi9uT5tmeL6Cp7Hb2SO72Tvhy+IPwEJrs7JKUZtTa\nXwMePt5IJAmKsGWiGwFK0nSyzrAk/CxLu17xCJpW3CeMO5alZCoTCW1fmEtonvv63DGHM3WSvI6m\nj9CaccfST5KX0zzH9lVVtWT/KElaXuyuIElLi3WG8UvyfpquDk/dXtlxSLIOOBF4QdsdQa2pTCRU\n1ZU0AwBOtKq6AFhy3Rqq6gU0mdslqar+gqbfmCQtCTODLUqSlo5pqTNMsqo6cdwxbEtVLckbp0vB\nVCYSJElaWsKWsmuDJEmaDn6rkSRpxArYyg4DTcNIsjLJhiQb2597z1FuTVtmY5I1Pct/IcnnklyV\n5PXto2pJ8rAkn2rXfTDJnv32K0mSplunLRJ2zn1qlyFaDz3ood9ZxGi69ZUrdh13CFomvse3uau+\nbxtqaYnpuGvDqcD5VXVGklPb+Rf3FkiyEjgNWE2T67g0yXlVdQvwD8DJwIU0jy07juZxXGcCL6qq\nf29H2f5T7EomqQPWI6TRG6Qe0WkiYRd244gcteDt16+/fBGj6daxqw4fdwhaJi6q88cdgqRZqjrv\n2nACcGT7+zrgAmYlEoBjgQ1VdTNAkg3AcUkuAPasqk+1y99CM9DUvwI/BXyi3X4DzfO+TSRIGjnr\nEdLoDVKPsGuDJEkd2EoGmoZ0/6q6FqD9uX+fMgcBm3rmN7fLDmp/n70c4PM0T+2BZoTtQ4YNVJIk\nTR4HW5QkacSapzYMnLvfN8klPfNrq2rtzEySjwEH9NnupfPcf79sRW1jOcDzgNcn+UvgPOCueR5L\nkiRNkaESCUmOA14HrADOrKozFiUqSZKmyoK6NtxYVavnWllVT5jzaMl1SQ6sqmuTHAhc36fYZu7p\n/gDN44YvaJcfPGv5Ne0xvwQc0x7jQcCT53UmkjSL9Qhpsi24a0OSFcAbgScCDwaekeTBixWYJEnT\nouunNtC0Fph5CsMa4AN9yqwHjkmyd/tUh2OA9W1XiDuSPKp9WsNvz2yfZP/25w7AnwP/OGygkpYf\n6xHS5Bvmm8ojgauq6mtVdRfwTprBnSRJ0ixbKgNNQzoDODrJRuDodp4kq5OcCdAOsvhy4OJ2etnM\nwIvA79M8oeEq4Ks0Ay1C82X/K8CXaFopvHnYQCUtS9YjpAk3TNeGfoM0HTFcOJIkTZ8iCxkjYeHH\nq7oJ+JHhzavqEuCknvmzgLPmKPeQPstfR9MUWZKGYT1CmnDDJBK2NRjTPYWSk2meRc0u+AxUSZIk\naZmzHiFNuGESCZu592OffjgYU692hOm1AHtm5Y/8gZAkaTnYOvhgi5I0raxHSBNumG81FwOHJfmx\nJDsDT6cZ3EmSJPWYefzjIJMkTTHrEdKEW3CLhKq6O8kf0oz6vAI4q6q+sGiRSZI0JYpFGUBRkqaC\n9Qhp8g3TtYGq+gjwkUWKRZKkqbUIj3SUpKlhPUKabEMlEiRJ0vZVwRbHSJAkSVPCRIIkSSMXtvYd\npFySJGnydJpIeNBDv8P69ZcvePtjVx2+4G3XX7Pw40qSNIzCFgmSNAzrEdLSYosESZI64JMYJEnS\ntDCRIEnSiBVhq09tkCRJU8LbI5KkTiQ5JMm/JbkyyReSvGDcMXVpCzsMNEmDWO7XlySpW7ZIkCR1\n5W7gT6rqsiR7AJcm2VBVXxx3YKNWwFbHSNBoLdvrS5LUPRMJkqROVNW1wLXt73ckuRI4CFgGFZ2w\nxac2aISW9/UlSeqaiQRJUueSHAo8HLhovJF0wxYJ6tJyu74kSd0zkSBJ6lSS3YH3Ai+sqtv7rD8Z\nOBlgtwN26zi60bFFgrowyPX1gIP8GihJWhhvj0iSOpNkJ5pKztur6n39ylTV2qpaXVWrd9lrl24D\nHJGqsLV2GGiSBjXo9bXfPiu6DVCSNDVMRUuSOpEkwJuAK6vqNeOOp2tbTA5ohJb79SVJ6pbfaiRJ\nXflF4NnA45Nc3k5PGndQ0pTw+pIkdcYWCZKkTlTVJ2F5DhRQwNbleerqyHK+viRJ3TORIEnSyMWu\nDZIkaWqYSJAkacSaxz96s1iSJE0HEwmSJHVgi8MSSZKkKTFRiYT111y+4G2PXXX42I4tSVreitgi\nQZLGyHqEtLgmKpEgSVpevr9lR75++z7jDmNRbLVFgiRJmhJ+q5EkacSqYEtloGkYSVYm2ZBkY/tz\n7znKrWnLbEyypmf5K5JsSnKKzMreAAAgAElEQVTnrPL3SfKuJFcluSjJoUMFKkmSJpKJBEmSOrC1\nMtA0pFOB86vqMOD8dv5ekqwETgOOAB4JnNaTcPhgu2y23wFuqaqfBP4O+OthA5UkSZPHRIIkSSPW\njJGww0DTkE4A1rW/rwNO7FPmWGBDVd1cVbcAG4DjAKrqwqq6djv7PRc4KomDP0iStMws+JtKkkOS\n/FuSK5N8IckLFjMwSZKmyRYy0DSk+88kAtqf+/cpcxCwqWd+c7tsW364TVXdDdwGTMcgFpI6Yz1C\nmnzDDLZ4N/AnVXVZkj2AS5NsqKovLlJskiRNhYKFdFfYN8klPfNrq2rtzEySjwEH9NnupfPcf7+A\nagTbSNJs1iOkCbfgREJ7h2PmbscdSa6kuVPhHwBJku4lC+mucGNVrZ5rZVU9Yc6jJdclObCqrk1y\nIHB9n2KbgSN75g8GLthOTJuBQ4DNSXYE7gfcvJ1tJOlerEdIk29RxkhoR21+OHDRYuxPkqRps5UM\nNA3pPGDmKQxrgA/0KbMeOCbJ3u0gi8e0y+a7398APl5VtkiQtGDWI6TJNHQiIcnuwHuBF1bV7X3W\nn5zkkiSX3HDTlmEPJ0nSxOn68Y/AGcDRSTYCR7fzJFmd5MwmproZeDlwcTu9rF1Gklcn2QzsmmRz\nktPb/b4J2CfJVcAf0+dpEJI0X9YjpMk1zBgJJNmJ5uJ/e1W9r1+Ztj/nWoDVD9vFuxaSpGVpEZ7E\nMG9VdRNwVJ/llwAn9cyfBZzVp9wpwCl9ln8PeOqiBitpWbIeIU22BScS2sc9vQm4sqpes3ghSZI0\nXZrHP/qUREkC6xHSNBjm9sgvAs8GHp/k8nZ60iLFJUmSJGk6WY+QJtwwT234JP0fAyVJkmZZhAEU\nJWkqWI+QJt9QYyRIkqTtK7BrgyRJmhomEiRJ6kCXgy1KkiSN0rJJJKy/5vKhtj921eFjO/6wx5Yk\njVk52KIkTSrrEdKPWjaJBEmSxqVwjARJkjQ9TCRIktQBWyRIkqRpYSJBkqQRc7BFSZI0TUwkSJLU\nARMJkiRpWphIkCQtWXfdtSPf2LTvuMMYWuFgi5IkaXqYSJAkqQMOtihJkqaFiQRJkkat7NogSZKm\nh4kESZJGzMEWJUnSNDGRIElSB0wkSJKkaWEiQZKkEXOwRUmSNE12GHcAkqTlJcmKJJ9J8qFxx9Kl\nqgw0SQuxXK8vSVK3bJEgSeraC4ArgT3HHUiXfGqDOrIsry9JUrdskSBJ6kySg4EnA2eOO5YuVfvU\nhkEmaVDL9fqSJHXPRIIkqUuvBU4Bts5VIMnJSS5JcsmWO7/dXWTS5Bvo+rrhpi3dRSZJmiomEiRJ\nnUhyPHB9VV26rXJVtbaqVlfV6hW779ZRdKPnGAkapYVcX/vts6Kj6CRJ06bTMRK+csWuHLvq8C4P\nuWjWX3P5UNsPc97DHnsYk/p+SVqSfhF4SpInAbsAeyZ5W1U9a8xxdcDuChq5ZXx9aTmwHjGeYw9j\nUt8vzY8tEiRJnaiql1TVwVV1KPB04OPLqZJjiwSN0nK/viRJ3fKpDZIkjViBLRIkSdLUsEWCJKlz\nVXVBVR0/7jg6U82TGwaZhpFkZZINSTa2P/eeo9yatszGJGt6lr8iyaYkd84q/9gklyW5O8lvDBel\nRmXZXV+SpM4NnUhIsiLJZ5J8aDECkiRpGm0lA01DOhU4v6oOA85v5+8lyUrgNOAI4JHAaT0Jhw+2\ny2b7b+A5wDuGDVCSrEdIk2sxWiS8ALhyEfYjSdJUKjofI+EEYF37+zrgxD5ljgU2VNXNVXULsAE4\nDqCqLqyqa3/kPKqurqor2MbjBSVpANYjpAk1VCIhycHAk4EzFyccSZKmUfPUhkGmId1/JhHQ/ty/\nT5mDgE0985vbZZI0ctYjpMk27GCLrwVOAfZYhFgkSZpaCxj3YN8kl/TMr62qtTMzST4GHNBnu5fO\nc//9shVDjs4gSfNmPUKaYAtOJCQ5Hri+qi5NcuQ2yp0MnAywC7su9HCSpGXo5+53I59+4tK8WbVi\nwPIL6K5wY1Wtnnt/9YS51iW5LsmBVXVtkgOB6/sU2wwc2TN/MHDBoEFK0qCsR0iTb5iuDb8IPCXJ\n1cA7gccnedvsQlW1tqpWV9XqnbjPEIeTJGkyNU9i6HSMhPOAmacwrAE+0KfMeuCYJHu3gywe0y6T\npFGzHiFNuAUnEqrqJVV1cFUdCjwd+HhVPWvRIpMkaYp0PEbCGcDRSTYCR7fzJFmd5EyAqroZeDlw\ncTu9rF1Gklcn2QzsmmRzktPb5Y9olz8V+KckXxg2UEnLj/UIafINO0aCJEmahwWMkTDEseom4Kg+\nyy8BTuqZPws4q0+5U2j6Ls9efjFNFwhJkrSMLUoioaouwH6VkiTNaRG6K0jS1LEeIU0mWyRIkjRi\nxaKMeyBJkrQkmEiQJKkDPldRkiRNCxMJ83TsqsOH2n79NZdP5LGH2RaGj12SJEmaZNYjFsZ6xNJm\nIkGSpFErx0iQJEnTw0SCJEldsG+DJEmaEiYSJEnqgC0SJEnStDCRIElSB8oWCZIkaUqYSJAkacQK\nWyRIkqTpYSJBkqRRK8BEgiRJmhImEiRJ6oBdGyRJ0rQwkSBJUhdMJEiSpClhIkGSpJGLYyRIkqSp\nYSJBkqQu2CJBkiRNCRMJkiSNWvnUBkmSND1MJEiS1AVbJEiSpClhIkGStGRd+Z29ecRlTxt3GHN4\n5YDlbZEgSZKmg4kESZK6YIsESZI0JUwkSJLUBRMJkiRpSphImADrr7l8qO2PXXX42I49ztglacko\nwMEWJUkdm+R6hJa2HcYdgCRJkiRJmhwmEiRJnUmyV5Jzk3wpyZVJHj3umLpSNdgkDWo5X1+SpG7Z\ntUGS1KXXAf+vqn4jyc7AruMOqDMmBzR6y/f6kiR1aqgWCWa+JUnzlWRP4LHAmwCq6q6qunW8UXWo\nMtgkDWDZX1+aONYjpMk2bNeGmcz3TwMPA64cPiRJ0pT6ceAG4M1JPpPkzCS7jTuorqQGm6QBLevr\nSxPJeoQ0wRacSDDzLUka0I7AzwP/UFUPB74NnDq7UJKTk1yS5JK7b/tO1zGORi1gGkKSlUk2JNnY\n/tx7jnJr2jIbk6zpWf6KJJuS3Dmr/B8n+WKSK5Kcn+SBw0WqRTTw9XXDTVu6jlECrEdI02CYFglm\nviVJg9gMbK6qi9r5c2kqPvdSVWuranVVrd7xftPSxXvAbg3Dd204FTi/qg4Dzqd/hXIlcBpwBPBI\n4LSehMMH22WzfQZYXVUPpXn/Xj1soFo0A19f++2zotMApR7WI6QJN0wiYeDM9w/4/hCHkyRNsqr6\nFrApyU+1i44CvjjGkLrVYYsE4ARgXfv7OuDEPmWOBTZU1c1VdQuwATgOoKourKprf+QUqv6tqmaa\niVwIHDx0pFoUy/760qSxHiFNuGGe2tAv8/0jfwCqai2wFmDPrLTXpyQtb/8beHs7ovzXgOeOOZ7u\nDP4fcN8kl/TMr23/p87H/WcSAVV1bZL9+5Q5CNjUM7+5XTZfvwP86wDlNXrL9/rSpLEeIU24BScS\nqupbbf/Jn6qqL2PmW5K0HVV1ObB63HGMxeBfgW+sqjlfqyQfAw7os+ql89x/v/4T84oyybNo3sfH\nzfNY6sCyvr40UaxHSJNvmBYJYOZbkqTtKxb9kY5V9YS51iW5LsmBbWuEA4Hr+xTbDBzZM38wcMH2\njpvkCTTJisdVlW2NJS2U9Qhpgg2VSDDzLUnS/HT8SMfzgDXAGe3PD/Qpsx54Zc8Ai8cAL9nWTpM8\nHPgn4Liq6peckKR5sR4hTbZhBluUJEnz1e1gi2cARyfZCBzdzpNkdZIzAarqZuDlwMXt9LJ2GUle\nnWQzsGuSzUlOb/f7N8DuwHuSXJ7kvKEjlSRJE2fYrg2SJGmJqaqbaPocz15+CXBSz/xZwFl9yp0C\nnNJn+ZzdKSRJ0vJhIqEjx646fMHbrr/m8qGOPcz2w8Q97LElaZp03LVBkjQllms9QkubiQRJ0pK1\n04otHLTHbeMOo6/LBt1gkQdblCRJGhcTCZIkjdrijHsgSZK0JDjYoiRJkiRJmjdbJEiS1AVbJEiS\npClhIkGSpA442KIkSZoWJhIkSeqCiQRJkjQlTCRIktQFEwmSJGlKmEiQJGnEUnZtkCRJ08NEgiRJ\nXaiMOwJJkqRFYSJBkqQu2CJBkiRNCRMJkiR1wK4NkiRpWphIkCSpCyYSJEnSlDCRIEnSqDnYoiRJ\nmiImEiRJ6oKJBEmSNCVMJEiS1AUTCZIkaUqYSJgAx646fKjt119z+Vi2heFjH8Ykxy5p+ti1QZLU\nNb/PalR2GHcAkiRJkiRpctgiQZKkLtgiQZIkTQkTCZIkjZpPbZAkSVNkqERCkj8CTqK5z/I54LlV\n9b3FCEySpLp6B37w3PuOOwxpKn3lil2XcP/pq8YdgEbMeoQ02RY8RkKSg4DnA6ur6iHACuDpixWY\nJElTpQacJGlKWY+QJt+wXRt2BO6b5AfArsA1w4ckSdIUMjkgSb2sR0gTbMEtEqrqm8D/Bf4buBa4\nrao+uliBSZI0LUIzRsIgkyRNK+sR0uQbpmvD3sAJwI8Bq4DdkjyrT7mTk1yS5JIf8P2FRypJ0iTr\nsGtDkpVJNiTZ2P7ce45ya9oyG5Os6Vn+iiSbktw5q/zvJflcksuTfDLJg4eLVNJyZD1CmnwLTiQA\nTwC+XlU3VNUPgPcBj5ldqKrWVtXqqlq9E/cZ4nCSJE2oAVsjLEKLhFOB86vqMOD8dv5ekqwETgOO\nAB4JnNaTcPhgu2y2d1TVz1XV4cCrgdcMHamk5ch6hDThhkkk/DfwqCS7JglwFHDl4oQlSZpGSf4o\nyReSfD7JOUl2GXdMnel2sMUTgHXt7+uAE/uUORbYUFU3V9UtwAbgOICqurCqrv2RU6i6vWd2t0WJ\nVItmWV9fmjTWI6QJN8wYCRcB5wKX0TyyZQdg7SLFJUmaMst+lO5uEwn3n0kEtD/371PmIGBTz/zm\ndtk2JflfSb5K0yLh+UNHqkWx7K8vTRTrEdLkG+qpDVV1Gk2zSEmS5mPZjtK9gO4K+ya5pGd+bVX9\n8It2ko8BB/TZ7qXzDanPsu1GWVVvBN6Y5LeAPwfWbGcTdWfZXl+aPNYjpMk27OMfJUmal6r6ZpKZ\nUbq/C3x0WY3SPXgi4caqWj3n7qqeMNe6JNclObCqrk1yIHB9n2KbgSN75g8GLhggvncC/zBAeY3Q\nsr++JEmdGmaMBEmS5m0ho3TfteU7XYc5GoN2axi+a8N53NNSYA3wgT5l1gPHJNm7fW+OaZfNKclh\nPbNPBjYOHakWhaPgS5K6ZCJBktSVgUfp3nnFrp0HOSodP7XhDODoJBuBo9t5kqxOciZAVd0MvBy4\nuJ1e1i4jyauTbAZ2TbI5yentfv+wHczvcuCPsVvDUuIo+JKkzti1YRk4dtXh4w5hLJbreUtL2A9H\n6aZpen0UcMm2N5kiHT7foKpuonl9Zy+/BDipZ/4s4Kw+5U4BTumz/AWLG6kW0fK+viRJnTKRIEnq\nRFVdlGRmlO67gc+wjEbpXoRWBtKclvv1JUnqlokESVJnlvUo3SYSNGLL+vqSJHXKRIIkSaO2OAMo\nSpIkLQkmEiRJGrG0kyRJ0jQwkSBJUhdskSBJkqaEj3+UJEmSJEnzZosESZI64FMbJEnStDCRIEla\nsuquH7Bl0zXjDmNxmEiQJElTwkSCJEldMJEgSZKmhIkESZJGrezaIEmSpoeJBEmSumAiQZIkTQkT\nCZIkdcAWCZIkaVqYSJAkqQsmEiRJ0pQwkSBJUgdskSBJkqaFiQSN1PprLh9q+2NXHb5IkQxu2NiH\nMc7zljQChS0SJEnS1DCRIElSF0wkSJKkKWEiQZKkEQt2bZAkSdPDRIIkSV0wkSBJkqaEiQRJkjqQ\nMpMgSZKmww7bK5DkrCTXJ/l8z7KVSTYk2dj+3Hu0YUqSNMFqAZMkTQHrEtJ02m4iATgbOG7WslOB\n86vqMOD8dl6SJM0hNdgkSVPibKxLSFNnu4mEqvoEcPOsxScA69rf1wEnLnJckiRNF1skSFqGrEtI\n02k+LRL6uX9VXQvQ/tx/roJJTk5ySZJLfsD3F3g4SZIkSVNiXnUJ6xHS0rXQRMK8VdXaqlpdVat3\n4j6jPpwkSUuSXRskaTDWI6Sla6GJhOuSHAjQ/rx+8UKSJGkKddi1Yb4DmSVZ05bZmGRNz/JXJNmU\n5M45tvuNJJVk9XCRSlqmrEtIE26hiYTzgJkvHGuADyxOOJIkTaEBWyMsQouE7Q5klmQlcBpwBPBI\n4LSehMMH22U/IskewPOBi4aOUtJyZV1CmnA7bq9AknOAI4F9k2ym+dJxBvDuJL8D/Dfw1FEGKUla\npqqoH9w17igWR7fdFU6g+d8NzUBmFwAvnlXmWGBDVd0MkGQDzcjq51TVhe2yfvt+OfBq4EWLHbSk\n6WNdQppO200kVNUz5lh11CLHIknSVAqdj3twr4HMkvQbyOwgYFPP/OZ22ZySPBw4pKo+lMREgqTt\nsi4hTaftJhIkSdIiqIEzCfsmuaRnfm1VrZ2ZSfIx4IA+2710nvvv19xgziCT7AD8HfCcee5fkiRN\nKRMJkiR1YAEtEm6sqjkHM6yqJ8x5rOS6JAe2rRHmGshsM/d0fwA4mKYLxFz2AB4CXNB2eTgAOC/J\nU6rqkm1sJ0mSpoyJBI3UsasOH9ux119z+VDbDxv7sMeXNEUW4UkMA5oZyOwM5h7IbD3wyp4BFo8B\nXjLXDqvqNmDfmfkkFwAvMokgSdLiGqYe0VX9a6FPbZAkSQPI1sGmIZ0BHJ1kI3B0O0+S1UnOBGgH\nWXw5cHE7vaxn4MVXt4Oi7Zpkc5LTh45IkiRNDVskSJIWVZKzgOOB66vqIe2ylcC7gEOBq4GnVdUt\n44pxLDpskVBVN9FnILO29cBJPfNnAWf1KXcKcMp2jnHk0IFqYF5fkqSlwBYJkqTFdjbNYwR7nQqc\nX1WHAee388tKarBJmsPZeH1JksbMRIIkaVFV1SeAm2ctPgFY1/6+Djix06DGrWie2jDIJPXh9SVJ\nWgrs2iBJ6sL9q+pagPZJAvuPO6Cu2cpAI7Tsry9JUrdMJEiSlpQkJwMnA+zCrmOOZhGZSNASMLXX\nlySpU3ZtkCR14bokBwK0P6+fq2BVra2q1VW1eifu01mAoxQcI0EjtayvL0lS90wkSJK6cB6wpv19\nDfCBMcbSvUHHR3CMBA1meV9fkqTOmUiQJC2qJOcAnwJ+KsnmJL8DnAEcnWQjcHQ7L2lAXl+SpKXA\nMRIkSYuqqp4xx6qjOg1kibG7ghaD15ckaSkwkSBJUhdMJEiSpClhIkGSpA7YIkGSJE0LEwmSJI1a\nAVvNJEiSpOlgIkGSpC6YR5AkSVPCRIIkSR2wa4MkSZoWJhKkOay/5vKhtj921eGLFIm0fG1ZuRu3\nH/eocYfR3zvOHax8mUnQ0vKgh36H9euH+183KisOHHcEkrRwy6EeYSJBkqQO2CJBkiRNCxMJkiSN\nWuEYCZIkaWqYSJAkacQCxK4NkiRpSuywvQJJzkpyfZLP9yz7myRfSnJFkn9Jstdow5QkacJtHXCS\npAlnPUKaXttNJABnA8fNWrYBeEhVPRT4CvCSRY5LkqSpkqqBJkmaAmdjPUKaSttNJFTVJ4CbZy37\naFXd3c5eCBw8gtgkSZoOtYBJkiac9Qhpes2nRcL2PA/410XYjyRJU6qaxz8OMknS9LMeIU2ooQZb\nTPJS4G7g7dsoczJwMsAu7DrM4SRJmlg+/lGS7mE9QppsC04kJFkDHA8cVTX3rZOqWgusBdgzK/0a\nJUlanmxlIEmA9QhpGiwokZDkOODFwOOq6juLG5IkSZKkaWQ9QpoO83n84znAp4CfSrI5ye8AbwD2\nADYkuTzJP444TkmSJldBtg42DSPJyiQbkmxsf+49R7k1bZmN7R3CmeWvSLIpyZ2zyj8nyQ3t//7L\nk5w0XKSSppn1CGl6bbdFQlU9o8/iN40gFkmSple3XRtOBc6vqjOSnNrOv7i3QJKVwGnAaprnRFya\n5LyqugX4IM2X/Y199v2uqvrDkUYvaSpYj5Cm12I8tUGSJG1Pt49/PAFY1/6+DjixT5ljgQ1VdXOb\nPNhA+7z3qrqwqq4dOgpJkjSVTCRIktSBVA00Den+M4mA9uf+fcocBGzqmd/cLtueX09yRZJzkxwy\nbKCSJGnyDPX4R2kpO3bV4eMOQZLuMXhyYN8kl/TMr21HMAcgyceAA/ps99J57j99lm0vyA8C51TV\n95P8Hk1rh8fP83iSJC0Ly6EeYiJBkqRRK2DwARRvrKrVc+6y6glzrUtyXZIDq+raJAcC1/cpthk4\nsmf+YOCCbQVUVTf1zP4z8NfbKi9JkqaTXRskSRqxMFi3hkXo2nAeMPMUhjXAB/qUWQ8ck2Tv9qkO\nx7TL5j6PJikx4ynAlcMGKkmSJo+JBEmSulA12DScM4Cjk2wEjm7nSbI6yZlNOHUz8HLg4nZ6WbuM\nJK9OshnYtX1k2+ntfp+f5AtJPgs8H3jOsIFKkqTJY9cGSZK60OHjH9suCEf1WX4JcFLP/FnAWX3K\nnQKc0mf5S4CXLGqwkiRp4phIkCRp1BY2RoIkSdKSZCJBkqQOLMK4B5IkSUuCiQRJkrpgIkGSJE0J\nEwmSJI3cogygKEmStCSYSJAkadQKEwmSJGlq+PhHSdKiSnJWkuuTfL5n2d8k+VKSK5L8S5K9xhnj\nWGwdcJL68PqSJC0FJhIkSYvtbOC4Wcs2AA+pqocCX2EZPkIwVQNN0hzOxutLkjRmJhIkSYuqqj4B\n3Dxr2Uer6u529kLg4M4Dk6aA15ckaSkwkSBJ6trzgH8ddxCdqxpskhZmeV5fkqROOdiiJKkzSV4K\n3A28fRtlTgZOBth51707imzECthqckCjNej19YCD/BooSVoYWyRIkjqRZA1wPPDMqrlvuVfV2qpa\nXVWrd9xlt+4CHKkBWyPYIkEDWsj1td8+K7oLUJI0VTpNRd/BLTd+rM79xjaK7Avc2FU8Hnvsxx73\n8af12A8c0X6lBUtyHPBi4HFV9Z1xxzMWJgc0Il5fWg6sR3jsJXb8aT32vOsRnSYSqmq/ba1PcklV\nre4qHo893mOP+/jL9djSqCU5BzgS2DfJZuA0mlHk7wNsSAJwYVX93tiCHAcTCVoEXl9arqxHeOyl\ndPzleuxedo6TJC2qqnpGn8Vv6jyQpcQxErRIvL4kSUuBiQRJkkauoLaOOwhJkqRFsdQSCWs99rI6\n9riPv1yPLWkc7NogSaO0XL/XLddjj/v4y/XYP7SkEglVNbYXxWMvv+Mv12NLGgO7NkjSSC3X73XL\n9djjPv5yPXavJZVIkCRpatkiQZIkTYkdxh0ANI8tSvLlJFclObXjYx+S5N+SXJnkC0le0PHxVyT5\nTJIPdXnc9th7JTk3yZfa8390h8f+o/b1/nySc5LsMuLjnZXk+iSf71m2MsmGJBvbn3t3eOy/aV/3\nK5L8S5K9RnFsSUtI1WCTJGm7rEdYj7AeMZ56RGrMX1aSrAC+AhwNbAYuBp5RVV/s6PgHAgdW1WVJ\n9gAuBU7s8Ph/DKwG9qyq47s4Zs+x1wH/UVVnJtkZ2LWqbu3guAcBnwQeXFXfTfJu4CNVdfYIj/lY\n4E7gLVX1kHbZq4Gbq+qM9h/P3lX14o6OfQzw8aq6O8lfA4zi2NKk23OHfepROx037jD62nDXOy6d\n7+OX7rfz/vWY/X5zoP3/v2veMO/9SwuxZ1bWETlq3GH09bE618+/tst6hPUI6xHjq0cshRYJjwSu\nqqqvVdVdwDuBE7o6eFVdW1WXtb/fAVwJHNTFsZMcDDwZOLOL48069p7AY2kfGVVVd3Vx8ffYEbhv\nkh2BXYFrRnmwqvoEcPOsxScA69rf///27i7ErquMw/jzt9GLtgpKWj+aYiPEQPHCapFqb7StdZTS\neiUVlUCrdxVEMCq9KYpSRMQLixJCVDBWQi2aGzuNqCjSSKKI9oOS0lpzNDZJExX1QkNeL/YOjHFm\nss/MOXvPOXl+sCF7zznrXTOcNcx6s9Z6vw28v6/YVfVoVZ1pbw8CW6YRW9IGUcDZs+NdkqQLcR7h\nPMJ5xEDziI2QSLgKOLrkfkRPA/B8Sa4BrgN+1VPIrwI7gSH+YnwDcAL4ZrskaneSy/oIXFV/Ar4M\n/BE4Bvytqh7tI/Z5Xl1Vx9o+HQOuHKAPAHcBPxootqS+uLVBkibNeYTzCOcRA80jNkIiIcs86/0v\nqCSXA98HPlFVf+8h3m3A8ar69bRjrWAT8Bbg61V1HfBPoJd9Ze0eojuArcDrgMuSfLiP2BtNknuB\nM8DeofsiacpMJEjSpDmPGIbziA1g6HnERkgkjICrl9xvYcrLU86X5KU0g39vVT3cU9gbgduT/IFm\nGdZNSb7TU2xofu6jqjqXNX2I5hdCH24BnquqE1X1H+Bh4B09xV7qhXZv27k9bsf7DJ5kB3Ab8KEa\n+rASSVNWTfnHcS5J0oU4j3Ae4TxioHnERkgkHAK2JdnaHtRxJ7C/r+BJQrO/56mq+kpfcavqs1W1\npaquofmef1JVvWXTquovwNEk29tHNwO9HAxDsxTphiSXtj//m2n2lPVtP7Cj/fcO4Id9BU6yAHwa\nuL2q/tVXXEkDKag6O9YlSbog5xHOI5xHDGTwREJ7UMQ9wCLNh2BfVT3RYxduBD5Ck8n7bXu9r8f4\nQ/o4sDfJ74A3A1/sI2ibvXwI+A3we5rP4a5pxkzyIPAYsD3JKMndwP3Au5McoTnt9/4eY38NeDlw\noP3MfWMasSVdnLqWpUqyo33NkfZ/N849/0KSo0n+scx7PpDkyTSlt747ze9DklbjPGJQziMu8nnE\n4OUfJUlaydyUf9x0Rb39FeMd6Lx4eveay991KUuV5FXAYZrSYUVTtuytVXU6yQ3A88CRqrp8yXu2\nAfuAm9rXXVlVvS7n1HNyySkAAAODSURBVORY/lGStFaDr0iQJOmi0O9hi13KUr0HOFBVp6rqNHAA\nWGi6WgfPnUZ9no8BD7SvxySCJEkXp01Dd0CSpLlXBWfHPvdgc5LDS+53VVXX5Zv/U5YqyXJlqdZS\nNu2NAEl+CVwC3FdVj3TskyRJmhMmEiRJ6sP4qwxOrra0O8mPgdcs86V7O7a/lrJpm4BtwDtpTkf/\nRZI3VdVfO8aUJElzwESCJEk9qPFXJKzeXtUtK30tyQtJXtuuRlipLNWIJiFwzhbgZxcIOwIOtiW3\nnkvyNE1i4dA4fZckSbPNMxIkSZq6Mc9HWP8ZCV3KUi0CtyZ5ZVvV4db22Wp+ALwLIMlmmq0Oz663\ns5IkabaYSJAkadoKOFvjXeuzbFmqJNcn2Q1QVaeAz9OsJjgEfK59RpIvJRkBl7blpu5r210EXkzy\nJPBT4FNV9eJ6OytJkmaLWxskSepDTXZrw6qhmsn9/9X1q6rDwEeX3O8B9izzup3AzmWeF/DJ9pIk\nSRcpEwmSJE1ZAbX+VQaSJEkbgokESZKmrarXFQmSJEnTZCJBkqQeuCJBkiTNCxMJkiT1wRUJkiRp\nTqTWX2JKkqSpSHICeL7DSzcDJycUtmtbr6+qK7o0mOSRtt1xnKyqhTHfI3XWcXxNcmyN017n8SVJ\n6p+JBEnSzEtyuKqu32htSbNu0uPB8SVJ8+ElQ3dAkiRJkiTNDhMJkiRJkiSpMxMJkqR5sGuDtiXN\nukmPB8eXJM0Bz0iQJEmSJEmduSJBkiRJkiR1ZiJBkjSzkiwkeTrJM0k+s8629iQ5nuTxSfVPmmWO\nL0nSSkwkSJJmUpJLgAeA9wLXAh9Mcu06mvwWsDCBrkkzz/ElSVqNiQRJ0qx6G/BMVT1bVf8Gvgfc\nsdbGqurnwKlJdU6acY4vSdKKTCRIkmbVVcDRJfej9pmk9XN8SZJWZCJBkjSrsswzSxFJk+H4kiSt\nyESCJGlWjYCrl9xvAf48UF+keeP4kiStyESCJGlWHQK2Jdma5GXAncD+gfskzQvHlyRpRSYSJEkz\nqarOAPcAi8BTwL6qemKt7SV5EHgM2J5klOTuyfRUmj2OL0nSalLldjdJkiRJktSNKxIkSZIkSVJn\nJhIkSZIkSVJnJhIkSZIkSVJnJhIkSZIkSVJnJhIkSZIkSVJnJhIkSZIkSVJnJhIkSZIkSVJnJhIk\nSZIkSVJn/wUHnstK1+v3fgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7bdae2a438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [32/1001 (3%)]\tLoss: 0.677612 (avg: 0.677612) \tsec/iter: 2.2921\n",
      "Train Epoch: 0 [352/1001 (34%)]\tLoss: 0.696311 (avg: 0.680245) \tsec/iter: 0.7424\n",
      "Train Epoch: 0 [672/1001 (66%)]\tLoss: 0.649672 (avg: 0.669781) \tsec/iter: 0.6732\n",
      "Train Epoch: 0 [992/1001 (97%)]\tLoss: 0.618369 (avg: 0.669806) \tsec/iter: 0.6460\n",
      "Train Epoch: 0 [1001/1001 (100%)]\tLoss: 0.862473 (avg: 0.671538) \tsec/iter: 0.6314\n",
      "Test set (epoch 0): Average loss: 0.6626, Accuracy: 597/1001 (59.64%)\n",
      "\n",
      "Train Epoch: 1 [32/1001 (3%)]\tLoss: 0.691414 (avg: 0.691414) \tsec/iter: 0.5904\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-58c79f509e9f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    105\u001b[0m     \u001b[0mloss_fn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    106\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 107\u001b[1;33m         \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloaders\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    108\u001b[0m         \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloaders\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    109\u001b[0m     \u001b[0macc_folds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0macc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-5-58c79f509e9f>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(train_loader)\u001b[0m\n\u001b[0;32m     67\u001b[0m             \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m             \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m             \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     70\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m             \u001b[0mtime_iter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m     91\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m         \"\"\"\n\u001b[1;32m---> 93\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 90\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m     91\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "print('Loading data')\n",
    "datareader = DataReader(data_dir='./data/%s/' % dataset.upper(),\n",
    "                        rnd_state=np.random.RandomState(seed),\n",
    "                        folds=n_folds,                    \n",
    "                        use_cont_node_attr=False)\n",
    "\n",
    "acc_folds = []\n",
    "for fold_id in range(n_folds):\n",
    "    print('\\nFOLD', fold_id)\n",
    "    loaders = []\n",
    "    for split in ['train', 'test']:\n",
    "        gdata = GraphData(fold_id=fold_id,\n",
    "                             datareader=datareader,\n",
    "                             split=split)\n",
    "\n",
    "        loader = torch.utils.data.DataLoader(gdata, \n",
    "                                             batch_size=batch_size,\n",
    "                                             shuffle=split.find('train') >= 0,\n",
    "                                             num_workers=threads)\n",
    "        loaders.append(loader)\n",
    "    \n",
    "    if model_name == 'gcn':\n",
    "        model = GCN(in_features=loaders[0].dataset.features_dim,\n",
    "                    out_features=loaders[0].dataset.n_classes,\n",
    "                    n_hidden=0,\n",
    "                    filters=[64,64,64],\n",
    "                    dropout=0.2,\n",
    "                    adj_sq=False,\n",
    "                    scale_identity=False).to(device)\n",
    "    elif model_name == 'unet':\n",
    "        model = GraphUnet(in_features=loaders[0].dataset.features_dim,\n",
    "                          out_features=loaders[0].dataset.n_classes,\n",
    "                          n_hidden=0,\n",
    "                          filters=[64,64,64],\n",
    "                          dropout=0.2,\n",
    "                          adj_sq=False,\n",
    "                          scale_identity=False,\n",
    "                          shuffle_nodes=shuffle_nodes,\n",
    "                          visualize=visualize).to(device)\n",
    "    else:\n",
    "        raise NotImplementedError(model_name)\n",
    "\n",
    "    print('\\nInitialize model')\n",
    "    print(model)\n",
    "    c = 0\n",
    "    for p in filter(lambda p: p.requires_grad, model.parameters()):\n",
    "        c += p.numel()\n",
    "    print('N trainable parameters:', c)\n",
    "\n",
    "    optimizer = optim.Adam(\n",
    "                filter(lambda p: p.requires_grad, model.parameters()),\n",
    "                lr=lr,\n",
    "                weight_decay=wdecay,\n",
    "                betas=(0.5, 0.999))\n",
    "    \n",
    "    scheduler = lr_scheduler.MultiStepLR(optimizer, [20, 30], gamma=0.1)\n",
    "\n",
    "    def train(train_loader):\n",
    "        scheduler.step()\n",
    "        model.train()\n",
    "        start = time.time()\n",
    "        train_loss, n_samples = 0, 0\n",
    "        for batch_idx, data in enumerate(train_loader):\n",
    "            for i in range(len(data)):\n",
    "                data[i] = data[i].to(device)\n",
    "            optimizer.zero_grad()\n",
    "            output = model(data)\n",
    "            loss = loss_fn(output, data[4])\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            time_iter = time.time() - start\n",
    "            train_loss += loss.item() * len(output)\n",
    "            n_samples += len(output)\n",
    "            if batch_idx % log_interval == 0 or batch_idx == len(train_loader) - 1:\n",
    "                print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f} (avg: {:.6f}) \\tsec/iter: {:.4f}'.format(\n",
    "                    epoch, n_samples, len(train_loader.dataset),\n",
    "                    100. * (batch_idx + 1) / len(train_loader), loss.item(), train_loss / n_samples, time_iter / (batch_idx + 1) ))\n",
    "    #             break \n",
    "    def test(test_loader):\n",
    "        model.eval()\n",
    "        start = time.time()\n",
    "        test_loss, correct, n_samples = 0, 0, 0\n",
    "        for batch_idx, data in enumerate(test_loader):\n",
    "            for i in range(len(data)):\n",
    "                data[i] = data[i].to(device)\n",
    "            output = model(data)\n",
    "            loss = loss_fn(output, data[4], reduction='sum')\n",
    "            test_loss += loss.item()\n",
    "            n_samples += len(output)\n",
    "            pred = output.detach().cpu().max(1, keepdim=True)[1]\n",
    "\n",
    "            correct += pred.eq(data[4].detach().cpu().view_as(pred)).sum().item()\n",
    "\n",
    "        time_iter = time.time() - start\n",
    "\n",
    "        test_loss /= n_samples\n",
    "\n",
    "        acc = 100. * correct / n_samples\n",
    "        print('Test set (epoch {}): Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\\n'.format(epoch, \n",
    "                                                                                              test_loss, \n",
    "                                                                                              correct, \n",
    "                                                                                              n_samples, acc))\n",
    "        return acc\n",
    "\n",
    "    loss_fn = F.cross_entropy\n",
    "    for epoch in range(epochs):\n",
    "        train(loaders[0])\n",
    "        acc = test(loaders[0])\n",
    "    acc_folds.append(acc)\n",
    "\n",
    "print(acc_folds)\n",
    "print('{}-fold cross validation avg acc (+- std): {} ({})'.format(n_folds, np.mean(acc_folds), np.std(acc_folds)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}