From 6ca5bcd9588153c5cf0e9b1adf16fa06c914e91c Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 14 Jun 2024 12:21:29 +0300 Subject: [PATCH 01/43] metrics started --- fedot_ind/core/models/early_tc/metrics.py | 122 ++++++++++++++++++++++ fedot_ind/core/models/early_tc/teaser.py | 0 2 files changed, 122 insertions(+) create mode 100644 fedot_ind/core/models/early_tc/metrics.py create mode 100644 fedot_ind/core/models/early_tc/teaser.py diff --git a/fedot_ind/core/models/early_tc/metrics.py b/fedot_ind/core/models/early_tc/metrics.py new file mode 100644 index 000000000..a4cb9cef5 --- /dev/null +++ b/fedot_ind/core/models/early_tc/metrics.py @@ -0,0 +1,122 @@ +from sklearn.metrics import confusion_matrix +import numpy as np +import pandas as pd +from fedot.core.data.data import InputData, OutputData +from typing import Tuple, List, Optional, Union, Literal + +def conf_matrix(actual, predicted): + cm = confusion_matrix(actual, predicted) + return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) + +def average_delay(boundaries, prediction, + point, + use_idx=True, + window_placement='lefter'): + cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) + # statistics + statistics = { + 'anomalies_num': len(cp_confusion['TPs']) + len(cp_confusion['FPs']), + 'FP_num': len(cp_confusion['FPs']), + 'missed': len(cp_confusion['FNs']) + } + time_func = { + 'righter': lambda triplet: triplet[1] - triplet[0], + 'lefter': lambda triplet: triplet[2] - triplet[1], + 'central': lambda triplet: triplet[1] - triplet[0] - (triplet[2] - triplet[0]) / 2 + }[window_placement] + + detection_history = { + i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() + } + return detection_history, statistics + + + +def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], + prediction: pd.DataFrame, + use_switch_point: bool = True, # if first anomaly dot is considered as changepoint + use_idx: bool = False): + if isinstance(boundaries, pd.DataFrame): + boundaries = boundaries.values.T + anomaly_tsp = prediction[prediction == 1].sort_index().index + TPs, FNs, FPs = {}, [], [] + + if boundaries.shape[1]: + + FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest + for i, (b_low, b_up) in enumerate(boundaries): + all_tsp_in_window = prediction[b_low: b_up].index + anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp + if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? + FNs.append(i if use_idx else all_tsp_in_window) + TPs[i] = [b_low, anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, b_up] + if not use_idx: + FNs.append(all_tsp_in_window - anomaly_tsp_in_window) + FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest + else: + FPs.append(anomaly_tsp) + + FPs = np.concatenate(FPs) + FNs = np.concatenate(FNs) + + return dict( + FP=FPs, + FN=FNs, + TP=TPs + ) + + +# cognate of single_detecting_boundaries +def get_boundaries(idx, actual_timestamps, window_size:int = None, + window_placement: Literal['left', 'right', 'central'] = 'left', + intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', + ): + # idx = idx + # cast everything to pandas object fir the subsequent comfort + if isinstance(idx, np.array): + if idx.dtype == np.dtype('O'): + idx = pd.to_datetime(pd.Series(idx)) + td = pd.Timedelta(window_size) + else: + idx = pd.Series(idx) + td = window_size + else: + raise TypeError('Unexpected type of ts index') + + boundaries = np.tile(actual_timestamps, (2, 1)) + # [0, ...] - lower bound, [1, ...] - upper + if window_placement == 'left': + boundaries[0] -= td + elif window_placement == 'central': + boundaries[0] -= td / 2 + boundaries[1] += td / 2 + elif window_placement == 'right': + boundaries[1] += td + else: + raise ValueError('Unknown mode') + + if not len(actual_timestamps): + return boundaries + + # intersection resolution + for i in range(len(actual_timestamps) - 1): + if not boundaries[0, i + 1] > boundaries[1, i]: + continue + + if intersection_mode == 'shift_to_left': + boundaries[0, i + 1] = boundaries[1, i] + elif intersection_mode == 'shift_to_right': + boundaries[1, i] = boundaries[0, i + 1] + elif intersection_mode == 'uniform': + boundaries[1, i], boundaries[0, i + 1] = boundaries[0, i + 1], boundaries[1, i] + else: + raise ValueError('Unknown intersection resolution') + + # filtering + idx_to_keep = np.abs(np.diff(boundaries, axis=0)) > 1e-6 + boundaries = boundaries[..., idx_to_keep] + boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) + return boundaries + + + diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py new file mode 100644 index 000000000..e69de29bb From e69fcd4ba6baa10dd02585333a052badf15f8a39 Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 20 Jun 2024 13:17:30 +0300 Subject: [PATCH 02/43] metrics ended --- fedot_ind/core/models/early_tc/metrics.py | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/fedot_ind/core/models/early_tc/metrics.py b/fedot_ind/core/models/early_tc/metrics.py index a4cb9cef5..f4a5f6544 100644 --- a/fedot_ind/core/models/early_tc/metrics.py +++ b/fedot_ind/core/models/early_tc/metrics.py @@ -30,7 +30,8 @@ def average_delay(boundaries, prediction, } return detection_history, statistics - +def tp_transform(tps): + return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], prediction: pd.DataFrame, @@ -49,7 +50,9 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? FNs.append(i if use_idx else all_tsp_in_window) - TPs[i] = [b_low, anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, b_up] + TPs[i] = [b_low, + anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, + b_up] if not use_idx: FNs.append(all_tsp_in_window - anomaly_tsp_in_window) FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest @@ -62,10 +65,9 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], return dict( FP=FPs, FN=FNs, - TP=TPs + TP=np.stack(TPs) ) - # cognate of single_detecting_boundaries def get_boundaries(idx, actual_timestamps, window_size:int = None, window_placement: Literal['left', 'right', 'central'] = 'left', @@ -118,5 +120,19 @@ def get_boundaries(idx, actual_timestamps, window_size:int = None, boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) return boundaries - +def nab(boundaries, predictions, mode='standard', custom_coefs=None): + inner_coefs = { + 'low_FP': [1.0, -0.11, -1.0], + 'standard': [1., -0.22, -1.], + 'lof_FN': [1., -0.11, -2.] + } + coefs = custom_coefs or inner_coefs[mode] + confusion_matrix = extract_cp_cm(boundaries, predictions) + + tps = confusion_matrix['tps'] + + score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], + coefs) + return score + From bc4064df91209922c9c39bc5c46f2db7d6cb4b09 Mon Sep 17 00:00:00 2001 From: leostre Date: Mon, 24 Jun 2024 03:05:51 +0300 Subject: [PATCH 03/43] in basis teaser is completed, need some make-up and add cut ts support --- fedot_ind/core/metrics/interval_metrics.py | 138 +++++++++++++++++++++ fedot_ind/core/models/early_tc/teaser.py | 123 ++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 fedot_ind/core/metrics/interval_metrics.py diff --git a/fedot_ind/core/metrics/interval_metrics.py b/fedot_ind/core/metrics/interval_metrics.py new file mode 100644 index 000000000..f4a5f6544 --- /dev/null +++ b/fedot_ind/core/metrics/interval_metrics.py @@ -0,0 +1,138 @@ +from sklearn.metrics import confusion_matrix +import numpy as np +import pandas as pd +from fedot.core.data.data import InputData, OutputData +from typing import Tuple, List, Optional, Union, Literal + +def conf_matrix(actual, predicted): + cm = confusion_matrix(actual, predicted) + return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) + +def average_delay(boundaries, prediction, + point, + use_idx=True, + window_placement='lefter'): + cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) + # statistics + statistics = { + 'anomalies_num': len(cp_confusion['TPs']) + len(cp_confusion['FPs']), + 'FP_num': len(cp_confusion['FPs']), + 'missed': len(cp_confusion['FNs']) + } + time_func = { + 'righter': lambda triplet: triplet[1] - triplet[0], + 'lefter': lambda triplet: triplet[2] - triplet[1], + 'central': lambda triplet: triplet[1] - triplet[0] - (triplet[2] - triplet[0]) / 2 + }[window_placement] + + detection_history = { + i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() + } + return detection_history, statistics + +def tp_transform(tps): + return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) + +def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], + prediction: pd.DataFrame, + use_switch_point: bool = True, # if first anomaly dot is considered as changepoint + use_idx: bool = False): + if isinstance(boundaries, pd.DataFrame): + boundaries = boundaries.values.T + anomaly_tsp = prediction[prediction == 1].sort_index().index + TPs, FNs, FPs = {}, [], [] + + if boundaries.shape[1]: + + FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest + for i, (b_low, b_up) in enumerate(boundaries): + all_tsp_in_window = prediction[b_low: b_up].index + anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp + if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? + FNs.append(i if use_idx else all_tsp_in_window) + TPs[i] = [b_low, + anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, + b_up] + if not use_idx: + FNs.append(all_tsp_in_window - anomaly_tsp_in_window) + FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest + else: + FPs.append(anomaly_tsp) + + FPs = np.concatenate(FPs) + FNs = np.concatenate(FNs) + + return dict( + FP=FPs, + FN=FNs, + TP=np.stack(TPs) + ) + +# cognate of single_detecting_boundaries +def get_boundaries(idx, actual_timestamps, window_size:int = None, + window_placement: Literal['left', 'right', 'central'] = 'left', + intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', + ): + # idx = idx + # cast everything to pandas object fir the subsequent comfort + if isinstance(idx, np.array): + if idx.dtype == np.dtype('O'): + idx = pd.to_datetime(pd.Series(idx)) + td = pd.Timedelta(window_size) + else: + idx = pd.Series(idx) + td = window_size + else: + raise TypeError('Unexpected type of ts index') + + boundaries = np.tile(actual_timestamps, (2, 1)) + # [0, ...] - lower bound, [1, ...] - upper + if window_placement == 'left': + boundaries[0] -= td + elif window_placement == 'central': + boundaries[0] -= td / 2 + boundaries[1] += td / 2 + elif window_placement == 'right': + boundaries[1] += td + else: + raise ValueError('Unknown mode') + + if not len(actual_timestamps): + return boundaries + + # intersection resolution + for i in range(len(actual_timestamps) - 1): + if not boundaries[0, i + 1] > boundaries[1, i]: + continue + + if intersection_mode == 'shift_to_left': + boundaries[0, i + 1] = boundaries[1, i] + elif intersection_mode == 'shift_to_right': + boundaries[1, i] = boundaries[0, i + 1] + elif intersection_mode == 'uniform': + boundaries[1, i], boundaries[0, i + 1] = boundaries[0, i + 1], boundaries[1, i] + else: + raise ValueError('Unknown intersection resolution') + + # filtering + idx_to_keep = np.abs(np.diff(boundaries, axis=0)) > 1e-6 + boundaries = boundaries[..., idx_to_keep] + boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) + return boundaries + +def nab(boundaries, predictions, mode='standard', custom_coefs=None): + inner_coefs = { + 'low_FP': [1.0, -0.11, -1.0], + 'standard': [1., -0.22, -1.], + 'lof_FN': [1., -0.11, -2.] + } + coefs = custom_coefs or inner_coefs[mode] + confusion_matrix = extract_cp_cm(boundaries, predictions) + + tps = confusion_matrix['tps'] + + score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], + coefs) + return score + + diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index e69de29bb..66dc88745 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -0,0 +1,123 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters + + +class TEASER(ModelImplementation): + def __init__(self, params: Optional[OperationParameters] = None): + super().__init__() + if params is None: + params = {} + self.interval_length = params.get('interval_length', 10) # rewrite as interval_length + self.acceptance_threshold = params.get('acceptance_threshold', 5) + self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' + # how to pass into ? % what needed + self.oc_svm_params = {} + self.weasel_params = {} + self.random_state = None # is needed? + + def _init_model(self, max_data_length): + self.prediction_idx = self._compute_prediction_points(max_data_length) + self.n_pred = len(self.prediction_idx) + self.oc_estimators = [OneClassSVM(**self.oc_svm_params) for _ in range(self.n_pred)] + self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] + self.scalers = [StandardScaler() for _ in range(self.n_pred)] # do we need them separate? no inverse path expected + + def fit(self, input_data: InputData): + input_data = self.__convert_pd(input_data) + X, y = input_data.features, input_data.target # what's passed in case of classification to training? + self._init_model(max_data_length=X.shape[-1]) + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i]] # what's dimensionality of input? will it work in case of multivariate? + X_part = self.scalers[i].fit_transform(X_part) + probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + filtered_probas = self._filter_positive(probas, y) # + X_oc = self._form_X_oc(filtered_probas) + self.oc_estimators[i].fit(X_oc, y) + + def _predict_one_slave(self, X, i): + X_part = X[..., :self.prediction_idx[i]] + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + X_oc = self._form_X_oc(probas) + return X_oc, np.argmax(probas, axis=-1) + + def _compute_prediction_points(self, n_idx): + """Computes indices for prediction, includes last index, first interval may be greater""" + prediction_idx = np.arange(n_idx - 1, -1, -self.interval_length)[::-1] + self.earliness = 1 - prediction_idx / n_idx + return prediction_idx + + def _filter_positive(self, predicted_probas, y): # different logic in sktime + predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() + return predicted_probas[predicted_labels == y] + + def _form_X_oc(self, predicted_probas): + d = (predicted_probas.max() - predicted_probas) + d[d == 0] = 1 + d = d.min(axis=-1).reshape(-1, 1) + return np.hstack([predicted_probas, d]) + + def _predict(self, X): + n = X.shape[0] + self.states = np.ones((n, self.n_pred, 2)) # num_consec, class + X_ocs, predicted_labels = zip( + *[self._predict_one_slave(X, i) for i in range(self.n_pred)] + ) + non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold + to_oc_check = np.argwhere(non_acceptance) + X_ocs = np.stack(X_ocs) + predicted_labels = np.stack(predicted_labels) + # for each point of estimation + for i in range(self.n_pred): + # find not accepted points + ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] + X_to_ith = X_ocs[i][ith_point_to_oc] + # if they are not outliers + final_verdict = self.oc_estimators[i].predict(X_to_ith) # 1 for accept -1 for reject + # mark as accepted + non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False + predicted_labels[non_acceptance] = -1 + return predicted_labels + + def _consecutive_count(self, predicted_labels: List[np.array]): + n = len(predicted_labels[0]) + consecutive_labels = np.ones((self.n_pred, n)) + for i in range(1, self.n_pred): + equal = predicted_labels[i - 1] == predicted_labels[i] + consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 + return consecutive_labels # n_pred x n_instances + + def __convert_pd(self, input_data): + if hasattr(input_data.features, 'values'): + input_data.features = input_data.features.values + if hasattr(input_data.target, 'values'): + input_data.target = input_data.target.values + return input_data + + def predict(self, input_data: InputData) -> OutputData: + input_data = self.__convert_pd(input_data) + prediction = self._predict(input_data.features) + return self._convert_to_output(input_data, predict=prediction) + + def predict_for_fit(self, input_data: InputData) -> OutputData: + return self.predict(input_data) + + def _score(self, X, y, hm_shift_to_acc=None): + hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + predictions = self._predict(X) + accuracies = (predictions == np.tile(y, (1, self.n_pred))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness / (hm_shift_to_acc * accuracies + self.earliness) + + def _tune_oc(self): + #TODO + pass From 6500db70edcb3f068ff1c2d292258ab89ab4767b Mon Sep 17 00:00:00 2001 From: leostre Date: Wed, 26 Jun 2024 17:09:09 +0300 Subject: [PATCH 04/43] teaser inherits sklearn's classifier mixin now --- fedot_ind/core/models/early_tc/__init__.py | 0 fedot_ind/core/models/early_tc/teaser.py | 61 +++++++++++++------ .../data/default_operation_params.json | 37 +++++------ .../data/industrial_model_repository.json | 11 ++++ fedot_ind/core/repository/model_repository.py | 5 +- fedot_ind/core/tuning/search_space.py | 24 ++++++++ tests/unit/core/models/test_teaser.py | 35 +++++++++++ 7 files changed, 130 insertions(+), 43 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/__init__.py create mode 100644 tests/unit/core/models/test_teaser.py diff --git a/fedot_ind/core/models/early_tc/__init__.py b/fedot_ind/core/models/early_tc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 66dc88745..66a031fd4 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -3,6 +3,7 @@ from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV from sktime.classification.dictionary_based import MUSE, WEASEL from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters @@ -13,21 +14,25 @@ def __init__(self, params: Optional[OperationParameters] = None): super().__init__() if params is None: params = {} + self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') self.interval_length = params.get('interval_length', 10) # rewrite as interval_length self.acceptance_threshold = params.get('acceptance_threshold', 5) self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' + # how to pass into ? % what needed - self.oc_svm_params = {} + self._oc_svm_params = [100, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1.5, 1] self.weasel_params = {} self.random_state = None # is needed? def _init_model(self, max_data_length): self.prediction_idx = self._compute_prediction_points(max_data_length) self.n_pred = len(self.prediction_idx) - self.oc_estimators = [OneClassSVM(**self.oc_svm_params) for _ in range(self.n_pred)] + self.oc_estimators = [None] * self.n_pred self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] - self.scalers = [StandardScaler() for _ in range(self.n_pred)] # do we need them separate? no inverse path expected + self.scalers = [StandardScaler() for _ in range(self.n_pred)] + self.__offset = max_data_length % self.interval_length + self.best_estimator_idx = -1 def fit(self, input_data: InputData): input_data = self.__convert_pd(input_data) @@ -35,17 +40,22 @@ def fit(self, input_data: InputData): self._init_model(max_data_length=X.shape[-1]) for i in range(self.n_pred): self._fit_one_interval(X, y, i) + self.best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i]] # what's dimensionality of input? will it work in case of multivariate? + X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? X_part = self.scalers[i].fit_transform(X_part) probas = self.slave_estimators[i].fit_predict_proba(X_part, y) - filtered_probas = self._filter_positive(probas, y) # + filtered_probas = self._filter_trues(probas, y) # X_oc = self._form_X_oc(filtered_probas) - self.oc_estimators[i].fit(X_oc, y) + self.oc_estimators[i] = GridSearchCV(OneClassSVM(), + param_grid={"gamma": self._oc_svm_params}, + scoring='accuracy', + cv=min(X.shape[0], 10) + ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ - def _predict_one_slave(self, X, i): - X_part = X[..., :self.prediction_idx[i]] + def _predict_one_slave(self, X, i, offset=0): + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] X_part = self.scalers[i].transform(X_part) probas = self.slave_estimators[i].predict_proba(X_part) X_oc = self._form_X_oc(probas) @@ -57,7 +67,7 @@ def _compute_prediction_points(self, n_idx): self.earliness = 1 - prediction_idx / n_idx return prediction_idx - def _filter_positive(self, predicted_probas, y): # different logic in sktime + def _filter_trues(self, predicted_probas, y): # different logic in sktime predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() return predicted_probas[predicted_labels == y] @@ -70,15 +80,20 @@ def _form_X_oc(self, predicted_probas): def _predict(self, X): n = X.shape[0] self.states = np.ones((n, self.n_pred, 2)) # num_consec, class + if self.prediction_mode == 'best_by_harmonic_mean': + estimator_indices = [self.best_estimator_idx] + else: + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = list(range(last_idx + 1)) X_ocs, predicted_labels = zip( - *[self._predict_one_slave(X, i) for i in range(self.n_pred)] + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) predicted_labels = np.stack(predicted_labels) # for each point of estimation - for i in range(self.n_pred): + for i in range(predicted_labels.shape[0]): # find not accepted points ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] X_to_ith = X_ocs[i][ith_point_to_oc] @@ -87,15 +102,16 @@ def _predict(self, X): # mark as accepted non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False predicted_labels[non_acceptance] = -1 - return predicted_labels + return predicted_labels # prediction_points x n_instances def _consecutive_count(self, predicted_labels: List[np.array]): n = len(predicted_labels[0]) - consecutive_labels = np.ones((self.n_pred, n)) - for i in range(1, self.n_pred): + prediction_points = len(predicted_labels) + consecutive_labels = np.ones((prediction_points, n)) + for i in range(1, prediction_points): equal = predicted_labels[i - 1] == predicted_labels[i] consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 - return consecutive_labels # n_pred x n_instances + return consecutive_labels # prediction_points x n_instances def __convert_pd(self, input_data): if hasattr(input_data.features, 'values'): @@ -115,9 +131,14 @@ def predict_for_fit(self, input_data: InputData) -> OutputData: def _score(self, X, y, hm_shift_to_acc=None): hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc predictions = self._predict(X) - accuracies = (predictions == np.tile(y, (1, self.n_pred))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness / (hm_shift_to_acc * accuracies + self.earliness) + prediction_points = predictions.shape[0] + accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - def _tune_oc(self): - #TODO - pass + def _get_applicable_index(self, last_available_idx): + idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') + if idx == 0: + raise RuntimeError('Too few points for prediction!') + idx -= 1 + offset = last_available_idx - self.prediction_idx[idx] + return idx, offset diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index ac513600e..1bbcc8614 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -31,6 +31,10 @@ "activation": "Softmax", "num_classes": 1 }, + "deepar_model": { + "epochs": 100, + "batch_size": 16 + }, "inception_model": { "epochs": 100, "batch_size": 32, @@ -43,6 +47,11 @@ "activation": "Softmax", "model_name": "ResNet18" }, + "tcn_model": { + "epochs": 100, + "batch_size": 32, + "activation": "ReLU" + }, "ssa_forecaster": { "window_size_method": "hac", "history_lookback": 30 @@ -121,6 +130,11 @@ "min_samples_leaf": 10, "bootstrap": false }, + "teaser": { + "interval_length": 10, + "acceptance_threshold": 3, + "hm_shift_to_acc": 2 + }, "dt": { "max_depth": 5, "min_samples_split": 10, @@ -344,27 +358,6 @@ "timeout": 10, "with_tuning": true }, -<<<<<<< HEAD - "minirocket_extractor": { - "num_features": 10000 - }, - "chronos_extractor": { - "num_features": 10000 - }, - "inception_model": { - "epochs": 100, - "batch_size": 32 - }, - "omniscale_model": { - "epochs": 100, - "batch_size": 32 - }, - "deepar_model": { - "epochs": 100, - "batch_size": 16 - }, -======= ->>>>>>> c5c358ba8f9b87b626014d2da9b2135c82684258 "tst_model": { "epochs": 100, "batch_size": 32 @@ -402,4 +395,4 @@ "max_homology_dimension": 1, "metric": "euclidean" } -} \ No newline at end of file +} diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 3f5c0a10f..8fb12f6de 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -617,6 +617,17 @@ "non_linear" ] }, + "teaser": { + "meta": "ts_model", + "presets": ["fast_train", "ts"], + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.ts]" + }, "xgboost": { "meta": "sklearn_class", "presets": ["*tree"], diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index 69e47be3d..a984cbd38 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -44,6 +44,7 @@ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from xgboost import XGBRegressor +from fedot_ind.core.models.early_tc.teaser import TEASER from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR @@ -132,7 +133,9 @@ class AtomizedModel(Enum): # solo nn models 'mlp': MLPClassifier, # external models - 'lgbm': LGBMClassifier + 'lgbm': LGBMClassifier, + # Early classification + 'teaser': TEASER } FEDOT_PREPROC_MODEL = { # data standartization diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 82e798059..37849840d 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -60,6 +60,30 @@ 'selection_strategy': {'hyperopt-dist': hp.choice, 'sampling-scope': [['sum', 'pairwise']]} }, + 'teaser': + {'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'acceptance_threshold': {'hyperopt-dist': hp.choice, + 'sampling_scope': [[1, 2, 3, 4, 5]]}, + 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + }, + 'deepar_model': + {'epochs': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[x for x in range(10, 100, 10)]]}, + 'batch_size': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[x for x in range(8, 64, 6)]]}, + 'dropout': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(0, 0.6, 0.1))]}, + 'rnn_layers':{'hyperopt-dist': hp.choice, + 'sampling-scope': [range(1, 6)]}, + 'hidden_size':{'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(10, 101, 10))]}, + 'cell_type':{'hyperopt-dist': hp.choice, + 'sampling-scope': [['GRU', 'LSTM', 'RNN']]}, + 'expected_distribution': {'hyperopt-dist': hp.choice, + 'sampling-scope': [['normal', 'cauchy']]} + }, 'patch_tst_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}, diff --git a/tests/unit/core/models/test_teaser.py b/tests/unit/core/models/test_teaser.py new file mode 100644 index 000000000..2bc19b8de --- /dev/null +++ b/tests/unit/core/models/test_teaser.py @@ -0,0 +1,35 @@ +import pytest +import numpy as np +from fedot_ind.core.models.early_tc import teaser as TEASER + + +@pytest.fixture(scope='module') +def teaser(): + teaser = TEASER.TEASER({'interval_length': 10, 'prediction_mode': ''}) + return teaser + +@pytest.fixture(scope='module') +def xy(): + return np.random.randn((2, 23)), np.random.randint(0, 2, size=(2, 1)) + +def test_get_applicable_index(teaser): + teaser._init_model(23) + idx, offset = teaser._get_last_applicable_idx(100) + assert offset == 100 - 22, 'Wrong offset estimation when right edge' + assert idx == len(teaser.prediction_idx) - 1 + idx, offset = teaser._get_last_applicable_idx(12) + assert offset == 100 - teaser.prediction_idx[idx], 'Wrong offset estimation in the middle' + assert idx == len(teaser.prediction_idx) - 1 + +def test_compute_prediction_points(teaser): + indices = teaser._compute_prediction_points(23) + assert 2 in indices + assert 22 in indices + assert 23 not in indices + +# def test_consecutive_count(teaser): +# pass + +# def test_score(teaser): + + From 4204d6a259192c5f67a4fbde023cb651f7858d34 Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 27 Jun 2024 18:22:34 +0300 Subject: [PATCH 05/43] class tree reconf. added proba_thresholding classifier (not registered) --- .../core/models/early_tc/base_early_tc.py | 115 +++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 48 +++++++ fedot_ind/core/models/early_tc/teaser.py | 119 +++++------------- 3 files changed, 193 insertions(+), 89 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/base_early_tc.py create mode 100644 fedot_ind/core/models/early_tc/prob_threshold.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py new file mode 100644 index 000000000..094d895c3 --- /dev/null +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -0,0 +1,115 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum + + +class BaseETC(ClassifierMixin, BaseEstimator): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') + self.interval_percentage = params.get('interval_percentage', 10) # rewrite as interval_length + self.acceptance_threshold = params.get('acceptance_threshold', 5) + self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + assert self.acceptance_threshold < self.interval_percentage, 'Not enough checkpoints for prediction proof' + + def _init_model(self, X, y): + max_data_length = X.shape[-1] + self.prediction_idx = self._compute_prediction_points(max_data_length) + self.n_pred = len(self.prediction_idx) + self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] + self.scalers = [StandardScaler() for _ in range(self.n_pred)] + self._best_estimator_idx = -1 + self.classes_ = [[-1, *np.unique(y)]] + + @property + def required_length(self): + if not hasattr(self, '_best_estimator_idx'): + return None + return self.prediction_idx[self._best_estimator_idx] + + def fit(self, X, y=None): + assert y is not None, 'Pass y' + y = np.array(y).flatten() + self._init_model(X, y) + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + self._best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? + X_part = self.scalers[i].fit_transform(X_part) + probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + return probas + + def _predict_one_slave(self, X, i, offset=0): + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + return probas, np.argmax(probas, axis=-1) + + def _compute_prediction_points(self, n_idx): + interval_length = int(n_idx * self.interval_percentage / 100) + prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1] + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + return prediction_idx + + def _select_estimators(self, X): + offset = 0 + if self.prediction_mode == 'best_by_harmonic_mean': + estimator_indices = [self._best_estimator_idx] + elif self.prediction_mode == 'all': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = np.arange(last_idx + 1) + else: + raise ValueError('Unknown prediction mode') + return estimator_indices, offset + + def _predict(self, X,): + estimator_indices, offset = self._select_estimators(X) + predicted_probas, predicted_labels = zip( + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary + ) + return predicted_labels, predicted_probas + + def _consecutive_count(self, predicted_labels: List[np.array]): + n = len(predicted_labels[0]) + prediction_points = len(predicted_labels) + consecutive_labels = np.ones((prediction_points, n)) + for i in range(1, prediction_points): + equal = predicted_labels[i - 1] == predicted_labels[i] + consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 + return consecutive_labels # prediction_points x n_instances + + def predict_proba(self, X): + raise NotImplementedError + + def predict(self, X): + raise NotImplementedError + + def _score(self, X, y, hm_shift_to_acc=None): + y = np.array(y).flatten() + hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + predictions, *_ = self._predict(X) + prediction_points = predictions.shape[0] + accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) + + def _get_applicable_index(self, last_available_idx): + idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') + if idx == 0: + raise RuntimeError('Too few points for prediction!') + idx -= 1 + offset = last_available_idx - self.prediction_idx[idx] + return idx, offset diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py new file mode 100644 index 000000000..6e9c487c1 --- /dev/null +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -0,0 +1,48 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC + +class ProbabilityThresholdClassifier(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + self.probability_threshold = params.get('probability_threshold', 0.85) + + def predict_proba(self, X): + _, predicted_probas, non_acceptance = self._predict(X) + predicted_probas[non_acceptance] = 0 + return predicted_probas.squeeze() + + def predict(self, X): + predicted_labels, _, non_acceptance = self._predict(X) + predicted_labels[non_acceptance] = -1 + # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] + return predicted_labels # prediction_points x n_instances + + def _predict(self, X): + predicted_labels, predicted_probas = super()._predict(X) + non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold + to_second_check = np.argwhere(non_acceptance) + predicted_probas = np.stack(predicted_probas) + predicted_labels = np.stack(predicted_labels) + # for each point of estimation + for i in range(predicted_labels.shape[0]): + # find not accepted points + ith_point_to_oc = to_second_check[to_second_check[:, 0] == i, 1] + # if they are not outliers + final_verdict = (predicted_probas[i, ith_point_to_oc] > self.acceptance_threshold).any() + # mark as accepted + non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False + return predicted_labels, predicted_probas, non_acceptance diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 66a031fd4..0f08a9063 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,51 +1,32 @@ from typing import Union, List, Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator from sktime.classification.dictionary_based import MUSE, WEASEL from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC -class TEASER(ModelImplementation): +class TEASER(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): - super().__init__() - if params is None: - params = {} - self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') - self.interval_length = params.get('interval_length', 10) # rewrite as interval_length - self.acceptance_threshold = params.get('acceptance_threshold', 5) - self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) - assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' - - # how to pass into ? % what needed - self._oc_svm_params = [100, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1.5, 1] + super().__init__(params) + self._oc_svm_params = (100., 10., 5., 2.5, 1.5, 1., 0.5, 0.25, 0.1) self.weasel_params = {} self.random_state = None # is needed? - def _init_model(self, max_data_length): - self.prediction_idx = self._compute_prediction_points(max_data_length) - self.n_pred = len(self.prediction_idx) + def _init_model(self, X, y): + super()._init_model(X, y) self.oc_estimators = [None] * self.n_pred - self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] - self.scalers = [StandardScaler() for _ in range(self.n_pred)] - self.__offset = max_data_length % self.interval_length - self.best_estimator_idx = -1 - - def fit(self, input_data: InputData): - input_data = self.__convert_pd(input_data) - X, y = input_data.features, input_data.target # what's passed in case of classification to training? - self._init_model(max_data_length=X.shape[-1]) - for i in range(self.n_pred): - self._fit_one_interval(X, y, i) - self.best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? - X_part = self.scalers[i].fit_transform(X_part) - probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + probas = super()._fit_one_interval(X, y, i) filtered_probas = self._filter_trues(probas, y) # X_oc = self._form_X_oc(filtered_probas) self.oc_estimators[i] = GridSearchCV(OneClassSVM(), @@ -55,17 +36,9 @@ def _fit_one_interval(self, X, y, i): ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ def _predict_one_slave(self, X, i, offset=0): - X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] - X_part = self.scalers[i].transform(X_part) - probas = self.slave_estimators[i].predict_proba(X_part) + probas, labels = super()._predict_one_slave(X, i, offset) X_oc = self._form_X_oc(probas) - return X_oc, np.argmax(probas, axis=-1) - - def _compute_prediction_points(self, n_idx): - """Computes indices for prediction, includes last index, first interval may be greater""" - prediction_idx = np.arange(n_idx - 1, -1, -self.interval_length)[::-1] - self.earliness = 1 - prediction_idx / n_idx - return prediction_idx + return X_oc, probas, labels def _filter_trues(self, predicted_probas, y): # different logic in sktime predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() @@ -78,67 +51,35 @@ def _form_X_oc(self, predicted_probas): return np.hstack([predicted_probas, d]) def _predict(self, X): - n = X.shape[0] - self.states = np.ones((n, self.n_pred, 2)) # num_consec, class - if self.prediction_mode == 'best_by_harmonic_mean': - estimator_indices = [self.best_estimator_idx] - else: - last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) - estimator_indices = list(range(last_idx + 1)) - X_ocs, predicted_labels = zip( + estimator_indices, offset = self._select_estimators(X) + X_ocs, predicted_probas, predicted_labels = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) + predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) + final_verdicts = np.zeros((len(estimator_indices), X.shape[0])) # for each point of estimation for i in range(predicted_labels.shape[0]): # find not accepted points ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] X_to_ith = X_ocs[i][ith_point_to_oc] # if they are not outliers - final_verdict = self.oc_estimators[i].predict(X_to_ith) # 1 for accept -1 for reject + final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject # mark as accepted - non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False + non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False + final_verdicts[i] = final_verdict + return predicted_labels, predicted_probas, non_acceptance, final_verdicts + + def predict_proba(self, X): + _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) + predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] + return predicted_probas.squeeze() + + def predict(self, X): + predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) predicted_labels[non_acceptance] = -1 + # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - - def _consecutive_count(self, predicted_labels: List[np.array]): - n = len(predicted_labels[0]) - prediction_points = len(predicted_labels) - consecutive_labels = np.ones((prediction_points, n)) - for i in range(1, prediction_points): - equal = predicted_labels[i - 1] == predicted_labels[i] - consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 - return consecutive_labels # prediction_points x n_instances - - def __convert_pd(self, input_data): - if hasattr(input_data.features, 'values'): - input_data.features = input_data.features.values - if hasattr(input_data.target, 'values'): - input_data.target = input_data.target.values - return input_data - - def predict(self, input_data: InputData) -> OutputData: - input_data = self.__convert_pd(input_data) - prediction = self._predict(input_data.features) - return self._convert_to_output(input_data, predict=prediction) - - def predict_for_fit(self, input_data: InputData) -> OutputData: - return self.predict(input_data) - - def _score(self, X, y, hm_shift_to_acc=None): - hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc - predictions = self._predict(X) - prediction_points = predictions.shape[0] - accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - - def _get_applicable_index(self, last_available_idx): - idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') - if idx == 0: - raise RuntimeError('Too few points for prediction!') - idx -= 1 - offset = last_available_idx - self.prediction_idx[idx] - return idx, offset From 5ac6f70fb514db63a4a146520852c80e5c1b25e8 Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 27 Jun 2024 18:22:34 +0300 Subject: [PATCH 06/43] class tree reconf. added proba_thresholding classifier (not registered) --- .../core/models/early_tc/base_early_tc.py | 117 +++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 46 +++++++ fedot_ind/core/models/early_tc/teaser.py | 123 +++++------------- 3 files changed, 194 insertions(+), 92 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/base_early_tc.py create mode 100644 fedot_ind/core/models/early_tc/prob_threshold.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py new file mode 100644 index 000000000..f97ba0593 --- /dev/null +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -0,0 +1,117 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum + + +class BaseETC(ClassifierMixin, BaseEstimator): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') + self.interval_percentage = params.get('interval_percentage', 10) + self.consecutive_predictions = params.get('consecutive_predictions', 3) + self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + self.random_state = params.get('random_state', None) + self.weasel_params = {} + assert self.consecutive_predictions < self.interval_percentage, 'Not enough checkpoints for prediction proof' + + def _init_model(self, X, y): + max_data_length = X.shape[-1] + self.prediction_idx = self._compute_prediction_points(max_data_length) + self.n_pred = len(self.prediction_idx) + self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] + self.scalers = [StandardScaler() for _ in range(self.n_pred)] + self._best_estimator_idx = -1 + self.classes_ = [np.unique(y)] + + @property + def required_length(self): + if not hasattr(self, '_best_estimator_idx'): + return None + return self.prediction_idx[self._best_estimator_idx] + + def fit(self, X, y=None): + assert y is not None, 'Pass y' + y = np.array(y).flatten() + self._init_model(X, y) + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + self._best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? + X_part = self.scalers[i].fit_transform(X_part) + probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + return probas + + def _predict_one_slave(self, X, i, offset=0): + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + return probas, np.argmax(probas, axis=-1) + + def _compute_prediction_points(self, n_idx): + interval_length = int(n_idx * self.interval_percentage / 100) + prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1] + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + return prediction_idx + + def _select_estimators(self, X): + offset = 0 + if self.prediction_mode == 'best_by_harmonic_mean': + estimator_indices = [self._best_estimator_idx] + elif self.prediction_mode == 'all': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = np.arange(last_idx + 1) + else: + raise ValueError('Unknown prediction mode') + return estimator_indices, offset + + def _predict(self, X,): + estimator_indices, offset = self._select_estimators(X) + predicted_probas, predicted_labels = zip( + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary + ) + return predicted_labels, predicted_probas + + def _consecutive_count(self, predicted_labels: List[np.array]): + n = len(predicted_labels[0]) + prediction_points = len(predicted_labels) + consecutive_labels = np.ones((prediction_points, n)) + for i in range(1, prediction_points): + equal = predicted_labels[i - 1] == predicted_labels[i] + consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 + return consecutive_labels # prediction_points x n_instances + + def predict_proba(self, X): + raise NotImplementedError + + def predict(self, X): + raise NotImplementedError + + def _score(self, X, y, hm_shift_to_acc=None): + y = np.array(y).flatten() + hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + predictions, *_ = self._predict(X) + prediction_points = predictions.shape[0] + accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) + + def _get_applicable_index(self, last_available_idx): + idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') + if idx == 0: + raise RuntimeError('Too few points for prediction!') + idx -= 1 + offset = last_available_idx - self.prediction_idx[idx] + return idx, offset diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py new file mode 100644 index 000000000..0433de34a --- /dev/null +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -0,0 +1,46 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC + +class ProbabilityThresholdClassifier(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + self.probability_threshold = params.get('probability_threshold', None) + + def _init_model(self, X, y): + super()._init_model(X, y) + if self.probability_threshold is None: + self.probability_threshold = 1 / len(self.classes_[0]) + + def predict_proba(self, X): + _, predicted_probas, non_acceptance = self._predict(X) + predicted_probas[non_acceptance] = 0 + return predicted_probas.squeeze() + + def predict(self, X): + predicted_labels, _, non_acceptance = self._predict(X) + predicted_labels[non_acceptance] = -1 + # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] + return predicted_labels # prediction_points x n_instances + + def _predict(self, X): + predicted_labels, predicted_probas = super()._predict(X) + predicted_probas = np.stack(predicted_probas) + predicted_labels = np.stack(predicted_labels) + non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions + double_check = predicted_probas.max(axis=-1) > self.probability_threshold + non_acceptance[non_acceptance & double_check] = False + return predicted_labels, predicted_probas, non_acceptance diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 66a031fd4..2809824c8 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,51 +1,30 @@ from typing import Union, List, Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator from sktime.classification.dictionary_based import MUSE, WEASEL from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC -class TEASER(ModelImplementation): +class TEASER(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): - super().__init__() - if params is None: - params = {} - self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') - self.interval_length = params.get('interval_length', 10) # rewrite as interval_length - self.acceptance_threshold = params.get('acceptance_threshold', 5) - self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) - assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' + super().__init__(params) + self._oc_svm_params = (100., 10., 5., 2.5, 1.5, 1., 0.5, 0.25, 0.1) - # how to pass into ? % what needed - self._oc_svm_params = [100, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1.5, 1] - self.weasel_params = {} - self.random_state = None # is needed? - - def _init_model(self, max_data_length): - self.prediction_idx = self._compute_prediction_points(max_data_length) - self.n_pred = len(self.prediction_idx) + def _init_model(self, X, y): + super()._init_model(X, y) self.oc_estimators = [None] * self.n_pred - self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] - self.scalers = [StandardScaler() for _ in range(self.n_pred)] - self.__offset = max_data_length % self.interval_length - self.best_estimator_idx = -1 - - def fit(self, input_data: InputData): - input_data = self.__convert_pd(input_data) - X, y = input_data.features, input_data.target # what's passed in case of classification to training? - self._init_model(max_data_length=X.shape[-1]) - for i in range(self.n_pred): - self._fit_one_interval(X, y, i) - self.best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? - X_part = self.scalers[i].fit_transform(X_part) - probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + probas = super()._fit_one_interval(X, y, i) filtered_probas = self._filter_trues(probas, y) # X_oc = self._form_X_oc(filtered_probas) self.oc_estimators[i] = GridSearchCV(OneClassSVM(), @@ -55,17 +34,9 @@ def _fit_one_interval(self, X, y, i): ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ def _predict_one_slave(self, X, i, offset=0): - X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] - X_part = self.scalers[i].transform(X_part) - probas = self.slave_estimators[i].predict_proba(X_part) + probas, labels = super()._predict_one_slave(X, i, offset) X_oc = self._form_X_oc(probas) - return X_oc, np.argmax(probas, axis=-1) - - def _compute_prediction_points(self, n_idx): - """Computes indices for prediction, includes last index, first interval may be greater""" - prediction_idx = np.arange(n_idx - 1, -1, -self.interval_length)[::-1] - self.earliness = 1 - prediction_idx / n_idx - return prediction_idx + return X_oc, probas, labels def _filter_trues(self, predicted_probas, y): # different logic in sktime predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() @@ -78,67 +49,35 @@ def _form_X_oc(self, predicted_probas): return np.hstack([predicted_probas, d]) def _predict(self, X): - n = X.shape[0] - self.states = np.ones((n, self.n_pred, 2)) # num_consec, class - if self.prediction_mode == 'best_by_harmonic_mean': - estimator_indices = [self.best_estimator_idx] - else: - last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) - estimator_indices = list(range(last_idx + 1)) - X_ocs, predicted_labels = zip( + estimator_indices, offset = self._select_estimators(X) + X_ocs, predicted_probas, predicted_labels = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) - non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold + non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) + predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) + final_verdicts = np.zeros((len(estimator_indices), X.shape[0])) # for each point of estimation for i in range(predicted_labels.shape[0]): # find not accepted points ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] X_to_ith = X_ocs[i][ith_point_to_oc] # if they are not outliers - final_verdict = self.oc_estimators[i].predict(X_to_ith) # 1 for accept -1 for reject + final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject # mark as accepted - non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False + non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False + final_verdicts[i] = final_verdict + return predicted_labels, predicted_probas, non_acceptance, final_verdicts + + def predict_proba(self, X): + _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) + predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] + return predicted_probas.squeeze() + + def predict(self, X): + predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) predicted_labels[non_acceptance] = -1 + # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - - def _consecutive_count(self, predicted_labels: List[np.array]): - n = len(predicted_labels[0]) - prediction_points = len(predicted_labels) - consecutive_labels = np.ones((prediction_points, n)) - for i in range(1, prediction_points): - equal = predicted_labels[i - 1] == predicted_labels[i] - consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 - return consecutive_labels # prediction_points x n_instances - - def __convert_pd(self, input_data): - if hasattr(input_data.features, 'values'): - input_data.features = input_data.features.values - if hasattr(input_data.target, 'values'): - input_data.target = input_data.target.values - return input_data - - def predict(self, input_data: InputData) -> OutputData: - input_data = self.__convert_pd(input_data) - prediction = self._predict(input_data.features) - return self._convert_to_output(input_data, predict=prediction) - - def predict_for_fit(self, input_data: InputData) -> OutputData: - return self.predict(input_data) - - def _score(self, X, y, hm_shift_to_acc=None): - hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc - predictions = self._predict(X) - prediction_points = predictions.shape[0] - accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - - def _get_applicable_index(self, last_available_idx): - idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') - if idx == 0: - raise RuntimeError('Too few points for prediction!') - idx -= 1 - offset = last_available_idx - self.prediction_idx[idx] - return idx, offset From 48e83289bc8bfe0b22bbca86e9be286311522e93 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 28 Jun 2024 14:11:17 +0300 Subject: [PATCH 07/43] both etc models are registered, available via api --- fedot_ind/core/models/early_tc/base_early_tc.py | 11 ++--------- .../core/models/early_tc/prob_threshold.py | 12 +----------- fedot_ind/core/models/early_tc/teaser.py | 10 +--------- .../data/default_operation_params.json | 9 +++++++-- .../data/industrial_model_repository.json | 17 +++++++++++++---- fedot_ind/core/repository/model_repository.py | 4 +++- fedot_ind/core/tuning/search_space.py | 12 ++++++++++-- 7 files changed, 37 insertions(+), 38 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index f97ba0593..c7b84bedf 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -1,16 +1,9 @@ -from typing import Union, List, Optional +from typing import Optional, List from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data -from fedot.core.data.data import InputData, OutputData -from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import GridSearchCV from sklearn.base import ClassifierMixin, BaseEstimator -from sktime.classification.dictionary_based import MUSE, WEASEL -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from sktime.classification.dictionary_based import WEASEL from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum class BaseETC(ClassifierMixin, BaseEstimator): diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 0433de34a..343077cbe 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -1,16 +1,6 @@ -from typing import Union, List, Optional +from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data -from fedot.core.data.data import InputData, OutputData -from sklearn.svm import OneClassSVM -from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import GridSearchCV -from sklearn.base import ClassifierMixin, BaseEstimator -from sktime.classification.dictionary_based import MUSE, WEASEL -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot_ind.core.models.early_tc.base_early_tc import BaseETC class ProbabilityThresholdClassifier(BaseETC): diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 2809824c8..f5d2590b3 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,16 +1,8 @@ -from typing import Union, List, Optional +from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data -from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM -from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV -from sklearn.base import ClassifierMixin, BaseEstimator -from sktime.classification.dictionary_based import MUSE, WEASEL -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot_ind.core.models.early_tc.base_early_tc import BaseETC diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 1bbcc8614..52536d5cb 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -131,8 +131,13 @@ "bootstrap": false }, "teaser": { - "interval_length": 10, - "acceptance_threshold": 3, + "interval_percentage": 10, + "consecutive_predictions": 3, + "hm_shift_to_acc": 2 + }, + "proba_threshold_etc": { + "interval_percentage": 10, + "consecutive_predictions": 3, "hm_shift_to_acc": 2 }, "dt": { diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 8fb12f6de..309d5c56c 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -511,7 +511,7 @@ }, "ridge": { "meta": "sklearn_regr", - "presets": ["fast_train", "ts"], + "presets": ["fast_train"], "tags": [ "simple", "linear", @@ -618,15 +618,24 @@ ] }, "teaser": { - "meta": "ts_model", - "presets": ["fast_train", "ts"], + "meta": "sklearn_class", "tags": [ "simple", "interpretable", "non_lagged", "non_linear" ], - "input_type": "[DataTypesEnum.ts]" + "input_type": "[DataTypesEnum.table]" + }, + "proba_threshold_etc": { + "meta": "sklearn_class", + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" }, "xgboost": { "meta": "sklearn_class", diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index a984cbd38..908622195 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -45,6 +45,7 @@ from xgboost import XGBRegressor from fedot_ind.core.models.early_tc.teaser import TEASER +from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR @@ -135,7 +136,8 @@ class AtomizedModel(Enum): # external models 'lgbm': LGBMClassifier, # Early classification - 'teaser': TEASER + 'teaser': TEASER, + 'proba_threshold_etc': ProbabilityThresholdClassifier } FEDOT_PREPROC_MODEL = { # data standartization diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 37849840d..11be89db9 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -68,15 +68,23 @@ 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, }, + 'proba_threshold_etc': + {'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'acceptance_threshold': {'hyperopt-dist': hp.choice, + 'sampling_scope': [[1, 2, 3, 4, 5]]}, + 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + }, 'deepar_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}, 'dropout': {'hyperopt-dist': hp.choice, - 'sampling-scope': [list(range(0, 0.6, 0.1))]}, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, 'rnn_layers':{'hyperopt-dist': hp.choice, - 'sampling-scope': [range(1, 6)]}, + 'sampling-scope': [list(range(1, 6))]}, 'hidden_size':{'hyperopt-dist': hp.choice, 'sampling-scope': [list(range(10, 101, 10))]}, 'cell_type':{'hyperopt-dist': hp.choice, From 18a895ae8440910ee008b3fb498d89bf842c19b4 Mon Sep 17 00:00:00 2001 From: leostre Date: Tue, 2 Jul 2024 15:27:34 +0300 Subject: [PATCH 08/43] ecec added --- .../core/models/early_tc/base_early_tc.py | 22 ++++--- fedot_ind/core/models/early_tc/ecec.py | 62 +++++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 19 ++++-- fedot_ind/core/models/early_tc/teaser.py | 9 +++ 4 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/ecec.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index c7b84bedf..5da8e61bd 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -33,6 +33,10 @@ def required_length(self): if not hasattr(self, '_best_estimator_idx'): return None return self.prediction_idx[self._best_estimator_idx] + + @property + def n_classes(self): + return len(self.classes_[0]) def fit(self, X, y=None): assert y is not None, 'Pass y' @@ -40,7 +44,6 @@ def fit(self, X, y=None): self._init_model(X, y) for i in range(self.n_pred): self._fit_one_interval(X, y, i) - self._best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? @@ -60,23 +63,23 @@ def _compute_prediction_points(self, n_idx): self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 return prediction_idx - def _select_estimators(self, X): + def _select_estimators(self, X, training=False): offset = 0 - if self.prediction_mode == 'best_by_harmonic_mean': + if not training and self.prediction_mode == 'best_by_harmonic_mean': estimator_indices = [self._best_estimator_idx] - elif self.prediction_mode == 'all': + elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) else: raise ValueError('Unknown prediction mode') return estimator_indices, offset - def _predict(self, X,): - estimator_indices, offset = self._select_estimators(X) - predicted_probas, predicted_labels = zip( + def _predict(self, X, training=True): + estimator_indices, offset = self._select_estimators(X, training) + prediction = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) - return predicted_labels, predicted_probas + return prediction # see the output in _predict_one_slave def _consecutive_count(self, predicted_labels: List[np.array]): n = len(predicted_labels[0]) @@ -96,11 +99,12 @@ def predict(self, X): def _score(self, X, y, hm_shift_to_acc=None): y = np.array(y).flatten() hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc - predictions, *_ = self._predict(X) + predictions = self._predict(X)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) + def _get_applicable_index(self, last_available_idx): idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') if idx == 0: diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py new file mode 100644 index 000000000..a00df631d --- /dev/null +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -0,0 +1,62 @@ +from typing import Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from sklearn.model_selection import cross_val_predict +from sklearn.base import clone +from sklearn.metrics import confusion_matrix + +class ECEC(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + super().__init__(params) + + def _init_model(self, X, y): + super()._init_model(X, y) + self._confidences = np.ones((X.shape[0], self.n_pred)) + + def _score(self, X, y, alpha): + y = y.astype(int) + predicted_labels = np.stack(super()._predict(X)[0]).astype(int) # n_pred x n_inst + n = predicted_labels.shape[0] + accuracies = (predicted_labels == np.tile(y, (1, n))) # n_pred x n_inst + confidences = np.ones((n, X.shape[0]), dtype='float32') + for i in range(n): + y_pred = predicted_labels[i] + reliability_i = confusion_matrix(y, y_pred, normalize='pred') + confidences[i] = 1 - reliability_i[y, y_pred] # n_inst + confidences = 1 - np.cumprod(confidences, axis=0) # n_pred x n_inst + candidates = self._select_thrs(confidences) # n_candidates + cfs = np.zeros_like(candidates) + for i, candidate in enumerate(candidates): + mask = confidences >= candidate # n_pred x n_inst + accuracy_for_candidate = (accuracies * mask).sum(1) / mask.sum(1) # n_pred + cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) + return candidates[np.argmin(cfs)] + + @staticmethod + def _select_thrs(confidences): + C = np.unique(confidences.round(3)) + difference = np.diff(C) + pair_means = C[:-1] + difference / 2 + difference_shifted = np.roll(difference, 1) + difference_idx = np.argwhere(difference > difference_shifted) + return pair_means[difference_idx].flatten() + + @staticmethod + def cost_func(earliness, accuracies, alpha): + return alpha * accuracies + (1 - alpha) * earliness + + def fit(self, X, y): + self.confidence_threshold = super().fit(X, y) + + + + + + + + + + + + diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 47fb76142..bfcccace5 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -21,16 +21,27 @@ def predict_proba(self, X): return predicted_probas.squeeze() def predict(self, X): - predicted_labels, _, non_acceptance = self._predict(X) + predicted_labels, _, non_acceptance = self._predict(X, training=False) predicted_labels[non_acceptance] = -1 # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - def _predict(self, X): - predicted_labels, predicted_probas = super()._predict(X) + def _predict(self, X, training=True): + predicted_probas, predicted_labels = super()._predict(X, training) predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) + # print(predicted_labels.shape, predicted_probas.shape) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions double_check = predicted_probas.max(axis=-1) > self.probability_threshold non_acceptance[non_acceptance & double_check] = False - return predicted_labels, predicted_probas, non_acceptance \ No newline at end of file + return predicted_labels, predicted_probas, non_acceptance + + def _score(self, X, y, hm_shift_to_acc=None): + scores = super()._score(X, y, hm_shift_to_acc) + self._best_estimator_idx = np.argmax(scores) + return scores + + def fit(self, X, y): + super().fit(X, y) + return self._score(X, y, self.hm_shift_to_acc) + diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 713104099..5ed6accbe 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -75,3 +75,12 @@ def predict(self, X): predicted_labels[non_acceptance] = -1 # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances + + def _score(self, X, y, hm_shift_to_acc=None): + scores = super()._score(X, y, hm_shift_to_acc) + self._best_estimator_idx = np.argmax(scores) + return scores + + def fit(self, X, y): + super().fit(X, y) + return self._score(X, y, self.hm_shift_to_acc) From 0c10a07f8ec84bad39728589a07e751e814ffb47 Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 4 Jul 2024 18:23:13 +0300 Subject: [PATCH 09/43] economy_k added --- .../core/models/early_tc/base_early_tc.py | 9 +- fedot_ind/core/models/early_tc/economy_k.py | 89 +++++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 6 +- fedot_ind/core/models/early_tc/teaser.py | 6 +- 4 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/economy_k.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 5da8e61bd..5e180c2a9 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -14,7 +14,8 @@ def __init__(self, params: Optional[OperationParameters] = None): self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') self.interval_percentage = params.get('interval_percentage', 10) self.consecutive_predictions = params.get('consecutive_predictions', 3) - self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + self.accuracy_importance = params.get('accuracy_importance', 1.) + self.min_ts_length = params.get('min_ts_step', 3) self.random_state = params.get('random_state', None) self.weasel_params = {} assert self.consecutive_predictions < self.interval_percentage, 'Not enough checkpoints for prediction proof' @@ -58,8 +59,8 @@ def _predict_one_slave(self, X, i, offset=0): return probas, np.argmax(probas, axis=-1) def _compute_prediction_points(self, n_idx): - interval_length = int(n_idx * self.interval_percentage / 100) - prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1] + interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) + prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1][1:] self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 return prediction_idx @@ -98,7 +99,7 @@ def predict(self, X): def _score(self, X, y, hm_shift_to_acc=None): y = np.array(y).flatten() - hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + hm_shift_to_acc = hm_shift_to_acc or self.accuracy_importance predictions = self._predict(X)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py new file mode 100644 index 000000000..639e680cd --- /dev/null +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -0,0 +1,89 @@ +from typing import Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from sklearn.cluster import KMeans +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import train_test_split, cross_val_predict + +class EconomyK(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__(params) + self.lambda_ = params.get('lambda', 1.) + self._cluster_factor = params.get('cluster_factor' , 1) + self._random_state = 2104 + self.__cv = 5 + + def _init_model(self, X, y): + super()._init_model(X, y) + self.n_clusters = int(self._cluster_factor * self.n_classes) + self._clusterizer = KMeans(self.n_clusters, random_state=self._random_state) + self.state = np.zeros((self.n_pred, self.n_clusters, self.n_classes, self.n_classes)) + + def fit(self, X, y): + y = y.flatten().astype(int) + self._init_model(X, y) + self._pyck_ = confusion_matrix(y, self._clusterizer.fit(X).labels_, normalize='true')[:self.n_classes, :self.n_clusters] + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] + X_part = self.scalers[i].fit_transform(X_part) + y_pred = cross_val_predict(self.slave_estimators[i], X_part, y, cv=self.__cv) + self.slave_estimators[i].fit(X_part, y) + states_by_i = np.zeros(( self.n_clusters, self.n_classes, self.n_classes)) + np.add.at(states_by_i, (self._clusterizer.labels_, y, y_pred), 1) + states_by_i /= np.mean(states_by_i, -2, keepdims=True) + states_by_i[np.isnan(states_by_i)] = 0 + states_by_i[:, np.eye(self.n_classes).astype(bool)] = 0 + self.state[i] = states_by_i + + def _predict_one_slave(self, X, i, offset=0): + cluster_centers = self._clusterizer.cluster_centers_[:, :self.prediction_idx[i] + 1] # n_clust x len + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] # n_inst x len + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + optimal_time, is_optimal = self._get_prediction_time(X_part, cluster_centers, i) + return probas, optimal_time, is_optimal + + def __cluster_probas(self, X, centroids): + length = centroids.shape[-1] + diffs = np.subtract.outer(X, centroids).swapaxes(1, 2) + diffs = diffs[..., np.eye(length).astype(bool)] # n_inst x n_clust x len + distances = np.linalg.norm(diffs, axis=-1) + delta_k = 1. - distances / distances.mean(axis=-1)[:, None] + s = 1. / (1. + np.exp(-self.lambda_ * delta_k)) + return s / s.sum(axis=-1)[:, None] # n_inst x n_clust + + def __expected_costs(self, X, cluster_centroids, i): + cluster_probas = self.__cluster_probas(X, cluster_centroids) # n_inst x n_clust + s_glob = np.sum(np.transpose( + np.sum(self.state[i:], axis=-1), axes=(0, 2, 1) + ) * self._pyck_[None, ...], axis=1) + costs = cluster_probas @ s_glob.T # n_inst x time_left + costs += self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? + return costs + + def _get_prediction_time(self, X, cluster_centroids, i): + costs = self.__expected_costs(X, cluster_centroids, i) + min_costs = np.argmin(costs, axis=-1) + is_optimal = min_costs == 0 + time_optimal = self.prediction_idx[min_costs + i] + return time_optimal, is_optimal # n_inst + + def predict_proba(self, X): + probas, times, is_optimal = self._predict(X) + is_optimal = np.stack(is_optimal) + idx = np.tile(np.arange(self.n_pred), (is_optimal.shape[1], 1)).T # n_pred x n_inst + idx[~is_optimal] = self.n_pred + idx = np.argmin(idx, 0) + probas = np.stack(probas) + return probas[idx], np.stack(times)[idx] + + def predict(self, X): + probas, times = self.predict_proba(X) + labels = probas.argmax(-1) + return labels, times diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index bfcccace5..51d169909 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -36,12 +36,12 @@ def _predict(self, X, training=True): non_acceptance[non_acceptance & double_check] = False return predicted_labels, predicted_probas, non_acceptance - def _score(self, X, y, hm_shift_to_acc=None): - scores = super()._score(X, y, hm_shift_to_acc) + def _score(self, X, y, accuracy_importance=None): + scores = super()._score(X, y, accuracy_importance) self._best_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) - return self._score(X, y, self.hm_shift_to_acc) + return self._score(X, y, self.accuracy_importance) diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 5ed6accbe..593a34e72 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -76,11 +76,11 @@ def predict(self, X): # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - def _score(self, X, y, hm_shift_to_acc=None): - scores = super()._score(X, y, hm_shift_to_acc) + def _score(self, X, y, accuracy_importance=None): + scores = super()._score(X, y, accuracy_importance) self._best_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) - return self._score(X, y, self.hm_shift_to_acc) + return self._score(X, y, self.accuracy_importance) From 251bca6d7837fc29d68cdfef586304b5cfc06b59 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 5 Jul 2024 12:30:52 +0300 Subject: [PATCH 10/43] mlstm init --- .../core/models/nn/network_impl/mlstm.py | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 fedot_ind/core/models/nn/network_impl/mlstm.py diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py new file mode 100644 index 000000000..31ca6fbd4 --- /dev/null +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -0,0 +1,127 @@ +from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel +from typing import Optional, Callable, Any, List, Union +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.data.data import InputData, OutputData +from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE +import torch.optim as optim +import torch.nn as nn +import torch.nn.functional as F +import torch +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +import pandas as pd +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams +from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter +import torch.utils.data as data +from fedot_ind.core.architecture.settings.computational import default_device + +class SqueezeExciteBlock(nn.Module): + def __init__(self, input_channels, filters, reduce=4): + super().__init__() + self.filters = filters + self.pool = nn.AvgPool1d(input_channels) + self.bottleneck = max(self.filters // reduce, 4) + self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) + self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) + torch.nn.init.kaiming_normal_(self.fc1.weight.data) + torch.nn.init.kaiming_normal_(self.fc2.weight.data) + + def forward(self, x): + input_x = x + x = self.pool(x) + x = F.relu(self.fc1(x.view(-1, 1, self.filters))) + x = F.sigmoid(self.fc2(x)) + x = x.view(-1, self.filters, 1) * input_x + return x + +class MLSTM_module(nn.Module): + def __init__(self, input_size, input_channels, + inner_size, inner_channels, + output_size, num_layers, dropout=0.25): + super().__init__() + self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) + self.lstm = nn.LSTM(input_size, inner_size, num_layers, + batch_first=True, dropout=dropout) + self.conv_branch = nn.Sequential( + nn.Conv1d(input_channels, inner_channels, + padding='same', + kernel_size=9), + nn.BatchNorm1d(inner_channels), + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels), + nn.Conv1d(inner_channels, inner_channels * 2, + padding='same', + kernel_size=5, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels * 2), + nn.Conv1d(inner_channels * 2, inner_channels, + padding='same', + kernel_size=3, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels), # n x c | n x c x l + nn.ReLU(), + ) + seq = next(iter(self.conv_branch.modules())) + idx = [0, 4, 8] + for i in idx: + torch.nn.init.kaiming_uniform_(seq[i].weight.data) + + def forward(self, x): + x_lstm, _ = self.lstm(x) # n x input_ch x inner_size + x_conv = self.conv_branch(x) # n x inner_ch x len + print(x_conv.size(), x_lstm.size()) + x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) + x = F.softmax(self.proj(x)) + return x + + +class MLSTM(BaseNeuralModel): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + # self.num_classes = params.get('num_classes', None) + # self.epochs = params.get('epochs', 100) + # self.batch_size = params.get('batch_size', 16) + # self.activation = params.get('activation', 'ReLU') + # self.learning_rate = 0.001 + + self.dropout = params.get('dropout', 0.25) + self.hidden_size = params.get('hidden_size', 64) + self.hidden_channels = params.get('hidden_channels', 32) + self.num_layers = params.get('num_layers', 2) + # self.target = None + # self.task_type = None + + def _init_model(self, ts: InputData): + _, input_channels, input_size = ts.features.shape + self.model = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + self.model_for_inference = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + optimizer = optim.Adam(self.model.parameters(), lr=0.001) + if ts.num_classes == 2: + loss_fn = CROSS_ENTROPY() + else: + loss_fn = MULTI_CLASS_CROSS_ENTROPY() + return loss_fn, optimizer + + @convert_to_3d_torch_array + def _fit_model(self, ts: InputData): + loss_fn, optimizer = self._init_model(ts) + train_loader, val_loader = self._prepare_data(ts, split_data=True) + self._train_loop( + train_loader=train_loader, + val_loader=val_loader, + loss_fn=loss_fn, + optimizer=optimizer + ) + From ace6626cae8b40c6489db63c3fa5566805efa9a8 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 5 Jul 2024 12:30:52 +0300 Subject: [PATCH 11/43] mlstm registered --- .../architecture/abstraction/decorators.py | 4 +- fedot_ind/core/models/early_tc/ecec.py | 62 ++++--- .../core/models/nn/network_impl/mlstm.py | 154 ++++++++++++++++++ .../data/default_operation_params.json | 4 + .../data/industrial_model_repository.json | 6 + fedot_ind/core/repository/model_repository.py | 5 +- 6 files changed, 210 insertions(+), 25 deletions(-) diff --git a/fedot_ind/core/architecture/abstraction/decorators.py b/fedot_ind/core/architecture/abstraction/decorators.py index a9226ba0b..e34aa52a9 100644 --- a/fedot_ind/core/architecture/abstraction/decorators.py +++ b/fedot_ind/core/architecture/abstraction/decorators.py @@ -42,13 +42,13 @@ def decorated_func(self, *args): def convert_to_3d_torch_array(func): def decorated_func(self, *args): - init_data = args[0] + init_data, *args = args data = DataConverter(data=init_data).convert_to_torch_format() if isinstance(init_data, InputData): init_data.features = data else: init_data = data - return func(self, init_data, args[1]) + return func(self, init_data, *args) return decorated_func diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index a00df631d..45f7f9fe4 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -12,26 +12,53 @@ def __init__(self, params: Optional[OperationParameters] = None): def _init_model(self, X, y): super()._init_model(X, y) - self._confidences = np.ones((X.shape[0], self.n_pred)) + self._reliabilities = np.zeros((self.n_pred, self.n_classes, self.n_classes)) + + def _predict_one_slave(self, X, i, offset=0): + predicted_probas, predicted_labels = super()._predict_one_slave(X, i, offset) + reliabilities = self._reliabilities[i, predicted_labels, predicted_labels].flatten() # n_inst + return predicted_labels.astype(int), predicted_probas, reliabilities + + def _predict(self, X, training=False): + predicted_labels, predicted_probas, reliabilities = super()._predict(X, training) + reliabilities = np.stack(reliabilities) + confidences = 1 - np.cumprod(1 - reliabilities, axis=0) + non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] + return predicted_labels, predicted_probas, non_confident, confidences - def _score(self, X, y, alpha): + def predict(self, X): + predicted_labels, _, non_confident, confidences = self._predict(X) + predicted_labels = np.stack(predicted_labels) + predicted_labels[non_confident] = -1 + return predicted_labels, confidences + + def predict_proba(self, X): + _, predicted_probas, non_confident, confidences = self._predict(X) + predicted_probas = np.stack(predicted_probas) + predicted_probas[non_confident] = -1 + return predicted_probas, confidences + + def _score(self, X, y, alpha, training=False): y = y.astype(int) - predicted_labels = np.stack(super()._predict(X)[0]).astype(int) # n_pred x n_inst + predicted_labels, *_ = super()._predict(X, training) # n_pred x n_inst + predicted_labels = np.stack(predicted_labels) n = predicted_labels.shape[0] - accuracies = (predicted_labels == np.tile(y, (1, n))) # n_pred x n_inst + accuracies = (predicted_labels == np.tile(y, (n, 1))) # n_pred x n_inst confidences = np.ones((n, X.shape[0]), dtype='float32') for i in range(n): y_pred = predicted_labels[i] reliability_i = confusion_matrix(y, y_pred, normalize='pred') confidences[i] = 1 - reliability_i[y, y_pred] # n_inst + self._reliabilities[i] = reliability_i confidences = 1 - np.cumprod(confidences, axis=0) # n_pred x n_inst candidates = self._select_thrs(confidences) # n_candidates - cfs = np.zeros_like(candidates) + cfs = np.zeros((len(candidates), n)) for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst accuracy_for_candidate = (accuracies * mask).sum(1) / mask.sum(1) # n_pred cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) - return candidates[np.argmin(cfs)] + self._best_estimator_idx = np.argmin(cfs.mean(0)) + return candidates[np.argmin(cfs, axis=0)] # n_pred @staticmethod def _select_thrs(confidences): @@ -39,24 +66,15 @@ def _select_thrs(confidences): difference = np.diff(C) pair_means = C[:-1] + difference / 2 difference_shifted = np.roll(difference, 1) - difference_idx = np.argwhere(difference > difference_shifted) - return pair_means[difference_idx].flatten() + difference_idx = np.argwhere(difference <= difference_shifted) + means_candidates = pair_means[difference_idx].flatten() + return means_candidates if len(means_candidates) else C @staticmethod def cost_func(earliness, accuracies, alpha): - return alpha * accuracies + (1 - alpha) * earliness + return alpha * (1 - accuracies) + (1 - alpha) * earliness def fit(self, X, y): - self.confidence_threshold = super().fit(X, y) - - - - - - - - - - - - + super().fit(X, y) + self.confidence_thresholds = self._score(X, y, self.accuracy_importance, training=True) + \ No newline at end of file diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 31ca6fbd4..9a0a52ba6 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -125,3 +125,157 @@ def _fit_model(self, ts: InputData): optimizer=optimizer ) + +from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel +from typing import Optional, Callable, Any, List, Union +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.data.data import InputData, OutputData +from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE +import torch.optim as optim +import torch.nn as nn +import torch.nn.functional as F +import torch +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +import pandas as pd +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams +from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter +import torch.utils.data as data +from fedot_ind.core.architecture.settings.computational import default_device + +class SqueezeExciteBlock(nn.Module): + def __init__(self, input_channels, filters, reduce=4): + super().__init__() + self.filters = filters + self.pool = nn.AvgPool1d(input_channels) + self.bottleneck = max(self.filters // reduce, 4) + self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) + self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) + torch.nn.init.kaiming_normal_(self.fc1.weight.data) + torch.nn.init.kaiming_normal_(self.fc2.weight.data) + + def forward(self, x): + input_x = x + x = self.pool(x) + x = F.relu(self.fc1(x.view(-1, 1, self.filters))) + x = F.sigmoid(self.fc2(x)) + x = x.view(-1, self.filters, 1) * input_x + return x + +class MLSTM_module(nn.Module): + def __init__(self, input_size, input_channels, + inner_size, inner_channels, + output_size, num_layers, dropout=0.25): + super().__init__() + self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) + self.lstm = nn.LSTM(input_size, inner_size, num_layers, + batch_first=True, dropout=dropout) + self.conv_branch = nn.Sequential( + nn.Conv1d(input_channels, inner_channels, + padding='same', + kernel_size=9), + nn.BatchNorm1d(inner_channels), + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels), + nn.Conv1d(inner_channels, inner_channels * 2, + padding='same', + kernel_size=5, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels * 2), + nn.Conv1d(inner_channels * 2, inner_channels, + padding='same', + kernel_size=3, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels), # n x c | n x c x l + nn.ReLU(), + ) + seq = next(iter(self.conv_branch.modules())) + idx = [0, 4, 8] + for i in idx: + torch.nn.init.kaiming_uniform_(seq[i].weight.data) + + def forward(self, x, hidden_state=None): + x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size + x_conv = self.conv_branch(x) # n x inner_ch x len + x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) + x = F.softmax(self.proj(x)) + return x#, hidden_state + + def augment_zero_padding(self, X: torch.Tensor): + res = [] + for i in self.prediction_idx: + zeroed_X = X[...] + zeroed_X[..., i + 1:] = 0 + res.append(zeroed_X) + res = torch.concat(res, 0) + return res[torch.randperm(res.size(0)), ...] + +class MLSTM(BaseNeuralModel): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + # self.num_classes = params.get('num_classes', None) + # self.epochs = params.get('epochs', 100) + # self.batch_size = params.get('batch_size', 16) + # self.activation = params.get('activation', 'ReLU') + # self.learning_rate = 0.001 + + self.dropout = params.get('dropout', 0.25) + self.hidden_size = params.get('hidden_size', 64) + self.hidden_channels = params.get('hidden_channels', 32) + self.num_layers = params.get('num_layers', 2) + # self.target = None + # self.task_type = None + self.interval_percentage = params.get('interval_percentage', 10) + self.min_ts_length = params.get('min_ts_length', 5) + + def _compute_prediction_points(self, n_idx): + interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) + prediction_idx = np.arange(0, n_idx, interval_length) + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + return prediction_idx + + def _init_model(self, ts: InputData): + _, input_channels, input_size = ts.features.shape + self.prediction_idx = self._compute_prediction_points(input_size) + self.model = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + self.model_for_inference = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + optimizer = optim.Adam(self.model.parameters(), lr=0.001) + if ts.num_classes == 2: + loss_fn = CROSS_ENTROPY() + else: + loss_fn = MULTI_CLASS_CROSS_ENTROPY() + return loss_fn, optimizer + + def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): + return super()._train_loop(train_loader, val_loader, loss_fn, optimizer) + + @convert_to_3d_torch_array + def _fit_model(self, ts: InputData): + if isinstance(ts, torch.Tensor): + ts = self.augment_zero_padding(ts) + else: + print(type(ts)) + loss_fn, optimizer = self._init_model(ts) + train_loader, val_loader = self._prepare_data(ts, split_data=True) + self._train_loop( + train_loader=train_loader, + val_loader=val_loader, + loss_fn=loss_fn, + optimizer=optimizer + ) + + + + diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 52536d5cb..b875c3225 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -172,6 +172,10 @@ "learning_rate": "constant", "solver": "adam" }, + "mlstm_model": { + "epochs": 100, + "batch_size": 16 + }, "ar": { "lag_1": 7, "lag_2": 12, diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 309d5c56c..0823954a2 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -282,6 +282,12 @@ "automl" ] }, + "mlstm_model": { + "meta": "fedot_NN_classification", + "presets": ["ts"], + "tags": [], + "input_type": "[DataTypesEnum.table]" + }, "xcm_model": { "meta": "fedot_NN_classification", "presets": [ diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index 908622195..9777b81c1 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -53,6 +53,7 @@ from fedot_ind.core.models.nn.network_impl.inception import InceptionTimeModel from fedot_ind.core.models.nn.network_impl.lora_nn import LoraModel from fedot_ind.core.models.nn.network_impl.mini_rocket import MiniRocketExtractor +from fedot_ind.core.models.nn.network_impl.mlstm import MLSTM from fedot_ind.core.models.nn.network_impl.nbeats import NBeatsModel from fedot_ind.core.models.nn.network_impl.omni_scale import OmniScaleModel from fedot_ind.core.models.nn.network_impl.resnet import ResNetModel @@ -221,7 +222,9 @@ class AtomizedModel(Enum): # linear_dummy_model 'dummy': DummyOverComplicatedNeuralNetwork, # linear_dummy_model - 'lora_model': LoraModel + 'lora_model': LoraModel, + # early ts classification + 'mlstm_model': MLSTM } From 42ba3f00fee57838534e5f48536e4295ca4773ce Mon Sep 17 00:00:00 2001 From: leostre Date: Tue, 9 Jul 2024 14:33:24 +0300 Subject: [PATCH 12/43] fitting w augmentation --- .../architecture/abstraction/decorators.py | 9 +- .../models/nn/network_impl/base_nn_model.py | 6 +- .../core/models/nn/network_impl/mlstm.py | 201 ++++-------------- 3 files changed, 55 insertions(+), 161 deletions(-) diff --git a/fedot_ind/core/architecture/abstraction/decorators.py b/fedot_ind/core/architecture/abstraction/decorators.py index e34aa52a9..f21218cd1 100644 --- a/fedot_ind/core/architecture/abstraction/decorators.py +++ b/fedot_ind/core/architecture/abstraction/decorators.py @@ -11,9 +11,10 @@ def fedot_data_type(func): def decorated_func(self, *args): - if not isinstance(args[0], InputData): - args[0] = DataConverter(data=args[0]) - features = args[0].features + data, *rest_args = args + if not isinstance(data, InputData): + data = DataConverter(data=data) + features = data.features if len(features.shape) < 4: try: @@ -22,7 +23,7 @@ def decorated_func(self, *args): input_data_squeezed = np.squeeze(features) else: input_data_squeezed = features - return func(self, input_data_squeezed, args[1]) + return func(self, input_data_squeezed, *rest_args) return decorated_func diff --git a/fedot_ind/core/models/nn/network_impl/base_nn_model.py b/fedot_ind/core/models/nn/network_impl/base_nn_model.py index a29fc694b..ff688fed9 100644 --- a/fedot_ind/core/models/nn/network_impl/base_nn_model.py +++ b/fedot_ind/core/models/nn/network_impl/base_nn_model.py @@ -78,7 +78,7 @@ def _fit_model(self, ts: InputData): def _init_model(self, ts) -> tuple: NotImplementedError() - def _prepare_data(self, ts, split_data: bool = True): + def _prepare_data(self, ts, split_data: bool = True, collate_fn=None): if split_data: train_data, val_data = train_test_data_setup( @@ -90,13 +90,13 @@ def _prepare_data(self, ts, split_data: bool = True): val_dataset = None train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=self.batch_size, shuffle=True) + train_dataset, batch_size=self.batch_size, shuffle=True, collate_fn=collate_fn) if val_dataset is None: val_loader = val_dataset else: val_loader = torch.utils.data.DataLoader( - val_dataset, batch_size=self.batch_size, shuffle=True) + val_dataset, batch_size=self.batch_size, shuffle=True, collate_fn=collate_fn) self.label_encoder = train_dataset.label_encoder return train_loader, val_loader diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 9a0a52ba6..612d2b11b 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -4,11 +4,13 @@ from fedot.core.data.data import InputData, OutputData from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE import torch.optim as optim +from torch.optim import lr_scheduler import torch.nn as nn import torch.nn.functional as F import torch +from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array, fedot_data_type import pandas as pd from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping @@ -70,149 +72,19 @@ def __init__(self, input_size, input_channels, for i in idx: torch.nn.init.kaiming_uniform_(seq[i].weight.data) - def forward(self, x): - x_lstm, _ = self.lstm(x) # n x input_ch x inner_size - x_conv = self.conv_branch(x) # n x inner_ch x len - print(x_conv.size(), x_lstm.size()) - x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) - x = F.softmax(self.proj(x)) - return x - - -class MLSTM(BaseNeuralModel): - def __init__(self, params: Optional[OperationParameters] = None): - if params is None: - params = {} - super().__init__() - # self.num_classes = params.get('num_classes', None) - # self.epochs = params.get('epochs', 100) - # self.batch_size = params.get('batch_size', 16) - # self.activation = params.get('activation', 'ReLU') - # self.learning_rate = 0.001 - - self.dropout = params.get('dropout', 0.25) - self.hidden_size = params.get('hidden_size', 64) - self.hidden_channels = params.get('hidden_channels', 32) - self.num_layers = params.get('num_layers', 2) - # self.target = None - # self.task_type = None - - def _init_model(self, ts: InputData): - _, input_channels, input_size = ts.features.shape - self.model = MLSTM_module(input_size, input_channels, - self.hidden_size, self.hidden_channels, - self.num_classes, self.num_layers, - self.dropout) - self.model_for_inference = MLSTM_module(input_size, input_channels, - self.hidden_size, self.hidden_channels, - self.num_classes, self.num_layers, - self.dropout) - optimizer = optim.Adam(self.model.parameters(), lr=0.001) - if ts.num_classes == 2: - loss_fn = CROSS_ENTROPY() - else: - loss_fn = MULTI_CLASS_CROSS_ENTROPY() - return loss_fn, optimizer - - @convert_to_3d_torch_array - def _fit_model(self, ts: InputData): - loss_fn, optimizer = self._init_model(ts) - train_loader, val_loader = self._prepare_data(ts, split_data=True) - self._train_loop( - train_loader=train_loader, - val_loader=val_loader, - loss_fn=loss_fn, - optimizer=optimizer - ) - - -from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel -from typing import Optional, Callable, Any, List, Union -from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.data.data import InputData, OutputData -from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE -import torch.optim as optim -import torch.nn as nn -import torch.nn.functional as F -import torch -from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array -import pandas as pd -from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams -from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter -import torch.utils.data as data -from fedot_ind.core.architecture.settings.computational import default_device - -class SqueezeExciteBlock(nn.Module): - def __init__(self, input_channels, filters, reduce=4): - super().__init__() - self.filters = filters - self.pool = nn.AvgPool1d(input_channels) - self.bottleneck = max(self.filters // reduce, 4) - self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) - self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) - torch.nn.init.kaiming_normal_(self.fc1.weight.data) - torch.nn.init.kaiming_normal_(self.fc2.weight.data) - - def forward(self, x): - input_x = x - x = self.pool(x) - x = F.relu(self.fc1(x.view(-1, 1, self.filters))) - x = F.sigmoid(self.fc2(x)) - x = x.view(-1, self.filters, 1) * input_x - return x - -class MLSTM_module(nn.Module): - def __init__(self, input_size, input_channels, - inner_size, inner_channels, - output_size, num_layers, dropout=0.25): - super().__init__() - self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) - self.lstm = nn.LSTM(input_size, inner_size, num_layers, - batch_first=True, dropout=dropout) - self.conv_branch = nn.Sequential( - nn.Conv1d(input_channels, inner_channels, - padding='same', - kernel_size=9), - nn.BatchNorm1d(inner_channels), - nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels), - nn.Conv1d(inner_channels, inner_channels * 2, - padding='same', - kernel_size=5, - ), # c x l | n x c x l - nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l - nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels * 2), - nn.Conv1d(inner_channels * 2, inner_channels, - padding='same', - kernel_size=3, - ), # c x l | n x c x l - nn.BatchNorm1d(inner_channels), # n x c | n x c x l - nn.ReLU(), - ) - seq = next(iter(self.conv_branch.modules())) - idx = [0, 4, 8] - for i in idx: - torch.nn.init.kaiming_uniform_(seq[i].weight.data) - - def forward(self, x, hidden_state=None): + def forward(self, x, hidden_state=None, return_hidden_state=False): + # hidden_state = hidden_state or self.hidden_state + if not self.training: + print(x.shape) x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size x_conv = self.conv_branch(x) # n x inner_ch x len x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) x = F.softmax(self.proj(x)) + # self.hidden_state = hidden_state + if return_hidden_state: + return x, hidden_state return x#, hidden_state - def augment_zero_padding(self, X: torch.Tensor): - res = [] - for i in self.prediction_idx: - zeroed_X = X[...] - zeroed_X[..., i + 1:] = 0 - res.append(zeroed_X) - res = torch.concat(res, 0) - return res[torch.randperm(res.size(0)), ...] class MLSTM(BaseNeuralModel): def __init__(self, params: Optional[OperationParameters] = None): @@ -229,11 +101,19 @@ def __init__(self, params: Optional[OperationParameters] = None): self.hidden_size = params.get('hidden_size', 64) self.hidden_channels = params.get('hidden_channels', 32) self.num_layers = params.get('num_layers', 2) - # self.target = None - # self.task_type = None self.interval_percentage = params.get('interval_percentage', 10) self.min_ts_length = params.get('min_ts_length', 5) + def __repr__(self): + return 'MLSTM' + + @convert_to_3d_torch_array + def _predict_model(self, ts: InputData, output_mode='default'): + self.model.eval() + x_test = torch.Tensor(ts).to(self._device) + pred = self.model(x_test) + return self._convert_predict(pred, output_mode) + def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) prediction_idx = np.arange(0, n_idx, interval_length) @@ -241,7 +121,7 @@ def _compute_prediction_points(self, n_idx): return prediction_idx def _init_model(self, ts: InputData): - _, input_channels, input_size = ts.features.shape + *_, input_channels, input_size = ts.features.shape self.prediction_idx = self._compute_prediction_points(input_size) self.model = MLSTM_module(input_size, input_channels, self.hidden_size, self.hidden_channels, @@ -255,27 +135,40 @@ def _init_model(self, ts: InputData): if ts.num_classes == 2: loss_fn = CROSS_ENTROPY() else: - loss_fn = MULTI_CLASS_CROSS_ENTROPY() + loss_fn = CROSS_ENTROPY() return loss_fn, optimizer - def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): - return super()._train_loop(train_loader, val_loader, loss_fn, optimizer) + # @convert_to_3d_torch_array + # def predict(self, ts: InputData, output_mode: str = 'default'): + # return super().predict(ts, output_mode) + + # def predict_for_fit(self, ts: InputData, output_mode: str = 'default'): + # return super().predict_for_fit(ts, output_mode) @convert_to_3d_torch_array - def _fit_model(self, ts: InputData): - if isinstance(ts, torch.Tensor): - ts = self.augment_zero_padding(ts) - else: - print(type(ts)) + def _fit_model(self, ts: InputData, mode='zero_padding'): + self.epochs = 1 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 loss_fn, optimizer = self._init_model(ts) - train_loader, val_loader = self._prepare_data(ts, split_data=True) + + train_loader, val_loader = self._prepare_data(ts, split_data=False, + collate_fn=getattr(self, '_augment_zero_padding')) self._train_loop( train_loader=train_loader, val_loader=val_loader, loss_fn=loss_fn, - optimizer=optimizer + optimizer=optimizer, ) - - - + def _augment_zero_padding(self, batch,): + prediction_idx = self.prediction_idx + x, y = zip(*batch) + X, y = torch.stack(x), torch.stack(y) + y = np.tile(y, (len(prediction_idx), 1)) + res = [] + for i in prediction_idx: + zeroed_X = X[...] + zeroed_X[..., i + 1:] = 0 + res.append(zeroed_X) + res = np.concatenate(res, 0) + perm = np.random.permutation(res.shape[0]) + return torch.tensor(res[perm, ...]), torch.tensor(y[perm]) From 5d9182ecbe2444389da61709da665b77f0f1e53e Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 01:27:24 +0300 Subject: [PATCH 13/43] all work, but need eval --- .../core/models/early_tc/base_early_tc.py | 3 + fedot_ind/core/models/early_tc/ecec.py | 61 +++++++++++++------ fedot_ind/core/models/early_tc/economy_k.py | 21 +++++-- .../core/models/early_tc/prob_threshold.py | 29 ++++++--- fedot_ind/core/models/early_tc/teaser.py | 19 ++++-- 5 files changed, 93 insertions(+), 40 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 5e180c2a9..38e5fd23f 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -71,6 +71,9 @@ def _select_estimators(self, X, training=False): elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) + elif 'last_available': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = [last_idx] else: raise ValueError('Unknown prediction mode') return estimator_indices, offset diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 45f7f9fe4..00962a233 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -9,6 +9,7 @@ class ECEC(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): super().__init__(params) + self.__cv = 5 def _init_model(self, X, y): super()._init_model(X, y) @@ -30,34 +31,39 @@ def predict(self, X): predicted_labels, _, non_confident, confidences = self._predict(X) predicted_labels = np.stack(predicted_labels) predicted_labels[non_confident] = -1 - return predicted_labels, confidences + if self.transform_score: + confidences = self._transform_score(confidences) + return self._remove_first_1d(predicted_labels, confidences) def predict_proba(self, X): _, predicted_probas, non_confident, confidences = self._predict(X) predicted_probas = np.stack(predicted_probas) predicted_probas[non_confident] = -1 - return predicted_probas, confidences + if self.transform_score: + confidences = self._transform_score(confidences) + return self._remove_first_1d(predicted_probas, confidences) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] + X_part = self.scalers[i].fit_transform(X_part) + self.slave_estimators[i].fit(X_part, y) + labels = cross_val_predict(self.slave_estimators[i], X_part, y, cv=self.__cv) + return labels - def _score(self, X, y, alpha, training=False): - y = y.astype(int) - predicted_labels, *_ = super()._predict(X, training) # n_pred x n_inst - predicted_labels = np.stack(predicted_labels) - n = predicted_labels.shape[0] - accuracies = (predicted_labels == np.tile(y, (n, 1))) # n_pred x n_inst - confidences = np.ones((n, X.shape[0]), dtype='float32') - for i in range(n): - y_pred = predicted_labels[i] - reliability_i = confusion_matrix(y, y_pred, normalize='pred') - confidences[i] = 1 - reliability_i[y, y_pred] # n_inst - self._reliabilities[i] = reliability_i - confidences = 1 - np.cumprod(confidences, axis=0) # n_pred x n_inst + def _score(self, y, y_pred, alpha): + matches = (y_pred == np.tile(y, (self.n_pred, 1))) # n_pred x n_inst + n, n_inst = matches.shape[:2] + confidences = np.ones((n, n_inst), dtype='float32') + for i in range(self.n_pred): + confidences[i] = self._reliabilities[i, y, y_pred[i]] + confidences = 1 - np.cumprod(1 - confidences, axis=0) # n_pred x n_inst candidates = self._select_thrs(confidences) # n_candidates cfs = np.zeros((len(candidates), n)) for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst - accuracy_for_candidate = (accuracies * mask).sum(1) / mask.sum(1) # n_pred + accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) - self._best_estimator_idx = np.argmin(cfs.mean(0)) + self._chosen_estimator_idx = np.argmin(cfs.mean(0)) return candidates[np.argmin(cfs, axis=0)] # n_pred @staticmethod @@ -75,6 +81,23 @@ def cost_func(earliness, accuracies, alpha): return alpha * (1 - accuracies) + (1 - alpha) * earliness def fit(self, X, y): - super().fit(X, y) - self.confidence_thresholds = self._score(X, y, self.accuracy_importance, training=True) + y = np.array(y).flatten().astype(int) + self._init_model(X, y) + labels = [] + for i in range(self.n_pred): + labels.append(self._fit_one_interval(X, y, i)) + predicted_labels = np.stack(labels) + for i in range(self.n_pred): + y_pred = predicted_labels[i] + reliability_i = confusion_matrix(y, y_pred, normalize='pred') + self._reliabilities[i] = reliability_i + self.confidence_thresholds = self._score(y, predicted_labels, self.accuracy_importance) + + def _transform_score(self, confidences): + thr = self.confidence_thresholds[self._estimator_for_predict[-1]] + confidences = confidences - thr + positive = confidences > 0 + confidences[positive] *= 1 / (1 - thr) + confidences[~positive] *= 1 / thr + return confidences \ No newline at end of file diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 639e680cd..8eb207550 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -11,8 +11,10 @@ def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} super().__init__(params) + self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) self._cluster_factor = params.get('cluster_factor' , 1) + # self.confidence_mode = params.get('confidence_mode', 'time') # or 'confidence' self._random_state = 2104 self.__cv = 5 @@ -77,13 +79,20 @@ def _get_prediction_time(self, X, cluster_centroids, i): def predict_proba(self, X): probas, times, is_optimal = self._predict(X) is_optimal = np.stack(is_optimal) - idx = np.tile(np.arange(self.n_pred), (is_optimal.shape[1], 1)).T # n_pred x n_inst - idx[~is_optimal] = self.n_pred - idx = np.argmin(idx, 0) - probas = np.stack(probas) - return probas[idx], np.stack(times)[idx] + probas, times = np.stack(probas), np.stack(times) + if self.transform_score: + times = self._transform_score(times) + return self._remove_first_1d(probas, times) def predict(self, X): probas, times = self.predict_proba(X) labels = probas.argmax(-1) - return labels, times + return self._remove_first_1d(labels, times) + + def _transform_score(self, time): + idx = self._estimator_for_predict[-1] + scores = -(1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) + scores[scores == 0] = 1 # no posibility for lininterp when sure + return scores + + diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 51d169909..8dcb8828f 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -7,7 +7,7 @@ class ProbabilityThresholdClassifier(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} - super().__init__() + super().__init__(params) self.probability_threshold = params.get('probability_threshold', None) def _init_model(self, X, y): @@ -18,19 +18,23 @@ def _init_model(self, X, y): def predict_proba(self, X): _, predicted_probas, non_acceptance = self._predict(X) predicted_probas[non_acceptance] = 0 - return predicted_probas.squeeze() + scores = predicted_probas.max(-1) + if self.transform_score: + scores = self._transform_score(scores) + return self._remove_first_1d(predicted_probas, scores) def predict(self, X): - predicted_labels, _, non_acceptance = self._predict(X, training=False) + predicted_labels, predicted_probas, non_acceptance = self._predict(X, training=False) predicted_labels[non_acceptance] = -1 - # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] - return predicted_labels # prediction_points x n_instances + scores = predicted_probas.max(-1) + if self.transform_score: + scores = self._transform_score(scores) + return self._remove_first_1d(predicted_labels, scores) # (prediction_points x) n_instances def _predict(self, X, training=True): predicted_probas, predicted_labels = super()._predict(X, training) predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) - # print(predicted_labels.shape, predicted_probas.shape) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions double_check = predicted_probas.max(axis=-1) > self.probability_threshold non_acceptance[non_acceptance & double_check] = False @@ -38,10 +42,17 @@ def _predict(self, X, training=True): def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) - self._best_estimator_idx = np.argmax(scores) + self._chosen_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) - return self._score(X, y, self.accuracy_importance) - + self._score(X, y, self.accuracy_importance) + + def _transform_score(self, confidences): + thr = self.probability_threshold + confidences = confidences - thr + positive = confidences > 0 + confidences[positive] *= 1 / (1 - thr) + confidences[~positive] *= 1 / thr + return confidences diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 593a34e72..6ddc19a45 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -56,31 +56,38 @@ def _predict(self, X): # for each point of estimation for i in range(predicted_labels.shape[0]): # find not accepted points - ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] - X_to_ith = X_ocs[i][ith_point_to_oc] + X_to_ith = X_ocs[i] # if they are not outliers final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject # mark as accepted - non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False + # non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False final_verdicts[i] = final_verdict + non_acceptance[non_acceptance & (final_verdict > 0)] = False return predicted_labels, predicted_probas, non_acceptance, final_verdicts def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] - return predicted_probas.squeeze() + if self.transform_score: + final_verdicts = self._transform_score(final_verdicts) + return self._remove_first_1d(predicted_probas, final_verdicts) def predict(self, X): predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) predicted_labels[non_acceptance] = -1 # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] - return predicted_labels # prediction_points x n_instances + if self.transform_score: + final_verdicts = self._transform_score(final_verdicts) + return self._remove_first_1d(predicted_labels, final_verdicts) # (prediction_points x) n_instances def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) - self._best_estimator_idx = np.argmax(scores) + self._chosen_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) return self._score(X, y, self.accuracy_importance) + + def _transform_score(self, scores): + return np.tanh(scores) From 41c329c5bde01ce902ee831552103ab4aa350f05 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 12:09:39 +0300 Subject: [PATCH 14/43] evth converged to one interface + refactored --- .../core/models/early_tc/base_early_tc.py | 53 ++-- fedot_ind/core/models/early_tc/ecec.py | 18 +- fedot_ind/core/models/early_tc/economy_k.py | 15 +- .../core/models/early_tc/prob_threshold.py | 14 +- fedot_ind/core/models/early_tc/teaser.py | 23 +- .../core/models/nn/network_impl/mlstm.py | 235 ++++++++++++------ .../data/default_operation_params.json | 14 +- .../data/industrial_model_repository.json | 22 +- fedot_ind/core/repository/model_repository.py | 8 +- fedot_ind/core/tuning/search_space.py | 32 ++- 10 files changed, 281 insertions(+), 153 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 38e5fd23f..9fbc162c5 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -11,10 +11,12 @@ def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} super().__init__() - self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') self.interval_percentage = params.get('interval_percentage', 10) - self.consecutive_predictions = params.get('consecutive_predictions', 3) + self.consecutive_predictions = params.get('consecutive_predictions', 1) self.accuracy_importance = params.get('accuracy_importance', 1.) + + self.prediction_mode = params.get('prediction_mode', 'last_available') + self.transform_score = params.get('transform_score', True) self.min_ts_length = params.get('min_ts_step', 3) self.random_state = params.get('random_state', None) self.weasel_params = {} @@ -26,14 +28,15 @@ def _init_model(self, X, y): self.n_pred = len(self.prediction_idx) self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] self.scalers = [StandardScaler() for _ in range(self.n_pred)] - self._best_estimator_idx = -1 + self._chosen_estimator_idx = -1 self.classes_ = [np.unique(y)] + self._estimator_for_predict = [-1] @property def required_length(self): - if not hasattr(self, '_best_estimator_idx'): + if not hasattr(self, '_chosen_estimator_idx'): return None - return self.prediction_idx[self._best_estimator_idx] + return self.prediction_idx[self._chosen_estimator_idx] @property def n_classes(self): @@ -47,7 +50,7 @@ def fit(self, X, y=None): self._fit_one_interval(X, y, i) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? + X_part = X[..., :self.prediction_idx[i] + 1] X_part = self.scalers[i].fit_transform(X_part) probas = self.slave_estimators[i].fit_predict_proba(X_part, y) return probas @@ -67,19 +70,21 @@ def _compute_prediction_points(self, n_idx): def _select_estimators(self, X, training=False): offset = 0 if not training and self.prediction_mode == 'best_by_harmonic_mean': - estimator_indices = [self._best_estimator_idx] + estimator_indices = [self._chosen_estimator_idx] + elif not training and self.prediction_mode == 'last_available': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = [last_idx] elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) - elif 'last_available': - last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) - estimator_indices = [last_idx] else: raise ValueError('Unknown prediction mode') return estimator_indices, offset def _predict(self, X, training=True): estimator_indices, offset = self._select_estimators(X, training) + if not training: + self._estimator_for_predict = estimator_indices prediction = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) @@ -94,20 +99,32 @@ def _consecutive_count(self, predicted_labels: List[np.array]): consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 return consecutive_labels # prediction_points x n_instances - def predict_proba(self, X): - raise NotImplementedError + def predict_proba(self, *args): + predicted_probas, scores, *_ = args + if self.transform_score: + scores = self._transform_score(scores) + scores = np.tile(scores[..., None], (1, 1, self.n_classes)) + prediction = np.stack([predicted_probas, scores], axis=0) + if prediction.shape[1] == 1: + prediction = prediction.squeeze(1) + return prediction def predict(self, X): - raise NotImplementedError + prediction = self.predict_proba(X) + labels = prediction[0:1].argmax(-1) + scores = prediction[1:2, ..., 0] + prediction = np.stack([labels, scores], 0) + if prediction.shape[1] == 1: + prediction = prediction.squeeze(1) + return prediction - def _score(self, X, y, hm_shift_to_acc=None): + def _score(self, X, y, accuracy_importance=None, training=True): y = np.array(y).flatten() - hm_shift_to_acc = hm_shift_to_acc or self.accuracy_importance - predictions = self._predict(X)[0] + accuracy_importance = accuracy_importance or self.accuracy_importance + predictions = self._predict(X, training)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - + return (1 + accuracy_importance) * accuracies * self.earliness[:prediction_points] / (accuracy_importance * accuracies + self.earliness[:prediction_points]) def _get_applicable_index(self, last_available_idx): idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 00962a233..f6e163d25 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -3,7 +3,6 @@ from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.models.early_tc.base_early_tc import BaseETC from sklearn.model_selection import cross_val_predict -from sklearn.base import clone from sklearn.metrics import confusion_matrix class ECEC(BaseETC): @@ -25,23 +24,14 @@ def _predict(self, X, training=False): reliabilities = np.stack(reliabilities) confidences = 1 - np.cumprod(1 - reliabilities, axis=0) non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] - return predicted_labels, predicted_probas, non_confident, confidences - - def predict(self, X): - predicted_labels, _, non_confident, confidences = self._predict(X) predicted_labels = np.stack(predicted_labels) - predicted_labels[non_confident] = -1 - if self.transform_score: - confidences = self._transform_score(confidences) - return self._remove_first_1d(predicted_labels, confidences) + predicted_probas = np.stack(predicted_probas) + return predicted_labels, predicted_probas, non_confident, confidences def predict_proba(self, X): _, predicted_probas, non_confident, confidences = self._predict(X) - predicted_probas = np.stack(predicted_probas) predicted_probas[non_confident] = -1 - if self.transform_score: - confidences = self._transform_score(confidences) - return self._remove_first_1d(predicted_probas, confidences) + return super().predict_proba(predicted_probas, confidences) def _fit_one_interval(self, X, y, i): X_part = X[..., :self.prediction_idx[i] + 1] @@ -52,7 +42,7 @@ def _fit_one_interval(self, X, y, i): def _score(self, y, y_pred, alpha): matches = (y_pred == np.tile(y, (self.n_pred, 1))) # n_pred x n_inst - n, n_inst = matches.shape[:2] + n, n_inst, *_ = matches.shape confidences = np.ones((n, n_inst), dtype='float32') for i in range(self.n_pred): confidences[i] = self._reliabilities[i, y, y_pred[i]] diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 8eb207550..bf09acd4b 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -4,7 +4,7 @@ from fedot_ind.core.models.early_tc.base_early_tc import BaseETC from sklearn.cluster import KMeans from sklearn.metrics import confusion_matrix -from sklearn.model_selection import train_test_split, cross_val_predict +from sklearn.model_selection import cross_val_predict class EconomyK(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): @@ -14,7 +14,6 @@ def __init__(self, params: Optional[OperationParameters] = None): self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) self._cluster_factor = params.get('cluster_factor' , 1) - # self.confidence_mode = params.get('confidence_mode', 'time') # or 'confidence' self._random_state = 2104 self.__cv = 5 @@ -77,17 +76,9 @@ def _get_prediction_time(self, X, cluster_centroids, i): return time_optimal, is_optimal # n_inst def predict_proba(self, X): - probas, times, is_optimal = self._predict(X) - is_optimal = np.stack(is_optimal) + probas, times, _ = self._predict(X, training=False) probas, times = np.stack(probas), np.stack(times) - if self.transform_score: - times = self._transform_score(times) - return self._remove_first_1d(probas, times) - - def predict(self, X): - probas, times = self.predict_proba(X) - labels = probas.argmax(-1) - return self._remove_first_1d(labels, times) + return super().predict_proba(probas, times) def _transform_score(self, time): idx = self._estimator_for_predict[-1] diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 8dcb8828f..773f79d8e 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -16,20 +16,10 @@ def _init_model(self, X, y): self.probability_threshold = 1 / len(self.classes_[0]) def predict_proba(self, X): - _, predicted_probas, non_acceptance = self._predict(X) + _, predicted_probas, non_acceptance = self._predict(X, training=False) predicted_probas[non_acceptance] = 0 scores = predicted_probas.max(-1) - if self.transform_score: - scores = self._transform_score(scores) - return self._remove_first_1d(predicted_probas, scores) - - def predict(self, X): - predicted_labels, predicted_probas, non_acceptance = self._predict(X, training=False) - predicted_labels[non_acceptance] = -1 - scores = predicted_probas.max(-1) - if self.transform_score: - scores = self._transform_score(scores) - return self._remove_first_1d(predicted_labels, scores) # (prediction_points x) n_instances + return super().predict_proba(predicted_probas, scores) def _predict(self, X, training=True): predicted_probas, predicted_labels = super()._predict(X, training) diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 6ddc19a45..35328c535 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,9 +1,9 @@ from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np -from sklearn.svm import OneClassSVM -from sklearn.model_selection import GridSearchCV from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from sklearn.model_selection import GridSearchCV +from sklearn.svm import OneClassSVM class TEASER(BaseETC): @@ -13,6 +13,7 @@ def __init__(self, params: Optional[OperationParameters] = None): def _init_model(self, X, y): super()._init_model(X, y) + def _init_model(self, X, y): super()._init_model(X, y) self.oc_estimators = [None] * self.n_pred @@ -42,13 +43,12 @@ def _form_X_oc(self, predicted_probas): d = d.min(axis=-1).reshape(-1, 1) return np.hstack([predicted_probas, d]) - def _predict(self, X): + def _predict(self, X, training=False): estimator_indices, offset = self._select_estimators(X) X_ocs, predicted_probas, predicted_labels = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions - to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) @@ -58,9 +58,8 @@ def _predict(self, X): # find not accepted points X_to_ith = X_ocs[i] # if they are not outliers - final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject + final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # mark as accepted - # non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False final_verdicts[i] = final_verdict non_acceptance[non_acceptance & (final_verdict > 0)] = False return predicted_labels, predicted_probas, non_acceptance, final_verdicts @@ -68,17 +67,7 @@ def _predict(self, X): def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] - if self.transform_score: - final_verdicts = self._transform_score(final_verdicts) - return self._remove_first_1d(predicted_probas, final_verdicts) - - def predict(self, X): - predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) - predicted_labels[non_acceptance] = -1 - # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] - if self.transform_score: - final_verdicts = self._transform_score(final_verdicts) - return self._remove_first_1d(predicted_labels, final_verdicts) # (prediction_points x) n_instances + return super().predict_proba(predicted_probas, final_verdicts) def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 612d2b11b..3e1d3c4b5 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -1,23 +1,20 @@ +import copy from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel from typing import Optional, Callable, Any, List, Union from fedot.core.operations.operation_parameters import OperationParameters from fedot.core.data.data import InputData, OutputData from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE import torch.optim as optim -from torch.optim import lr_scheduler +import torch.optim.lr_scheduler as lr_scheduler import torch.nn as nn import torch.nn.functional as F import torch -from tqdm import tqdm +from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array, fedot_data_type +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array import pandas as pd -from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter import torch.utils.data as data -from fedot_ind.core.architecture.settings.computational import default_device class SqueezeExciteBlock(nn.Module): def __init__(self, input_channels, filters, reduce=4): @@ -46,20 +43,22 @@ def __init__(self, input_size, input_channels, self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) self.lstm = nn.LSTM(input_size, inner_size, num_layers, batch_first=True, dropout=dropout) + + squeeze_excite_size = input_size #if not interval else interval self.conv_branch = nn.Sequential( nn.Conv1d(input_channels, inner_channels, padding='same', kernel_size=9), nn.BatchNorm1d(inner_channels), nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels), + SqueezeExciteBlock(squeeze_excite_size, inner_channels), nn.Conv1d(inner_channels, inner_channels * 2, padding='same', kernel_size=5, ), # c x l | n x c x l nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels * 2), + SqueezeExciteBlock(squeeze_excite_size, inner_channels * 2), nn.Conv1d(inner_channels * 2, inner_channels, padding='same', kernel_size=3, @@ -72,103 +71,197 @@ def __init__(self, input_size, input_channels, for i in idx: torch.nn.init.kaiming_uniform_(seq[i].weight.data) - def forward(self, x, hidden_state=None, return_hidden_state=False): - # hidden_state = hidden_state or self.hidden_state - if not self.training: - print(x.shape) + def forward(self, x, hidden_state=None, return_hidden=False): x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size x_conv = self.conv_branch(x) # n x inner_ch x len x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) x = F.softmax(self.proj(x)) - # self.hidden_state = hidden_state - if return_hidden_state: + if return_hidden: return x, hidden_state - return x#, hidden_state - + return x + class MLSTM(BaseNeuralModel): def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} super().__init__() - # self.num_classes = params.get('num_classes', None) - # self.epochs = params.get('epochs', 100) - # self.batch_size = params.get('batch_size', 16) - # self.activation = params.get('activation', 'ReLU') - # self.learning_rate = 0.001 - self.dropout = params.get('dropout', 0.25) self.hidden_size = params.get('hidden_size', 64) self.hidden_channels = params.get('hidden_channels', 32) self.num_layers = params.get('num_layers', 2) self.interval_percentage = params.get('interval_percentage', 10) self.min_ts_length = params.get('min_ts_length', 5) - + self.fitting_mode = params.get('fitting_mode', 'zero_padding') + self.proba_thr = params.get('proba_thr', None) + def __repr__(self): return 'MLSTM' - @convert_to_3d_torch_array - def _predict_model(self, ts: InputData, output_mode='default'): - self.model.eval() - x_test = torch.Tensor(ts).to(self._device) - pred = self.model(x_test) - return self._convert_predict(pred, output_mode) - def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) - prediction_idx = np.arange(0, n_idx, interval_length) + prediction_idx = np.arange(interval_length - 1, n_idx, interval_length) self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 - return prediction_idx + return prediction_idx, interval_length def _init_model(self, ts: InputData): - *_, input_channels, input_size = ts.features.shape - self.prediction_idx = self._compute_prediction_points(input_size) - self.model = MLSTM_module(input_size, input_channels, + _, input_channels, input_size = ts.features.shape + self.input_size = input_size + self.prediction_idx, self.interval = self._compute_prediction_points(input_size) + self.model = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, + input_channels, self.hidden_size, self.hidden_channels, self.num_classes, self.num_layers, self.dropout) - self.model_for_inference = MLSTM_module(input_size, input_channels, + self.model_for_inference = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, + input_channels, self.hidden_size, self.hidden_channels, self.num_classes, self.num_layers, self.dropout) optimizer = optim.Adam(self.model.parameters(), lr=0.001) - if ts.num_classes == 2: - loss_fn = CROSS_ENTROPY() - else: - loss_fn = CROSS_ENTROPY() + loss_fn = CROSS_ENTROPY() return loss_fn, optimizer - # @convert_to_3d_torch_array - # def predict(self, ts: InputData, output_mode: str = 'default'): - # return super().predict(ts, output_mode) - - # def predict_for_fit(self, ts: InputData, output_mode: str = 'default'): - # return super().predict_for_fit(ts, output_mode) - @convert_to_3d_torch_array - def _fit_model(self, ts: InputData, mode='zero_padding'): - self.epochs = 1 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 + def _fit_model(self, ts: InputData): + mode = self.fitting_mode loss_fn, optimizer = self._init_model(ts) + train_loader, val_loader = self._prepare_data(ts, split_data=True, + collate_fn=getattr(self, '_augment_with_zeros')) + if mode == 'zero_padding': + super()._train_loop( + train_loader=train_loader, + val_loader=val_loader, + loss_fn=loss_fn, + optimizer=optimizer + ) + elif mode == 'moving_window': + self._train_loop( + train_loader=train_loader, + val_loader=None, + loss_fn=loss_fn, + optimizer=optimizer + ) + else: + raise ValueError('Unknown fitting mode') + + def _moving_window_output(self, inputs): + hidden_state = None + output = -torch.ones((inputs.shape[0], self.num_classes)) + for i in self.prediction_idx: + if i >= inputs.shape[-1]: + break + batch_interval = inputs[..., i - self.prediction_idx[0] : i + 1] + output, hidden_state = self.model(batch_interval, hidden_state, return_hidden=True) + return output - train_loader, val_loader = self._prepare_data(ts, split_data=False, - collate_fn=getattr(self, '_augment_zero_padding')) - self._train_loop( - train_loader=train_loader, - val_loader=val_loader, - loss_fn=loss_fn, - optimizer=optimizer, - ) + def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): + early_stopping = EarlyStopping() + scheduler = lr_scheduler.OneCycleLR(optimizer=optimizer, + steps_per_epoch=len(train_loader), + epochs=self.epochs, + max_lr=self.learning_rate) + if val_loader is None: + print('Not enough class samples for validation') + + best_model = None + best_val_loss = float('inf') + val_interval = self.get_validation_frequency( + self.epochs, self.learning_rate) - def _augment_zero_padding(self, batch,): - prediction_idx = self.prediction_idx - x, y = zip(*batch) - X, y = torch.stack(x), torch.stack(y) - y = np.tile(y, (len(prediction_idx), 1)) - res = [] - for i in prediction_idx: - zeroed_X = X[...] - zeroed_X[..., i + 1:] = 0 - res.append(zeroed_X) - res = np.concatenate(res, 0) - perm = np.random.permutation(res.shape[0]) - return torch.tensor(res[perm, ...]), torch.tensor(y[perm]) + for epoch in range(1, self.epochs + 1): + training_loss = 0.0 + valid_loss = 0.0 + self.model.train() + total = 0 + correct = 0 + for batch in tqdm(train_loader): + optimizer.zero_grad() + inputs, targets = batch + output = self._moving_window_output(inputs) + loss = loss_fn(output, targets.float()) + loss.backward() + optimizer.step() + training_loss += loss.data.item() * inputs.size(0) + total += targets.size(0) + correct += (torch.argmax(output, 1) == + torch.argmax(targets, 1)).sum().item() + + accuracy = correct / total + training_loss /= len(train_loader.dataset) + print('Epoch: {}, Accuracy = {}, Training Loss: {:.2f}'.format( + epoch, accuracy, training_loss)) + + if val_loader is not None and epoch % val_interval == 0: + self.model.eval() + total = 0 + correct = 0 + for batch in val_loader: + inputs, targets = batch + + output = self.model(inputs) + + loss = loss_fn(output, targets.float()) + + valid_loss += loss.data.item() * inputs.size(0) + total += targets.size(0) + correct += (torch.argmax(output, 1) == + torch.argmax(targets, 1)).sum().item() + if valid_loss < best_val_loss: + best_val_loss = valid_loss + best_model = copy.deepcopy(self.model) + + early_stopping(training_loss, self.model, './') + adjust_learning_rate(optimizer, scheduler, + epoch + 1, self.learning_rate, printout=False) + scheduler.step() + + if early_stopping.early_stop: + print("Early stopping") + break + + if best_model is not None: + self.model = best_model + + @convert_to_3d_torch_array + def _predict_model(self, x_test: InputData, output_mode: str = 'default'): + self.model.eval() + if self.fitting_mode == 'zero_padding': + x_test = self._padding(x_test).to(self._device) + pred = self.model(x_test) + elif self.fitting_mode == 'moving_window': + pred = self._moving_window_output(torch.tensor(x_test).float()) + else: + raise ValueError('Unknown prediction mode') + return self._convert_predict(pred, output_mode) + + def _padding(self, ts: np.array): + if ts.shape[-1] == self.input_size: + return torch.tensor(ts).float() + n, ch, size = ts.shape + x = torch.zeros((n, ch, self.input_size)).float() + x[..., :size] = ts + return x + + def _augment_with_zeros(self, batch: np.array): + X, y = zip(*batch) + X, y = np.stack(X), np.stack(y) + X_res, y_res = [], [] + for i in self.prediction_idx: + x = X[...] + x[..., :i + i] = 0 + X_res.append(x) + y_res.append(y) + X_res = np.concatenate(X_res) + y_res = np.concatenate(y_res) + perm = np.random.permutation(X_res.shape[0]) + return torch.tensor(X_res[perm]), torch.tensor(y_res[perm]) + + def _transform_score(self, probas): + # linear interp + thr = self.proba_thr + probas = probas - thr + positive = probas > 0 + probas[positive] *= 1 / (1 - thr) + probas[~positive] *= 1 / thr + return probas diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index b875c3225..46c5d8c3c 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -130,15 +130,25 @@ "min_samples_leaf": 10, "bootstrap": false }, + "ecec": { + "interval_percentage": 10, + "accuracy_importance": 0.7 + }, + "economy_k": { + "interval_percentage": 10, + "accuracy_importance": 0.7, + "cluster_factor": 1, + "lambda": 1 + }, "teaser": { "interval_percentage": 10, "consecutive_predictions": 3, - "hm_shift_to_acc": 2 + "accuracy_importance": 2 }, "proba_threshold_etc": { "interval_percentage": 10, "consecutive_predictions": 3, - "hm_shift_to_acc": 2 + "accuracy_importance": 2 }, "dt": { "max_depth": 5, diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 0823954a2..aad712d78 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -623,10 +623,18 @@ "non_linear" ] }, - "teaser": { + "ecec": { + "meta": "sklearn_class", + "tags": [ + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" + }, + "economy_k": { "meta": "sklearn_class", "tags": [ - "simple", "interpretable", "non_lagged", "non_linear" @@ -643,6 +651,16 @@ ], "input_type": "[DataTypesEnum.table]" }, + "teaser": { + "meta": "sklearn_class", + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" + }, "xgboost": { "meta": "sklearn_class", "presets": ["*tree"], diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index 9777b81c1..722325817 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -44,8 +44,10 @@ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from xgboost import XGBRegressor -from fedot_ind.core.models.early_tc.teaser import TEASER +from fedot_ind.core.models.early_tc.ecec import ECEC +from fedot_ind.core.models.early_tc.economy_k import EconomyK from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier +from fedot_ind.core.models.early_tc.teaser import TEASER from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR @@ -137,8 +139,10 @@ class AtomizedModel(Enum): # external models 'lgbm': LGBMClassifier, # Early classification + 'ecec': ECEC, + 'economy_k': EconomyK, + 'proba_threshold_etc': ProbabilityThresholdClassifier, 'teaser': TEASER, - 'proba_threshold_etc': ProbabilityThresholdClassifier } FEDOT_PREPROC_MODEL = { # data standartization diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 11be89db9..25faa8a07 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -60,15 +60,41 @@ 'selection_strategy': {'hyperopt-dist': hp.choice, 'sampling-scope': [['sum', 'pairwise']]} }, - 'teaser': + 'ecec': { + 'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'accuracy_importance': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[i / 10 for i in range(11)]]}, + }, + 'economy_k': { + 'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'lambda': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[1e-6, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4, 1e6]]}, + 'accuracy_importance': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[i / 10 for i in range(11)]]}, + }, + 'mlstm_model': { + 'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'dropout': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, + 'hidden_size': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(10, 101, 10))]}, + 'num_layers': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(1, 6))]}, + 'hidden_channels': {'hyperopt-dist': hp.choice, + 'sampling-scope': [8, 16, 32, 64, 96]}, + }, + 'proba_threshold_etc': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, - 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'accuracy_importance': {'hyperopt-dist': hp.choice, 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, }, - 'proba_threshold_etc': + 'teaser': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, From 96c10091d8f8c6f431d8c3cf29b9f6fca13c74ef Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 15:02:54 +0300 Subject: [PATCH 15/43] slight fixes --- fedot_ind/core/models/early_tc/base_early_tc.py | 2 +- fedot_ind/core/models/early_tc/economy_k.py | 9 ++++++--- .../core/repository/data/default_operation_params.json | 4 ++-- fedot_ind/core/tuning/search_space.py | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 9fbc162c5..cbed8c463 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -124,7 +124,7 @@ def _score(self, X, y, accuracy_importance=None, training=True): predictions = self._predict(X, training)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + accuracy_importance) * accuracies * self.earliness[:prediction_points] / (accuracy_importance * accuracies + self.earliness[:prediction_points]) + return (1 - accuracy_importance) * self.earliness[:prediction_points] + accuracy_importance * accuracies def _get_applicable_index(self, last_available_idx): idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index bf09acd4b..c39097189 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -65,7 +65,7 @@ def __expected_costs(self, X, cluster_centroids, i): np.sum(self.state[i:], axis=-1), axes=(0, 2, 1) ) * self._pyck_[None, ...], axis=1) costs = cluster_probas @ s_glob.T # n_inst x time_left - costs += self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? + costs -= self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? return costs def _get_prediction_time(self, X, cluster_centroids, i): @@ -82,8 +82,11 @@ def predict_proba(self, X): def _transform_score(self, time): idx = self._estimator_for_predict[-1] - scores = -(1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) - scores[scores == 0] = 1 # no posibility for lininterp when sure + scores = (1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) # [1 / n; 1 ] - 1 / n) * n /(n - 1) * 2 - 1 + n = self.n_pred + scores -= 1 / n + scores *= n / (n - 1) * 2 + scores -= 1 return scores diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 46c5d8c3c..0167e0195 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -143,12 +143,12 @@ "teaser": { "interval_percentage": 10, "consecutive_predictions": 3, - "accuracy_importance": 2 + "accuracy_importance": 0.5 }, "proba_threshold_etc": { "interval_percentage": 10, "consecutive_predictions": 3, - "accuracy_importance": 2 + "accuracy_importance": 0.5 }, "dt": { "max_depth": 5, diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 25faa8a07..8ab927edb 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -92,15 +92,15 @@ 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, 'accuracy_importance': {'hyperopt-dist': hp.choice, - 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, 'teaser': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, - 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, - 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + 'accuracy_importance': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, 'deepar_model': {'epochs': {'hyperopt-dist': hp.choice, From 743c404b0183e075897a2d026cb7d1cba936fd62 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 14 Jun 2024 12:21:29 +0300 Subject: [PATCH 16/43] metrics started --- fedot_ind/core/models/early_tc/metrics.py | 122 ++++++++++++++++++++++ fedot_ind/core/models/early_tc/teaser.py | 0 2 files changed, 122 insertions(+) create mode 100644 fedot_ind/core/models/early_tc/metrics.py create mode 100644 fedot_ind/core/models/early_tc/teaser.py diff --git a/fedot_ind/core/models/early_tc/metrics.py b/fedot_ind/core/models/early_tc/metrics.py new file mode 100644 index 000000000..a4cb9cef5 --- /dev/null +++ b/fedot_ind/core/models/early_tc/metrics.py @@ -0,0 +1,122 @@ +from sklearn.metrics import confusion_matrix +import numpy as np +import pandas as pd +from fedot.core.data.data import InputData, OutputData +from typing import Tuple, List, Optional, Union, Literal + +def conf_matrix(actual, predicted): + cm = confusion_matrix(actual, predicted) + return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) + +def average_delay(boundaries, prediction, + point, + use_idx=True, + window_placement='lefter'): + cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) + # statistics + statistics = { + 'anomalies_num': len(cp_confusion['TPs']) + len(cp_confusion['FPs']), + 'FP_num': len(cp_confusion['FPs']), + 'missed': len(cp_confusion['FNs']) + } + time_func = { + 'righter': lambda triplet: triplet[1] - triplet[0], + 'lefter': lambda triplet: triplet[2] - triplet[1], + 'central': lambda triplet: triplet[1] - triplet[0] - (triplet[2] - triplet[0]) / 2 + }[window_placement] + + detection_history = { + i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() + } + return detection_history, statistics + + + +def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], + prediction: pd.DataFrame, + use_switch_point: bool = True, # if first anomaly dot is considered as changepoint + use_idx: bool = False): + if isinstance(boundaries, pd.DataFrame): + boundaries = boundaries.values.T + anomaly_tsp = prediction[prediction == 1].sort_index().index + TPs, FNs, FPs = {}, [], [] + + if boundaries.shape[1]: + + FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest + for i, (b_low, b_up) in enumerate(boundaries): + all_tsp_in_window = prediction[b_low: b_up].index + anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp + if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? + FNs.append(i if use_idx else all_tsp_in_window) + TPs[i] = [b_low, anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, b_up] + if not use_idx: + FNs.append(all_tsp_in_window - anomaly_tsp_in_window) + FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest + else: + FPs.append(anomaly_tsp) + + FPs = np.concatenate(FPs) + FNs = np.concatenate(FNs) + + return dict( + FP=FPs, + FN=FNs, + TP=TPs + ) + + +# cognate of single_detecting_boundaries +def get_boundaries(idx, actual_timestamps, window_size:int = None, + window_placement: Literal['left', 'right', 'central'] = 'left', + intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', + ): + # idx = idx + # cast everything to pandas object fir the subsequent comfort + if isinstance(idx, np.array): + if idx.dtype == np.dtype('O'): + idx = pd.to_datetime(pd.Series(idx)) + td = pd.Timedelta(window_size) + else: + idx = pd.Series(idx) + td = window_size + else: + raise TypeError('Unexpected type of ts index') + + boundaries = np.tile(actual_timestamps, (2, 1)) + # [0, ...] - lower bound, [1, ...] - upper + if window_placement == 'left': + boundaries[0] -= td + elif window_placement == 'central': + boundaries[0] -= td / 2 + boundaries[1] += td / 2 + elif window_placement == 'right': + boundaries[1] += td + else: + raise ValueError('Unknown mode') + + if not len(actual_timestamps): + return boundaries + + # intersection resolution + for i in range(len(actual_timestamps) - 1): + if not boundaries[0, i + 1] > boundaries[1, i]: + continue + + if intersection_mode == 'shift_to_left': + boundaries[0, i + 1] = boundaries[1, i] + elif intersection_mode == 'shift_to_right': + boundaries[1, i] = boundaries[0, i + 1] + elif intersection_mode == 'uniform': + boundaries[1, i], boundaries[0, i + 1] = boundaries[0, i + 1], boundaries[1, i] + else: + raise ValueError('Unknown intersection resolution') + + # filtering + idx_to_keep = np.abs(np.diff(boundaries, axis=0)) > 1e-6 + boundaries = boundaries[..., idx_to_keep] + boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) + return boundaries + + + diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py new file mode 100644 index 000000000..e69de29bb From c3de1151c4562ca4246755f6d9eb1e1052c227ed Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 20 Jun 2024 13:17:30 +0300 Subject: [PATCH 17/43] metrics ended --- fedot_ind/core/models/early_tc/metrics.py | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/fedot_ind/core/models/early_tc/metrics.py b/fedot_ind/core/models/early_tc/metrics.py index a4cb9cef5..f4a5f6544 100644 --- a/fedot_ind/core/models/early_tc/metrics.py +++ b/fedot_ind/core/models/early_tc/metrics.py @@ -30,7 +30,8 @@ def average_delay(boundaries, prediction, } return detection_history, statistics - +def tp_transform(tps): + return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], prediction: pd.DataFrame, @@ -49,7 +50,9 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? FNs.append(i if use_idx else all_tsp_in_window) - TPs[i] = [b_low, anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, b_up] + TPs[i] = [b_low, + anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, + b_up] if not use_idx: FNs.append(all_tsp_in_window - anomaly_tsp_in_window) FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest @@ -62,10 +65,9 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], return dict( FP=FPs, FN=FNs, - TP=TPs + TP=np.stack(TPs) ) - # cognate of single_detecting_boundaries def get_boundaries(idx, actual_timestamps, window_size:int = None, window_placement: Literal['left', 'right', 'central'] = 'left', @@ -118,5 +120,19 @@ def get_boundaries(idx, actual_timestamps, window_size:int = None, boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) return boundaries - +def nab(boundaries, predictions, mode='standard', custom_coefs=None): + inner_coefs = { + 'low_FP': [1.0, -0.11, -1.0], + 'standard': [1., -0.22, -1.], + 'lof_FN': [1., -0.11, -2.] + } + coefs = custom_coefs or inner_coefs[mode] + confusion_matrix = extract_cp_cm(boundaries, predictions) + + tps = confusion_matrix['tps'] + + score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], + coefs) + return score + From d4ee881c22c3afaa31536e43f756d9a637dfdf3c Mon Sep 17 00:00:00 2001 From: leostre Date: Mon, 24 Jun 2024 03:05:51 +0300 Subject: [PATCH 18/43] in basis teaser is completed, need some make-up and add cut ts support --- fedot_ind/core/metrics/interval_metrics.py | 138 +++++++++++++++++++++ fedot_ind/core/models/early_tc/teaser.py | 123 ++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 fedot_ind/core/metrics/interval_metrics.py diff --git a/fedot_ind/core/metrics/interval_metrics.py b/fedot_ind/core/metrics/interval_metrics.py new file mode 100644 index 000000000..f4a5f6544 --- /dev/null +++ b/fedot_ind/core/metrics/interval_metrics.py @@ -0,0 +1,138 @@ +from sklearn.metrics import confusion_matrix +import numpy as np +import pandas as pd +from fedot.core.data.data import InputData, OutputData +from typing import Tuple, List, Optional, Union, Literal + +def conf_matrix(actual, predicted): + cm = confusion_matrix(actual, predicted) + return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) + +def average_delay(boundaries, prediction, + point, + use_idx=True, + window_placement='lefter'): + cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) + # statistics + statistics = { + 'anomalies_num': len(cp_confusion['TPs']) + len(cp_confusion['FPs']), + 'FP_num': len(cp_confusion['FPs']), + 'missed': len(cp_confusion['FNs']) + } + time_func = { + 'righter': lambda triplet: triplet[1] - triplet[0], + 'lefter': lambda triplet: triplet[2] - triplet[1], + 'central': lambda triplet: triplet[1] - triplet[0] - (triplet[2] - triplet[0]) / 2 + }[window_placement] + + detection_history = { + i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() + } + return detection_history, statistics + +def tp_transform(tps): + return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) + +def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], + prediction: pd.DataFrame, + use_switch_point: bool = True, # if first anomaly dot is considered as changepoint + use_idx: bool = False): + if isinstance(boundaries, pd.DataFrame): + boundaries = boundaries.values.T + anomaly_tsp = prediction[prediction == 1].sort_index().index + TPs, FNs, FPs = {}, [], [] + + if boundaries.shape[1]: + + FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest + for i, (b_low, b_up) in enumerate(boundaries): + all_tsp_in_window = prediction[b_low: b_up].index + anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp + if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? + FNs.append(i if use_idx else all_tsp_in_window) + TPs[i] = [b_low, + anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, + b_up] + if not use_idx: + FNs.append(all_tsp_in_window - anomaly_tsp_in_window) + FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest + else: + FPs.append(anomaly_tsp) + + FPs = np.concatenate(FPs) + FNs = np.concatenate(FNs) + + return dict( + FP=FPs, + FN=FNs, + TP=np.stack(TPs) + ) + +# cognate of single_detecting_boundaries +def get_boundaries(idx, actual_timestamps, window_size:int = None, + window_placement: Literal['left', 'right', 'central'] = 'left', + intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', + ): + # idx = idx + # cast everything to pandas object fir the subsequent comfort + if isinstance(idx, np.array): + if idx.dtype == np.dtype('O'): + idx = pd.to_datetime(pd.Series(idx)) + td = pd.Timedelta(window_size) + else: + idx = pd.Series(idx) + td = window_size + else: + raise TypeError('Unexpected type of ts index') + + boundaries = np.tile(actual_timestamps, (2, 1)) + # [0, ...] - lower bound, [1, ...] - upper + if window_placement == 'left': + boundaries[0] -= td + elif window_placement == 'central': + boundaries[0] -= td / 2 + boundaries[1] += td / 2 + elif window_placement == 'right': + boundaries[1] += td + else: + raise ValueError('Unknown mode') + + if not len(actual_timestamps): + return boundaries + + # intersection resolution + for i in range(len(actual_timestamps) - 1): + if not boundaries[0, i + 1] > boundaries[1, i]: + continue + + if intersection_mode == 'shift_to_left': + boundaries[0, i + 1] = boundaries[1, i] + elif intersection_mode == 'shift_to_right': + boundaries[1, i] = boundaries[0, i + 1] + elif intersection_mode == 'uniform': + boundaries[1, i], boundaries[0, i + 1] = boundaries[0, i + 1], boundaries[1, i] + else: + raise ValueError('Unknown intersection resolution') + + # filtering + idx_to_keep = np.abs(np.diff(boundaries, axis=0)) > 1e-6 + boundaries = boundaries[..., idx_to_keep] + boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) + return boundaries + +def nab(boundaries, predictions, mode='standard', custom_coefs=None): + inner_coefs = { + 'low_FP': [1.0, -0.11, -1.0], + 'standard': [1., -0.22, -1.], + 'lof_FN': [1., -0.11, -2.] + } + coefs = custom_coefs or inner_coefs[mode] + confusion_matrix = extract_cp_cm(boundaries, predictions) + + tps = confusion_matrix['tps'] + + score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], + coefs) + return score + + diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index e69de29bb..66dc88745 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -0,0 +1,123 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters + + +class TEASER(ModelImplementation): + def __init__(self, params: Optional[OperationParameters] = None): + super().__init__() + if params is None: + params = {} + self.interval_length = params.get('interval_length', 10) # rewrite as interval_length + self.acceptance_threshold = params.get('acceptance_threshold', 5) + self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' + # how to pass into ? % what needed + self.oc_svm_params = {} + self.weasel_params = {} + self.random_state = None # is needed? + + def _init_model(self, max_data_length): + self.prediction_idx = self._compute_prediction_points(max_data_length) + self.n_pred = len(self.prediction_idx) + self.oc_estimators = [OneClassSVM(**self.oc_svm_params) for _ in range(self.n_pred)] + self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] + self.scalers = [StandardScaler() for _ in range(self.n_pred)] # do we need them separate? no inverse path expected + + def fit(self, input_data: InputData): + input_data = self.__convert_pd(input_data) + X, y = input_data.features, input_data.target # what's passed in case of classification to training? + self._init_model(max_data_length=X.shape[-1]) + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i]] # what's dimensionality of input? will it work in case of multivariate? + X_part = self.scalers[i].fit_transform(X_part) + probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + filtered_probas = self._filter_positive(probas, y) # + X_oc = self._form_X_oc(filtered_probas) + self.oc_estimators[i].fit(X_oc, y) + + def _predict_one_slave(self, X, i): + X_part = X[..., :self.prediction_idx[i]] + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + X_oc = self._form_X_oc(probas) + return X_oc, np.argmax(probas, axis=-1) + + def _compute_prediction_points(self, n_idx): + """Computes indices for prediction, includes last index, first interval may be greater""" + prediction_idx = np.arange(n_idx - 1, -1, -self.interval_length)[::-1] + self.earliness = 1 - prediction_idx / n_idx + return prediction_idx + + def _filter_positive(self, predicted_probas, y): # different logic in sktime + predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() + return predicted_probas[predicted_labels == y] + + def _form_X_oc(self, predicted_probas): + d = (predicted_probas.max() - predicted_probas) + d[d == 0] = 1 + d = d.min(axis=-1).reshape(-1, 1) + return np.hstack([predicted_probas, d]) + + def _predict(self, X): + n = X.shape[0] + self.states = np.ones((n, self.n_pred, 2)) # num_consec, class + X_ocs, predicted_labels = zip( + *[self._predict_one_slave(X, i) for i in range(self.n_pred)] + ) + non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold + to_oc_check = np.argwhere(non_acceptance) + X_ocs = np.stack(X_ocs) + predicted_labels = np.stack(predicted_labels) + # for each point of estimation + for i in range(self.n_pred): + # find not accepted points + ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] + X_to_ith = X_ocs[i][ith_point_to_oc] + # if they are not outliers + final_verdict = self.oc_estimators[i].predict(X_to_ith) # 1 for accept -1 for reject + # mark as accepted + non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False + predicted_labels[non_acceptance] = -1 + return predicted_labels + + def _consecutive_count(self, predicted_labels: List[np.array]): + n = len(predicted_labels[0]) + consecutive_labels = np.ones((self.n_pred, n)) + for i in range(1, self.n_pred): + equal = predicted_labels[i - 1] == predicted_labels[i] + consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 + return consecutive_labels # n_pred x n_instances + + def __convert_pd(self, input_data): + if hasattr(input_data.features, 'values'): + input_data.features = input_data.features.values + if hasattr(input_data.target, 'values'): + input_data.target = input_data.target.values + return input_data + + def predict(self, input_data: InputData) -> OutputData: + input_data = self.__convert_pd(input_data) + prediction = self._predict(input_data.features) + return self._convert_to_output(input_data, predict=prediction) + + def predict_for_fit(self, input_data: InputData) -> OutputData: + return self.predict(input_data) + + def _score(self, X, y, hm_shift_to_acc=None): + hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + predictions = self._predict(X) + accuracies = (predictions == np.tile(y, (1, self.n_pred))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness / (hm_shift_to_acc * accuracies + self.earliness) + + def _tune_oc(self): + #TODO + pass From 588846a5c6c3ac76fdd15cc6ee66bd39912cc8f9 Mon Sep 17 00:00:00 2001 From: leostre Date: Wed, 26 Jun 2024 17:09:09 +0300 Subject: [PATCH 19/43] teaser inherits sklearn's classifier mixin now --- fedot_ind/core/models/early_tc/__init__.py | 0 fedot_ind/core/models/early_tc/teaser.py | 61 +++++++++++++------ .../data/default_operation_params.json | 7 ++- .../data/industrial_model_repository.json | 11 ++++ fedot_ind/core/repository/model_repository.py | 5 +- fedot_ind/core/tuning/search_space.py | 24 ++++++++ tests/unit/core/models/test_teaser.py | 35 +++++++++++ 7 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/__init__.py create mode 100644 tests/unit/core/models/test_teaser.py diff --git a/fedot_ind/core/models/early_tc/__init__.py b/fedot_ind/core/models/early_tc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 66dc88745..66a031fd4 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -3,6 +3,7 @@ from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV from sktime.classification.dictionary_based import MUSE, WEASEL from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters @@ -13,21 +14,25 @@ def __init__(self, params: Optional[OperationParameters] = None): super().__init__() if params is None: params = {} + self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') self.interval_length = params.get('interval_length', 10) # rewrite as interval_length self.acceptance_threshold = params.get('acceptance_threshold', 5) self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' + # how to pass into ? % what needed - self.oc_svm_params = {} + self._oc_svm_params = [100, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1.5, 1] self.weasel_params = {} self.random_state = None # is needed? def _init_model(self, max_data_length): self.prediction_idx = self._compute_prediction_points(max_data_length) self.n_pred = len(self.prediction_idx) - self.oc_estimators = [OneClassSVM(**self.oc_svm_params) for _ in range(self.n_pred)] + self.oc_estimators = [None] * self.n_pred self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] - self.scalers = [StandardScaler() for _ in range(self.n_pred)] # do we need them separate? no inverse path expected + self.scalers = [StandardScaler() for _ in range(self.n_pred)] + self.__offset = max_data_length % self.interval_length + self.best_estimator_idx = -1 def fit(self, input_data: InputData): input_data = self.__convert_pd(input_data) @@ -35,17 +40,22 @@ def fit(self, input_data: InputData): self._init_model(max_data_length=X.shape[-1]) for i in range(self.n_pred): self._fit_one_interval(X, y, i) + self.best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i]] # what's dimensionality of input? will it work in case of multivariate? + X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? X_part = self.scalers[i].fit_transform(X_part) probas = self.slave_estimators[i].fit_predict_proba(X_part, y) - filtered_probas = self._filter_positive(probas, y) # + filtered_probas = self._filter_trues(probas, y) # X_oc = self._form_X_oc(filtered_probas) - self.oc_estimators[i].fit(X_oc, y) + self.oc_estimators[i] = GridSearchCV(OneClassSVM(), + param_grid={"gamma": self._oc_svm_params}, + scoring='accuracy', + cv=min(X.shape[0], 10) + ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ - def _predict_one_slave(self, X, i): - X_part = X[..., :self.prediction_idx[i]] + def _predict_one_slave(self, X, i, offset=0): + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] X_part = self.scalers[i].transform(X_part) probas = self.slave_estimators[i].predict_proba(X_part) X_oc = self._form_X_oc(probas) @@ -57,7 +67,7 @@ def _compute_prediction_points(self, n_idx): self.earliness = 1 - prediction_idx / n_idx return prediction_idx - def _filter_positive(self, predicted_probas, y): # different logic in sktime + def _filter_trues(self, predicted_probas, y): # different logic in sktime predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() return predicted_probas[predicted_labels == y] @@ -70,15 +80,20 @@ def _form_X_oc(self, predicted_probas): def _predict(self, X): n = X.shape[0] self.states = np.ones((n, self.n_pred, 2)) # num_consec, class + if self.prediction_mode == 'best_by_harmonic_mean': + estimator_indices = [self.best_estimator_idx] + else: + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = list(range(last_idx + 1)) X_ocs, predicted_labels = zip( - *[self._predict_one_slave(X, i) for i in range(self.n_pred)] + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) predicted_labels = np.stack(predicted_labels) # for each point of estimation - for i in range(self.n_pred): + for i in range(predicted_labels.shape[0]): # find not accepted points ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] X_to_ith = X_ocs[i][ith_point_to_oc] @@ -87,15 +102,16 @@ def _predict(self, X): # mark as accepted non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False predicted_labels[non_acceptance] = -1 - return predicted_labels + return predicted_labels # prediction_points x n_instances def _consecutive_count(self, predicted_labels: List[np.array]): n = len(predicted_labels[0]) - consecutive_labels = np.ones((self.n_pred, n)) - for i in range(1, self.n_pred): + prediction_points = len(predicted_labels) + consecutive_labels = np.ones((prediction_points, n)) + for i in range(1, prediction_points): equal = predicted_labels[i - 1] == predicted_labels[i] consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 - return consecutive_labels # n_pred x n_instances + return consecutive_labels # prediction_points x n_instances def __convert_pd(self, input_data): if hasattr(input_data.features, 'values'): @@ -115,9 +131,14 @@ def predict_for_fit(self, input_data: InputData) -> OutputData: def _score(self, X, y, hm_shift_to_acc=None): hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc predictions = self._predict(X) - accuracies = (predictions == np.tile(y, (1, self.n_pred))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness / (hm_shift_to_acc * accuracies + self.earliness) + prediction_points = predictions.shape[0] + accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - def _tune_oc(self): - #TODO - pass + def _get_applicable_index(self, last_available_idx): + idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') + if idx == 0: + raise RuntimeError('Too few points for prediction!') + idx -= 1 + offset = last_available_idx - self.prediction_idx[idx] + return idx, offset diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 2ac985979..3f3cfbfe5 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -124,6 +124,11 @@ "min_samples_leaf": 10, "bootstrap": false }, + "teaser": { + "interval_length": 10, + "acceptance_threshold": 3, + "hm_shift_to_acc": 2 + }, "dt": { "max_depth": 5, "min_samples_split": 10, @@ -438,4 +443,4 @@ "kernel": "rbf", "gamma": "auto" } -} \ No newline at end of file +} diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index c4a87aefb..78674cfba 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -729,6 +729,17 @@ "non_linear" ] }, + "teaser": { + "meta": "ts_model", + "presets": ["fast_train", "ts"], + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.ts]" + }, "xgboost": { "meta": "sklearn_class", "presets": ["*tree"], diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index ab73b3c85..da4ae22e3 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -45,6 +45,7 @@ from fedot_ind.core.models.detection.custom.stat_detector import StatisticalDetector from fedot_ind.core.models.detection.probalistic.kalman import UnscentedKalmanFilter from fedot_ind.core.models.detection.subspaces.sst import SingularSpectrumTransformation +from fedot_ind.core.models.early_tc.teaser import TEASER from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR @@ -88,7 +89,9 @@ class AtomizedModel(Enum): # external models 'lgbm': LGBMClassifier, # for detection - 'one_class_svm': OneClassSVM + 'one_class_svm': OneClassSVM, + # Early classification + 'teaser': TEASER } FEDOT_PREPROC_MODEL = { # data standartization diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 9d1f6c858..4cff3aa96 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -65,6 +65,30 @@ 'selection_strategy': {'hyperopt-dist': hp.choice, 'sampling-scope': [['sum', 'pairwise']]} }, + 'teaser': + {'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'acceptance_threshold': {'hyperopt-dist': hp.choice, + 'sampling_scope': [[1, 2, 3, 4, 5]]}, + 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + }, + 'deepar_model': + {'epochs': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[x for x in range(10, 100, 10)]]}, + 'batch_size': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[x for x in range(8, 64, 6)]]}, + 'dropout': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(0, 0.6, 0.1))]}, + 'rnn_layers':{'hyperopt-dist': hp.choice, + 'sampling-scope': [range(1, 6)]}, + 'hidden_size':{'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(10, 101, 10))]}, + 'cell_type':{'hyperopt-dist': hp.choice, + 'sampling-scope': [['GRU', 'LSTM', 'RNN']]}, + 'expected_distribution': {'hyperopt-dist': hp.choice, + 'sampling-scope': [['normal', 'cauchy']]} + }, 'patch_tst_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}, diff --git a/tests/unit/core/models/test_teaser.py b/tests/unit/core/models/test_teaser.py new file mode 100644 index 000000000..2bc19b8de --- /dev/null +++ b/tests/unit/core/models/test_teaser.py @@ -0,0 +1,35 @@ +import pytest +import numpy as np +from fedot_ind.core.models.early_tc import teaser as TEASER + + +@pytest.fixture(scope='module') +def teaser(): + teaser = TEASER.TEASER({'interval_length': 10, 'prediction_mode': ''}) + return teaser + +@pytest.fixture(scope='module') +def xy(): + return np.random.randn((2, 23)), np.random.randint(0, 2, size=(2, 1)) + +def test_get_applicable_index(teaser): + teaser._init_model(23) + idx, offset = teaser._get_last_applicable_idx(100) + assert offset == 100 - 22, 'Wrong offset estimation when right edge' + assert idx == len(teaser.prediction_idx) - 1 + idx, offset = teaser._get_last_applicable_idx(12) + assert offset == 100 - teaser.prediction_idx[idx], 'Wrong offset estimation in the middle' + assert idx == len(teaser.prediction_idx) - 1 + +def test_compute_prediction_points(teaser): + indices = teaser._compute_prediction_points(23) + assert 2 in indices + assert 22 in indices + assert 23 not in indices + +# def test_consecutive_count(teaser): +# pass + +# def test_score(teaser): + + From d939a340d1e6dead45344749dcbf43d6af2a49bc Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 27 Jun 2024 18:22:34 +0300 Subject: [PATCH 20/43] class tree reconf. added proba_thresholding classifier (not registered) --- .../core/models/early_tc/base_early_tc.py | 117 +++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 46 +++++++ fedot_ind/core/models/early_tc/teaser.py | 123 +++++------------- 3 files changed, 194 insertions(+), 92 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/base_early_tc.py create mode 100644 fedot_ind/core/models/early_tc/prob_threshold.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py new file mode 100644 index 000000000..f97ba0593 --- /dev/null +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -0,0 +1,117 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum + + +class BaseETC(ClassifierMixin, BaseEstimator): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') + self.interval_percentage = params.get('interval_percentage', 10) + self.consecutive_predictions = params.get('consecutive_predictions', 3) + self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + self.random_state = params.get('random_state', None) + self.weasel_params = {} + assert self.consecutive_predictions < self.interval_percentage, 'Not enough checkpoints for prediction proof' + + def _init_model(self, X, y): + max_data_length = X.shape[-1] + self.prediction_idx = self._compute_prediction_points(max_data_length) + self.n_pred = len(self.prediction_idx) + self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] + self.scalers = [StandardScaler() for _ in range(self.n_pred)] + self._best_estimator_idx = -1 + self.classes_ = [np.unique(y)] + + @property + def required_length(self): + if not hasattr(self, '_best_estimator_idx'): + return None + return self.prediction_idx[self._best_estimator_idx] + + def fit(self, X, y=None): + assert y is not None, 'Pass y' + y = np.array(y).flatten() + self._init_model(X, y) + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + self._best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? + X_part = self.scalers[i].fit_transform(X_part) + probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + return probas + + def _predict_one_slave(self, X, i, offset=0): + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + return probas, np.argmax(probas, axis=-1) + + def _compute_prediction_points(self, n_idx): + interval_length = int(n_idx * self.interval_percentage / 100) + prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1] + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + return prediction_idx + + def _select_estimators(self, X): + offset = 0 + if self.prediction_mode == 'best_by_harmonic_mean': + estimator_indices = [self._best_estimator_idx] + elif self.prediction_mode == 'all': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = np.arange(last_idx + 1) + else: + raise ValueError('Unknown prediction mode') + return estimator_indices, offset + + def _predict(self, X,): + estimator_indices, offset = self._select_estimators(X) + predicted_probas, predicted_labels = zip( + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary + ) + return predicted_labels, predicted_probas + + def _consecutive_count(self, predicted_labels: List[np.array]): + n = len(predicted_labels[0]) + prediction_points = len(predicted_labels) + consecutive_labels = np.ones((prediction_points, n)) + for i in range(1, prediction_points): + equal = predicted_labels[i - 1] == predicted_labels[i] + consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 + return consecutive_labels # prediction_points x n_instances + + def predict_proba(self, X): + raise NotImplementedError + + def predict(self, X): + raise NotImplementedError + + def _score(self, X, y, hm_shift_to_acc=None): + y = np.array(y).flatten() + hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + predictions, *_ = self._predict(X) + prediction_points = predictions.shape[0] + accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) + return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) + + def _get_applicable_index(self, last_available_idx): + idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') + if idx == 0: + raise RuntimeError('Too few points for prediction!') + idx -= 1 + offset = last_available_idx - self.prediction_idx[idx] + return idx, offset diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py new file mode 100644 index 000000000..0433de34a --- /dev/null +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -0,0 +1,46 @@ +from typing import Union, List, Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data +from fedot.core.data.data import InputData, OutputData +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator +from sktime.classification.dictionary_based import MUSE, WEASEL +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC + +class ProbabilityThresholdClassifier(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + self.probability_threshold = params.get('probability_threshold', None) + + def _init_model(self, X, y): + super()._init_model(X, y) + if self.probability_threshold is None: + self.probability_threshold = 1 / len(self.classes_[0]) + + def predict_proba(self, X): + _, predicted_probas, non_acceptance = self._predict(X) + predicted_probas[non_acceptance] = 0 + return predicted_probas.squeeze() + + def predict(self, X): + predicted_labels, _, non_acceptance = self._predict(X) + predicted_labels[non_acceptance] = -1 + # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] + return predicted_labels # prediction_points x n_instances + + def _predict(self, X): + predicted_labels, predicted_probas = super()._predict(X) + predicted_probas = np.stack(predicted_probas) + predicted_labels = np.stack(predicted_labels) + non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions + double_check = predicted_probas.max(axis=-1) > self.probability_threshold + non_acceptance[non_acceptance & double_check] = False + return predicted_labels, predicted_probas, non_acceptance diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 66a031fd4..2809824c8 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,51 +1,30 @@ from typing import Union, List, Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV +from sklearn.base import ClassifierMixin, BaseEstimator from sktime.classification.dictionary_based import MUSE, WEASEL from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC -class TEASER(ModelImplementation): +class TEASER(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): - super().__init__() - if params is None: - params = {} - self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') - self.interval_length = params.get('interval_length', 10) # rewrite as interval_length - self.acceptance_threshold = params.get('acceptance_threshold', 5) - self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) - assert self.acceptance_threshold < self.interval_length, 'Not enough checkpoints for prediction proof' + super().__init__(params) + self._oc_svm_params = (100., 10., 5., 2.5, 1.5, 1., 0.5, 0.25, 0.1) - # how to pass into ? % what needed - self._oc_svm_params = [100, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1.5, 1] - self.weasel_params = {} - self.random_state = None # is needed? - - def _init_model(self, max_data_length): - self.prediction_idx = self._compute_prediction_points(max_data_length) - self.n_pred = len(self.prediction_idx) + def _init_model(self, X, y): + super()._init_model(X, y) self.oc_estimators = [None] * self.n_pred - self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] - self.scalers = [StandardScaler() for _ in range(self.n_pred)] - self.__offset = max_data_length % self.interval_length - self.best_estimator_idx = -1 - - def fit(self, input_data: InputData): - input_data = self.__convert_pd(input_data) - X, y = input_data.features, input_data.target # what's passed in case of classification to training? - self._init_model(max_data_length=X.shape[-1]) - for i in range(self.n_pred): - self._fit_one_interval(X, y, i) - self.best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? - X_part = self.scalers[i].fit_transform(X_part) - probas = self.slave_estimators[i].fit_predict_proba(X_part, y) + probas = super()._fit_one_interval(X, y, i) filtered_probas = self._filter_trues(probas, y) # X_oc = self._form_X_oc(filtered_probas) self.oc_estimators[i] = GridSearchCV(OneClassSVM(), @@ -55,17 +34,9 @@ def _fit_one_interval(self, X, y, i): ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ def _predict_one_slave(self, X, i, offset=0): - X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] - X_part = self.scalers[i].transform(X_part) - probas = self.slave_estimators[i].predict_proba(X_part) + probas, labels = super()._predict_one_slave(X, i, offset) X_oc = self._form_X_oc(probas) - return X_oc, np.argmax(probas, axis=-1) - - def _compute_prediction_points(self, n_idx): - """Computes indices for prediction, includes last index, first interval may be greater""" - prediction_idx = np.arange(n_idx - 1, -1, -self.interval_length)[::-1] - self.earliness = 1 - prediction_idx / n_idx - return prediction_idx + return X_oc, probas, labels def _filter_trues(self, predicted_probas, y): # different logic in sktime predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() @@ -78,67 +49,35 @@ def _form_X_oc(self, predicted_probas): return np.hstack([predicted_probas, d]) def _predict(self, X): - n = X.shape[0] - self.states = np.ones((n, self.n_pred, 2)) # num_consec, class - if self.prediction_mode == 'best_by_harmonic_mean': - estimator_indices = [self.best_estimator_idx] - else: - last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) - estimator_indices = list(range(last_idx + 1)) - X_ocs, predicted_labels = zip( + estimator_indices, offset = self._select_estimators(X) + X_ocs, predicted_probas, predicted_labels = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) - non_acceptance = self._consecutive_count(predicted_labels) < self.acceptance_threshold + non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) + predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) + final_verdicts = np.zeros((len(estimator_indices), X.shape[0])) # for each point of estimation for i in range(predicted_labels.shape[0]): # find not accepted points ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] X_to_ith = X_ocs[i][ith_point_to_oc] # if they are not outliers - final_verdict = self.oc_estimators[i].predict(X_to_ith) # 1 for accept -1 for reject + final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject # mark as accepted - non_acceptance[i, np.argwhere(final_verdict == 1).flatten()] = False + non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False + final_verdicts[i] = final_verdict + return predicted_labels, predicted_probas, non_acceptance, final_verdicts + + def predict_proba(self, X): + _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) + predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] + return predicted_probas.squeeze() + + def predict(self, X): + predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) predicted_labels[non_acceptance] = -1 + # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - - def _consecutive_count(self, predicted_labels: List[np.array]): - n = len(predicted_labels[0]) - prediction_points = len(predicted_labels) - consecutive_labels = np.ones((prediction_points, n)) - for i in range(1, prediction_points): - equal = predicted_labels[i - 1] == predicted_labels[i] - consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 - return consecutive_labels # prediction_points x n_instances - - def __convert_pd(self, input_data): - if hasattr(input_data.features, 'values'): - input_data.features = input_data.features.values - if hasattr(input_data.target, 'values'): - input_data.target = input_data.target.values - return input_data - - def predict(self, input_data: InputData) -> OutputData: - input_data = self.__convert_pd(input_data) - prediction = self._predict(input_data.features) - return self._convert_to_output(input_data, predict=prediction) - - def predict_for_fit(self, input_data: InputData) -> OutputData: - return self.predict(input_data) - - def _score(self, X, y, hm_shift_to_acc=None): - hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc - predictions = self._predict(X) - prediction_points = predictions.shape[0] - accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - - def _get_applicable_index(self, last_available_idx): - idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') - if idx == 0: - raise RuntimeError('Too few points for prediction!') - idx -= 1 - offset = last_available_idx - self.prediction_idx[idx] - return idx, offset From 4d3e57dd0efee6add236d1ae68dd107feee52f83 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 28 Jun 2024 14:11:17 +0300 Subject: [PATCH 21/43] both etc models are registered, available via api --- fedot_ind/core/models/early_tc/base_early_tc.py | 11 ++--------- .../core/models/early_tc/prob_threshold.py | 12 +----------- fedot_ind/core/models/early_tc/teaser.py | 10 +--------- .../data/default_operation_params.json | 9 +++++++-- .../data/industrial_model_repository.json | 17 +++++++++++++---- fedot_ind/core/repository/model_repository.py | 4 +++- fedot_ind/core/tuning/search_space.py | 12 ++++++++++-- 7 files changed, 37 insertions(+), 38 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index f97ba0593..c7b84bedf 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -1,16 +1,9 @@ -from typing import Union, List, Optional +from typing import Optional, List from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data -from fedot.core.data.data import InputData, OutputData -from sklearn.svm import OneClassSVM from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import GridSearchCV from sklearn.base import ClassifierMixin, BaseEstimator -from sktime.classification.dictionary_based import MUSE, WEASEL -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from sktime.classification.dictionary_based import WEASEL from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum class BaseETC(ClassifierMixin, BaseEstimator): diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 0433de34a..343077cbe 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -1,16 +1,6 @@ -from typing import Union, List, Optional +from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data -from fedot.core.data.data import InputData, OutputData -from sklearn.svm import OneClassSVM -from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import GridSearchCV -from sklearn.base import ClassifierMixin, BaseEstimator -from sktime.classification.dictionary_based import MUSE, WEASEL -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot_ind.core.models.early_tc.base_early_tc import BaseETC class ProbabilityThresholdClassifier(BaseETC): diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 2809824c8..f5d2590b3 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,16 +1,8 @@ -from typing import Union, List, Optional +from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_input_data -from fedot.core.data.data import InputData, OutputData from sklearn.svm import OneClassSVM -from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV -from sklearn.base import ClassifierMixin, BaseEstimator -from sktime.classification.dictionary_based import MUSE, WEASEL -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot_ind.core.models.early_tc.base_early_tc import BaseETC diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 3f3cfbfe5..11980e4b4 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -125,8 +125,13 @@ "bootstrap": false }, "teaser": { - "interval_length": 10, - "acceptance_threshold": 3, + "interval_percentage": 10, + "consecutive_predictions": 3, + "hm_shift_to_acc": 2 + }, + "proba_threshold_etc": { + "interval_percentage": 10, + "consecutive_predictions": 3, "hm_shift_to_acc": 2 }, "dt": { diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 78674cfba..ccbd1e0a7 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -623,7 +623,7 @@ }, "ridge": { "meta": "sklearn_regr", - "presets": ["fast_train", "ts"], + "presets": ["fast_train"], "tags": [ "simple", "linear", @@ -730,15 +730,24 @@ ] }, "teaser": { - "meta": "ts_model", - "presets": ["fast_train", "ts"], + "meta": "sklearn_class", "tags": [ "simple", "interpretable", "non_lagged", "non_linear" ], - "input_type": "[DataTypesEnum.ts]" + "input_type": "[DataTypesEnum.table]" + }, + "proba_threshold_etc": { + "meta": "sklearn_class", + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" }, "xgboost": { "meta": "sklearn_class", diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index da4ae22e3..ac23c4f22 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -46,6 +46,7 @@ from fedot_ind.core.models.detection.probalistic.kalman import UnscentedKalmanFilter from fedot_ind.core.models.detection.subspaces.sst import SingularSpectrumTransformation from fedot_ind.core.models.early_tc.teaser import TEASER +from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR @@ -91,7 +92,8 @@ class AtomizedModel(Enum): # for detection 'one_class_svm': OneClassSVM, # Early classification - 'teaser': TEASER + 'teaser': TEASER, + 'proba_threshold_etc': ProbabilityThresholdClassifier } FEDOT_PREPROC_MODEL = { # data standartization diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 4cff3aa96..994db4e51 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -73,15 +73,23 @@ 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, }, + 'proba_threshold_etc': + {'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'acceptance_threshold': {'hyperopt-dist': hp.choice, + 'sampling_scope': [[1, 2, 3, 4, 5]]}, + 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + }, 'deepar_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}, 'dropout': {'hyperopt-dist': hp.choice, - 'sampling-scope': [list(range(0, 0.6, 0.1))]}, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, 'rnn_layers':{'hyperopt-dist': hp.choice, - 'sampling-scope': [range(1, 6)]}, + 'sampling-scope': [list(range(1, 6))]}, 'hidden_size':{'hyperopt-dist': hp.choice, 'sampling-scope': [list(range(10, 101, 10))]}, 'cell_type':{'hyperopt-dist': hp.choice, From 4da7c13cb5b778b435c532111480426e137af223 Mon Sep 17 00:00:00 2001 From: leostre Date: Tue, 2 Jul 2024 15:27:34 +0300 Subject: [PATCH 22/43] ecec added --- .../core/models/early_tc/base_early_tc.py | 22 ++++--- fedot_ind/core/models/early_tc/ecec.py | 62 +++++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 7 ++- fedot_ind/core/models/early_tc/teaser.py | 9 +++ 4 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/ecec.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index c7b84bedf..5da8e61bd 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -33,6 +33,10 @@ def required_length(self): if not hasattr(self, '_best_estimator_idx'): return None return self.prediction_idx[self._best_estimator_idx] + + @property + def n_classes(self): + return len(self.classes_[0]) def fit(self, X, y=None): assert y is not None, 'Pass y' @@ -40,7 +44,6 @@ def fit(self, X, y=None): self._init_model(X, y) for i in range(self.n_pred): self._fit_one_interval(X, y, i) - self._best_estimator_idx = np.argmax(self._score(X, y, self.hm_shift_to_acc)) def _fit_one_interval(self, X, y, i): X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? @@ -60,23 +63,23 @@ def _compute_prediction_points(self, n_idx): self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 return prediction_idx - def _select_estimators(self, X): + def _select_estimators(self, X, training=False): offset = 0 - if self.prediction_mode == 'best_by_harmonic_mean': + if not training and self.prediction_mode == 'best_by_harmonic_mean': estimator_indices = [self._best_estimator_idx] - elif self.prediction_mode == 'all': + elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) else: raise ValueError('Unknown prediction mode') return estimator_indices, offset - def _predict(self, X,): - estimator_indices, offset = self._select_estimators(X) - predicted_probas, predicted_labels = zip( + def _predict(self, X, training=True): + estimator_indices, offset = self._select_estimators(X, training) + prediction = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) - return predicted_labels, predicted_probas + return prediction # see the output in _predict_one_slave def _consecutive_count(self, predicted_labels: List[np.array]): n = len(predicted_labels[0]) @@ -96,11 +99,12 @@ def predict(self, X): def _score(self, X, y, hm_shift_to_acc=None): y = np.array(y).flatten() hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc - predictions, *_ = self._predict(X) + predictions = self._predict(X)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) + def _get_applicable_index(self, last_available_idx): idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') if idx == 0: diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py new file mode 100644 index 000000000..a00df631d --- /dev/null +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -0,0 +1,62 @@ +from typing import Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from sklearn.model_selection import cross_val_predict +from sklearn.base import clone +from sklearn.metrics import confusion_matrix + +class ECEC(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + super().__init__(params) + + def _init_model(self, X, y): + super()._init_model(X, y) + self._confidences = np.ones((X.shape[0], self.n_pred)) + + def _score(self, X, y, alpha): + y = y.astype(int) + predicted_labels = np.stack(super()._predict(X)[0]).astype(int) # n_pred x n_inst + n = predicted_labels.shape[0] + accuracies = (predicted_labels == np.tile(y, (1, n))) # n_pred x n_inst + confidences = np.ones((n, X.shape[0]), dtype='float32') + for i in range(n): + y_pred = predicted_labels[i] + reliability_i = confusion_matrix(y, y_pred, normalize='pred') + confidences[i] = 1 - reliability_i[y, y_pred] # n_inst + confidences = 1 - np.cumprod(confidences, axis=0) # n_pred x n_inst + candidates = self._select_thrs(confidences) # n_candidates + cfs = np.zeros_like(candidates) + for i, candidate in enumerate(candidates): + mask = confidences >= candidate # n_pred x n_inst + accuracy_for_candidate = (accuracies * mask).sum(1) / mask.sum(1) # n_pred + cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) + return candidates[np.argmin(cfs)] + + @staticmethod + def _select_thrs(confidences): + C = np.unique(confidences.round(3)) + difference = np.diff(C) + pair_means = C[:-1] + difference / 2 + difference_shifted = np.roll(difference, 1) + difference_idx = np.argwhere(difference > difference_shifted) + return pair_means[difference_idx].flatten() + + @staticmethod + def cost_func(earliness, accuracies, alpha): + return alpha * accuracies + (1 - alpha) * earliness + + def fit(self, X, y): + self.confidence_threshold = super().fit(X, y) + + + + + + + + + + + + diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 343077cbe..f6c9ab65d 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -21,15 +21,16 @@ def predict_proba(self, X): return predicted_probas.squeeze() def predict(self, X): - predicted_labels, _, non_acceptance = self._predict(X) + predicted_labels, _, non_acceptance = self._predict(X, training=False) predicted_labels[non_acceptance] = -1 # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - def _predict(self, X): - predicted_labels, predicted_probas = super()._predict(X) + def _predict(self, X, training=True): + predicted_probas, predicted_labels = super()._predict(X, training) predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) + # print(predicted_labels.shape, predicted_probas.shape) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions double_check = predicted_probas.max(axis=-1) > self.probability_threshold non_acceptance[non_acceptance & double_check] = False diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index f5d2590b3..ff58cf72d 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -73,3 +73,12 @@ def predict(self, X): predicted_labels[non_acceptance] = -1 # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances + + def _score(self, X, y, hm_shift_to_acc=None): + scores = super()._score(X, y, hm_shift_to_acc) + self._best_estimator_idx = np.argmax(scores) + return scores + + def fit(self, X, y): + super().fit(X, y) + return self._score(X, y, self.hm_shift_to_acc) From d11fa8d2bb776f110b98719b991c53cd2542701e Mon Sep 17 00:00:00 2001 From: leostre Date: Thu, 4 Jul 2024 18:23:13 +0300 Subject: [PATCH 23/43] economy_k added --- .../core/models/early_tc/base_early_tc.py | 9 +- fedot_ind/core/models/early_tc/economy_k.py | 89 +++++++++++++++++++ .../core/models/early_tc/prob_threshold.py | 10 +++ fedot_ind/core/models/early_tc/teaser.py | 6 +- 4 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 fedot_ind/core/models/early_tc/economy_k.py diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 5da8e61bd..5e180c2a9 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -14,7 +14,8 @@ def __init__(self, params: Optional[OperationParameters] = None): self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') self.interval_percentage = params.get('interval_percentage', 10) self.consecutive_predictions = params.get('consecutive_predictions', 3) - self.hm_shift_to_acc = params.get('hm_shift_to_acc', 1.) + self.accuracy_importance = params.get('accuracy_importance', 1.) + self.min_ts_length = params.get('min_ts_step', 3) self.random_state = params.get('random_state', None) self.weasel_params = {} assert self.consecutive_predictions < self.interval_percentage, 'Not enough checkpoints for prediction proof' @@ -58,8 +59,8 @@ def _predict_one_slave(self, X, i, offset=0): return probas, np.argmax(probas, axis=-1) def _compute_prediction_points(self, n_idx): - interval_length = int(n_idx * self.interval_percentage / 100) - prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1] + interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) + prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1][1:] self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 return prediction_idx @@ -98,7 +99,7 @@ def predict(self, X): def _score(self, X, y, hm_shift_to_acc=None): y = np.array(y).flatten() - hm_shift_to_acc = hm_shift_to_acc or self.hm_shift_to_acc + hm_shift_to_acc = hm_shift_to_acc or self.accuracy_importance predictions = self._predict(X)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py new file mode 100644 index 000000000..639e680cd --- /dev/null +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -0,0 +1,89 @@ +from typing import Optional +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from sklearn.cluster import KMeans +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import train_test_split, cross_val_predict + +class EconomyK(BaseETC): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__(params) + self.lambda_ = params.get('lambda', 1.) + self._cluster_factor = params.get('cluster_factor' , 1) + self._random_state = 2104 + self.__cv = 5 + + def _init_model(self, X, y): + super()._init_model(X, y) + self.n_clusters = int(self._cluster_factor * self.n_classes) + self._clusterizer = KMeans(self.n_clusters, random_state=self._random_state) + self.state = np.zeros((self.n_pred, self.n_clusters, self.n_classes, self.n_classes)) + + def fit(self, X, y): + y = y.flatten().astype(int) + self._init_model(X, y) + self._pyck_ = confusion_matrix(y, self._clusterizer.fit(X).labels_, normalize='true')[:self.n_classes, :self.n_clusters] + for i in range(self.n_pred): + self._fit_one_interval(X, y, i) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] + X_part = self.scalers[i].fit_transform(X_part) + y_pred = cross_val_predict(self.slave_estimators[i], X_part, y, cv=self.__cv) + self.slave_estimators[i].fit(X_part, y) + states_by_i = np.zeros(( self.n_clusters, self.n_classes, self.n_classes)) + np.add.at(states_by_i, (self._clusterizer.labels_, y, y_pred), 1) + states_by_i /= np.mean(states_by_i, -2, keepdims=True) + states_by_i[np.isnan(states_by_i)] = 0 + states_by_i[:, np.eye(self.n_classes).astype(bool)] = 0 + self.state[i] = states_by_i + + def _predict_one_slave(self, X, i, offset=0): + cluster_centers = self._clusterizer.cluster_centers_[:, :self.prediction_idx[i] + 1] # n_clust x len + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] # n_inst x len + X_part = self.scalers[i].transform(X_part) + probas = self.slave_estimators[i].predict_proba(X_part) + optimal_time, is_optimal = self._get_prediction_time(X_part, cluster_centers, i) + return probas, optimal_time, is_optimal + + def __cluster_probas(self, X, centroids): + length = centroids.shape[-1] + diffs = np.subtract.outer(X, centroids).swapaxes(1, 2) + diffs = diffs[..., np.eye(length).astype(bool)] # n_inst x n_clust x len + distances = np.linalg.norm(diffs, axis=-1) + delta_k = 1. - distances / distances.mean(axis=-1)[:, None] + s = 1. / (1. + np.exp(-self.lambda_ * delta_k)) + return s / s.sum(axis=-1)[:, None] # n_inst x n_clust + + def __expected_costs(self, X, cluster_centroids, i): + cluster_probas = self.__cluster_probas(X, cluster_centroids) # n_inst x n_clust + s_glob = np.sum(np.transpose( + np.sum(self.state[i:], axis=-1), axes=(0, 2, 1) + ) * self._pyck_[None, ...], axis=1) + costs = cluster_probas @ s_glob.T # n_inst x time_left + costs += self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? + return costs + + def _get_prediction_time(self, X, cluster_centroids, i): + costs = self.__expected_costs(X, cluster_centroids, i) + min_costs = np.argmin(costs, axis=-1) + is_optimal = min_costs == 0 + time_optimal = self.prediction_idx[min_costs + i] + return time_optimal, is_optimal # n_inst + + def predict_proba(self, X): + probas, times, is_optimal = self._predict(X) + is_optimal = np.stack(is_optimal) + idx = np.tile(np.arange(self.n_pred), (is_optimal.shape[1], 1)).T # n_pred x n_inst + idx[~is_optimal] = self.n_pred + idx = np.argmin(idx, 0) + probas = np.stack(probas) + return probas[idx], np.stack(times)[idx] + + def predict(self, X): + probas, times = self.predict_proba(X) + labels = probas.argmax(-1) + return labels, times diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index f6c9ab65d..51d169909 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -35,3 +35,13 @@ def _predict(self, X, training=True): double_check = predicted_probas.max(axis=-1) > self.probability_threshold non_acceptance[non_acceptance & double_check] = False return predicted_labels, predicted_probas, non_acceptance + + def _score(self, X, y, accuracy_importance=None): + scores = super()._score(X, y, accuracy_importance) + self._best_estimator_idx = np.argmax(scores) + return scores + + def fit(self, X, y): + super().fit(X, y) + return self._score(X, y, self.accuracy_importance) + diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index ff58cf72d..c03c5d1e6 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -74,11 +74,11 @@ def predict(self, X): # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] return predicted_labels # prediction_points x n_instances - def _score(self, X, y, hm_shift_to_acc=None): - scores = super()._score(X, y, hm_shift_to_acc) + def _score(self, X, y, accuracy_importance=None): + scores = super()._score(X, y, accuracy_importance) self._best_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) - return self._score(X, y, self.hm_shift_to_acc) + return self._score(X, y, self.accuracy_importance) From 9f1624401b0693126c2984247505c00ea6cf11bb Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 5 Jul 2024 12:30:52 +0300 Subject: [PATCH 24/43] mlstm init --- .../core/models/nn/network_impl/mlstm.py | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 fedot_ind/core/models/nn/network_impl/mlstm.py diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py new file mode 100644 index 000000000..31ca6fbd4 --- /dev/null +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -0,0 +1,127 @@ +from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel +from typing import Optional, Callable, Any, List, Union +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.data.data import InputData, OutputData +from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE +import torch.optim as optim +import torch.nn as nn +import torch.nn.functional as F +import torch +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +import pandas as pd +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams +from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter +import torch.utils.data as data +from fedot_ind.core.architecture.settings.computational import default_device + +class SqueezeExciteBlock(nn.Module): + def __init__(self, input_channels, filters, reduce=4): + super().__init__() + self.filters = filters + self.pool = nn.AvgPool1d(input_channels) + self.bottleneck = max(self.filters // reduce, 4) + self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) + self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) + torch.nn.init.kaiming_normal_(self.fc1.weight.data) + torch.nn.init.kaiming_normal_(self.fc2.weight.data) + + def forward(self, x): + input_x = x + x = self.pool(x) + x = F.relu(self.fc1(x.view(-1, 1, self.filters))) + x = F.sigmoid(self.fc2(x)) + x = x.view(-1, self.filters, 1) * input_x + return x + +class MLSTM_module(nn.Module): + def __init__(self, input_size, input_channels, + inner_size, inner_channels, + output_size, num_layers, dropout=0.25): + super().__init__() + self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) + self.lstm = nn.LSTM(input_size, inner_size, num_layers, + batch_first=True, dropout=dropout) + self.conv_branch = nn.Sequential( + nn.Conv1d(input_channels, inner_channels, + padding='same', + kernel_size=9), + nn.BatchNorm1d(inner_channels), + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels), + nn.Conv1d(inner_channels, inner_channels * 2, + padding='same', + kernel_size=5, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels * 2), + nn.Conv1d(inner_channels * 2, inner_channels, + padding='same', + kernel_size=3, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels), # n x c | n x c x l + nn.ReLU(), + ) + seq = next(iter(self.conv_branch.modules())) + idx = [0, 4, 8] + for i in idx: + torch.nn.init.kaiming_uniform_(seq[i].weight.data) + + def forward(self, x): + x_lstm, _ = self.lstm(x) # n x input_ch x inner_size + x_conv = self.conv_branch(x) # n x inner_ch x len + print(x_conv.size(), x_lstm.size()) + x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) + x = F.softmax(self.proj(x)) + return x + + +class MLSTM(BaseNeuralModel): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + # self.num_classes = params.get('num_classes', None) + # self.epochs = params.get('epochs', 100) + # self.batch_size = params.get('batch_size', 16) + # self.activation = params.get('activation', 'ReLU') + # self.learning_rate = 0.001 + + self.dropout = params.get('dropout', 0.25) + self.hidden_size = params.get('hidden_size', 64) + self.hidden_channels = params.get('hidden_channels', 32) + self.num_layers = params.get('num_layers', 2) + # self.target = None + # self.task_type = None + + def _init_model(self, ts: InputData): + _, input_channels, input_size = ts.features.shape + self.model = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + self.model_for_inference = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + optimizer = optim.Adam(self.model.parameters(), lr=0.001) + if ts.num_classes == 2: + loss_fn = CROSS_ENTROPY() + else: + loss_fn = MULTI_CLASS_CROSS_ENTROPY() + return loss_fn, optimizer + + @convert_to_3d_torch_array + def _fit_model(self, ts: InputData): + loss_fn, optimizer = self._init_model(ts) + train_loader, val_loader = self._prepare_data(ts, split_data=True) + self._train_loop( + train_loader=train_loader, + val_loader=val_loader, + loss_fn=loss_fn, + optimizer=optimizer + ) + From d118462f3c88c44f66959552ada7af205f13349d Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 5 Jul 2024 12:30:52 +0300 Subject: [PATCH 25/43] mlstm registered --- .../architecture/abstraction/decorators.py | 5 +- fedot_ind/core/models/early_tc/ecec.py | 62 ++++--- .../core/models/nn/network_impl/mlstm.py | 154 ++++++++++++++++++ .../data/default_operation_params.json | 4 + .../data/industrial_model_repository.json | 6 + fedot_ind/core/repository/model_repository.py | 5 +- 6 files changed, 211 insertions(+), 25 deletions(-) diff --git a/fedot_ind/core/architecture/abstraction/decorators.py b/fedot_ind/core/architecture/abstraction/decorators.py index 1e854be56..0b915c85c 100644 --- a/fedot_ind/core/architecture/abstraction/decorators.py +++ b/fedot_ind/core/architecture/abstraction/decorators.py @@ -42,13 +42,14 @@ def decorated_func(self, *args): def convert_to_3d_torch_array(func): def decorated_func(self, *args): - init_data = args[0] + init_data, *args = args data = DataConverter(data=init_data).convert_to_torch_format() if isinstance(init_data, InputData): init_data.features = data else: init_data = data - return func(self, init_data, *args[1:]) + return func(self, init_data, *args) + return decorated_func diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index a00df631d..45f7f9fe4 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -12,26 +12,53 @@ def __init__(self, params: Optional[OperationParameters] = None): def _init_model(self, X, y): super()._init_model(X, y) - self._confidences = np.ones((X.shape[0], self.n_pred)) + self._reliabilities = np.zeros((self.n_pred, self.n_classes, self.n_classes)) + + def _predict_one_slave(self, X, i, offset=0): + predicted_probas, predicted_labels = super()._predict_one_slave(X, i, offset) + reliabilities = self._reliabilities[i, predicted_labels, predicted_labels].flatten() # n_inst + return predicted_labels.astype(int), predicted_probas, reliabilities + + def _predict(self, X, training=False): + predicted_labels, predicted_probas, reliabilities = super()._predict(X, training) + reliabilities = np.stack(reliabilities) + confidences = 1 - np.cumprod(1 - reliabilities, axis=0) + non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] + return predicted_labels, predicted_probas, non_confident, confidences - def _score(self, X, y, alpha): + def predict(self, X): + predicted_labels, _, non_confident, confidences = self._predict(X) + predicted_labels = np.stack(predicted_labels) + predicted_labels[non_confident] = -1 + return predicted_labels, confidences + + def predict_proba(self, X): + _, predicted_probas, non_confident, confidences = self._predict(X) + predicted_probas = np.stack(predicted_probas) + predicted_probas[non_confident] = -1 + return predicted_probas, confidences + + def _score(self, X, y, alpha, training=False): y = y.astype(int) - predicted_labels = np.stack(super()._predict(X)[0]).astype(int) # n_pred x n_inst + predicted_labels, *_ = super()._predict(X, training) # n_pred x n_inst + predicted_labels = np.stack(predicted_labels) n = predicted_labels.shape[0] - accuracies = (predicted_labels == np.tile(y, (1, n))) # n_pred x n_inst + accuracies = (predicted_labels == np.tile(y, (n, 1))) # n_pred x n_inst confidences = np.ones((n, X.shape[0]), dtype='float32') for i in range(n): y_pred = predicted_labels[i] reliability_i = confusion_matrix(y, y_pred, normalize='pred') confidences[i] = 1 - reliability_i[y, y_pred] # n_inst + self._reliabilities[i] = reliability_i confidences = 1 - np.cumprod(confidences, axis=0) # n_pred x n_inst candidates = self._select_thrs(confidences) # n_candidates - cfs = np.zeros_like(candidates) + cfs = np.zeros((len(candidates), n)) for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst accuracy_for_candidate = (accuracies * mask).sum(1) / mask.sum(1) # n_pred cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) - return candidates[np.argmin(cfs)] + self._best_estimator_idx = np.argmin(cfs.mean(0)) + return candidates[np.argmin(cfs, axis=0)] # n_pred @staticmethod def _select_thrs(confidences): @@ -39,24 +66,15 @@ def _select_thrs(confidences): difference = np.diff(C) pair_means = C[:-1] + difference / 2 difference_shifted = np.roll(difference, 1) - difference_idx = np.argwhere(difference > difference_shifted) - return pair_means[difference_idx].flatten() + difference_idx = np.argwhere(difference <= difference_shifted) + means_candidates = pair_means[difference_idx].flatten() + return means_candidates if len(means_candidates) else C @staticmethod def cost_func(earliness, accuracies, alpha): - return alpha * accuracies + (1 - alpha) * earliness + return alpha * (1 - accuracies) + (1 - alpha) * earliness def fit(self, X, y): - self.confidence_threshold = super().fit(X, y) - - - - - - - - - - - - + super().fit(X, y) + self.confidence_thresholds = self._score(X, y, self.accuracy_importance, training=True) + \ No newline at end of file diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 31ca6fbd4..9a0a52ba6 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -125,3 +125,157 @@ def _fit_model(self, ts: InputData): optimizer=optimizer ) + +from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel +from typing import Optional, Callable, Any, List, Union +from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.data.data import InputData, OutputData +from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE +import torch.optim as optim +import torch.nn as nn +import torch.nn.functional as F +import torch +from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +import pandas as pd +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams +from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter +import torch.utils.data as data +from fedot_ind.core.architecture.settings.computational import default_device + +class SqueezeExciteBlock(nn.Module): + def __init__(self, input_channels, filters, reduce=4): + super().__init__() + self.filters = filters + self.pool = nn.AvgPool1d(input_channels) + self.bottleneck = max(self.filters // reduce, 4) + self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) + self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) + torch.nn.init.kaiming_normal_(self.fc1.weight.data) + torch.nn.init.kaiming_normal_(self.fc2.weight.data) + + def forward(self, x): + input_x = x + x = self.pool(x) + x = F.relu(self.fc1(x.view(-1, 1, self.filters))) + x = F.sigmoid(self.fc2(x)) + x = x.view(-1, self.filters, 1) * input_x + return x + +class MLSTM_module(nn.Module): + def __init__(self, input_size, input_channels, + inner_size, inner_channels, + output_size, num_layers, dropout=0.25): + super().__init__() + self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) + self.lstm = nn.LSTM(input_size, inner_size, num_layers, + batch_first=True, dropout=dropout) + self.conv_branch = nn.Sequential( + nn.Conv1d(input_channels, inner_channels, + padding='same', + kernel_size=9), + nn.BatchNorm1d(inner_channels), + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels), + nn.Conv1d(inner_channels, inner_channels * 2, + padding='same', + kernel_size=5, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l + nn.ReLU(), + SqueezeExciteBlock(input_size, inner_channels * 2), + nn.Conv1d(inner_channels * 2, inner_channels, + padding='same', + kernel_size=3, + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels), # n x c | n x c x l + nn.ReLU(), + ) + seq = next(iter(self.conv_branch.modules())) + idx = [0, 4, 8] + for i in idx: + torch.nn.init.kaiming_uniform_(seq[i].weight.data) + + def forward(self, x, hidden_state=None): + x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size + x_conv = self.conv_branch(x) # n x inner_ch x len + x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) + x = F.softmax(self.proj(x)) + return x#, hidden_state + + def augment_zero_padding(self, X: torch.Tensor): + res = [] + for i in self.prediction_idx: + zeroed_X = X[...] + zeroed_X[..., i + 1:] = 0 + res.append(zeroed_X) + res = torch.concat(res, 0) + return res[torch.randperm(res.size(0)), ...] + +class MLSTM(BaseNeuralModel): + def __init__(self, params: Optional[OperationParameters] = None): + if params is None: + params = {} + super().__init__() + # self.num_classes = params.get('num_classes', None) + # self.epochs = params.get('epochs', 100) + # self.batch_size = params.get('batch_size', 16) + # self.activation = params.get('activation', 'ReLU') + # self.learning_rate = 0.001 + + self.dropout = params.get('dropout', 0.25) + self.hidden_size = params.get('hidden_size', 64) + self.hidden_channels = params.get('hidden_channels', 32) + self.num_layers = params.get('num_layers', 2) + # self.target = None + # self.task_type = None + self.interval_percentage = params.get('interval_percentage', 10) + self.min_ts_length = params.get('min_ts_length', 5) + + def _compute_prediction_points(self, n_idx): + interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) + prediction_idx = np.arange(0, n_idx, interval_length) + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + return prediction_idx + + def _init_model(self, ts: InputData): + _, input_channels, input_size = ts.features.shape + self.prediction_idx = self._compute_prediction_points(input_size) + self.model = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + self.model_for_inference = MLSTM_module(input_size, input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) + optimizer = optim.Adam(self.model.parameters(), lr=0.001) + if ts.num_classes == 2: + loss_fn = CROSS_ENTROPY() + else: + loss_fn = MULTI_CLASS_CROSS_ENTROPY() + return loss_fn, optimizer + + def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): + return super()._train_loop(train_loader, val_loader, loss_fn, optimizer) + + @convert_to_3d_torch_array + def _fit_model(self, ts: InputData): + if isinstance(ts, torch.Tensor): + ts = self.augment_zero_padding(ts) + else: + print(type(ts)) + loss_fn, optimizer = self._init_model(ts) + train_loader, val_loader = self._prepare_data(ts, split_data=True) + self._train_loop( + train_loader=train_loader, + val_loader=val_loader, + loss_fn=loss_fn, + optimizer=optimizer + ) + + + + diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 11980e4b4..98a7e2986 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -166,6 +166,10 @@ "learning_rate": "constant", "solver": "adam" }, + "mlstm_model": { + "epochs": 100, + "batch_size": 16 + }, "ar": { "lag_1": 7, "lag_2": 12, diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index ccbd1e0a7..42f58446a 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -370,6 +370,12 @@ "automl" ] }, + "mlstm_model": { + "meta": "fedot_NN_classification", + "presets": ["ts"], + "tags": [], + "input_type": "[DataTypesEnum.table]" + }, "xcm_model": { "meta": "fedot_NN_classification", "presets": [ diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index ac23c4f22..c7283a1bc 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -55,6 +55,7 @@ from fedot_ind.core.models.nn.network_impl.inception import InceptionTimeModel from fedot_ind.core.models.nn.network_impl.lora_nn import LoraModel from fedot_ind.core.models.nn.network_impl.mini_rocket import MiniRocketExtractor +from fedot_ind.core.models.nn.network_impl.mlstm import MLSTM from fedot_ind.core.models.nn.network_impl.nbeats import NBeatsModel from fedot_ind.core.models.nn.network_impl.resnet import ResNetModel from fedot_ind.core.models.nn.network_impl.tst import TSTModel @@ -193,7 +194,9 @@ class AtomizedModel(Enum): # linear_dummy_model 'dummy': DummyOverComplicatedNeuralNetwork, # linear_dummy_model - 'lora_model': LoraModel + 'lora_model': LoraModel, + # early ts classification + 'mlstm_model': MLSTM } From 7a2c477720a098a4668be1f61c368aa42818193d Mon Sep 17 00:00:00 2001 From: leostre Date: Tue, 9 Jul 2024 14:33:24 +0300 Subject: [PATCH 26/43] fitting w augmentation --- .../architecture/abstraction/decorators.py | 9 +- .../models/nn/network_impl/base_nn_model.py | 6 +- .../core/models/nn/network_impl/mlstm.py | 201 ++++-------------- 3 files changed, 55 insertions(+), 161 deletions(-) diff --git a/fedot_ind/core/architecture/abstraction/decorators.py b/fedot_ind/core/architecture/abstraction/decorators.py index 0b915c85c..3c01bb56d 100644 --- a/fedot_ind/core/architecture/abstraction/decorators.py +++ b/fedot_ind/core/architecture/abstraction/decorators.py @@ -11,9 +11,10 @@ def fedot_data_type(func): def decorated_func(self, *args): - if not isinstance(args[0], InputData): - args[0] = DataConverter(data=args[0]) - features = args[0].features + data, *rest_args = args + if not isinstance(data, InputData): + data = DataConverter(data=data) + features = data.features if len(features.shape) < 4: try: @@ -22,7 +23,7 @@ def decorated_func(self, *args): input_data_squeezed = np.squeeze(features) else: input_data_squeezed = features - return func(self, input_data_squeezed, args[1]) + return func(self, input_data_squeezed, *rest_args) return decorated_func diff --git a/fedot_ind/core/models/nn/network_impl/base_nn_model.py b/fedot_ind/core/models/nn/network_impl/base_nn_model.py index de83a2f06..e9d6c7274 100644 --- a/fedot_ind/core/models/nn/network_impl/base_nn_model.py +++ b/fedot_ind/core/models/nn/network_impl/base_nn_model.py @@ -90,7 +90,7 @@ def _fit_model(self, ts: InputData, split_data: bool = True): def _init_model(self, ts) -> tuple: raise NotImplementedError() - def _prepare_data(self, ts, split_data: bool = True): + def _prepare_data(self, ts, split_data: bool = True, collate_fn=None): if split_data: train_data, val_data = train_test_data_setup( @@ -102,13 +102,13 @@ def _prepare_data(self, ts, split_data: bool = True): val_dataset = None train_loader = torch.utils.data.DataLoader( - train_dataset, batch_size=self.batch_size, shuffle=True) + train_dataset, batch_size=self.batch_size, shuffle=True, collate_fn=collate_fn) if val_dataset is None: val_loader = val_dataset else: val_loader = torch.utils.data.DataLoader( - val_dataset, batch_size=self.batch_size, shuffle=True) + val_dataset, batch_size=self.batch_size, shuffle=True, collate_fn=collate_fn) self.label_encoder = train_dataset.label_encoder return train_loader, val_loader diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 9a0a52ba6..612d2b11b 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -4,11 +4,13 @@ from fedot.core.data.data import InputData, OutputData from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE import torch.optim as optim +from torch.optim import lr_scheduler import torch.nn as nn import torch.nn.functional as F import torch +from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array, fedot_data_type import pandas as pd from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping @@ -70,149 +72,19 @@ def __init__(self, input_size, input_channels, for i in idx: torch.nn.init.kaiming_uniform_(seq[i].weight.data) - def forward(self, x): - x_lstm, _ = self.lstm(x) # n x input_ch x inner_size - x_conv = self.conv_branch(x) # n x inner_ch x len - print(x_conv.size(), x_lstm.size()) - x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) - x = F.softmax(self.proj(x)) - return x - - -class MLSTM(BaseNeuralModel): - def __init__(self, params: Optional[OperationParameters] = None): - if params is None: - params = {} - super().__init__() - # self.num_classes = params.get('num_classes', None) - # self.epochs = params.get('epochs', 100) - # self.batch_size = params.get('batch_size', 16) - # self.activation = params.get('activation', 'ReLU') - # self.learning_rate = 0.001 - - self.dropout = params.get('dropout', 0.25) - self.hidden_size = params.get('hidden_size', 64) - self.hidden_channels = params.get('hidden_channels', 32) - self.num_layers = params.get('num_layers', 2) - # self.target = None - # self.task_type = None - - def _init_model(self, ts: InputData): - _, input_channels, input_size = ts.features.shape - self.model = MLSTM_module(input_size, input_channels, - self.hidden_size, self.hidden_channels, - self.num_classes, self.num_layers, - self.dropout) - self.model_for_inference = MLSTM_module(input_size, input_channels, - self.hidden_size, self.hidden_channels, - self.num_classes, self.num_layers, - self.dropout) - optimizer = optim.Adam(self.model.parameters(), lr=0.001) - if ts.num_classes == 2: - loss_fn = CROSS_ENTROPY() - else: - loss_fn = MULTI_CLASS_CROSS_ENTROPY() - return loss_fn, optimizer - - @convert_to_3d_torch_array - def _fit_model(self, ts: InputData): - loss_fn, optimizer = self._init_model(ts) - train_loader, val_loader = self._prepare_data(ts, split_data=True) - self._train_loop( - train_loader=train_loader, - val_loader=val_loader, - loss_fn=loss_fn, - optimizer=optimizer - ) - - -from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel -from typing import Optional, Callable, Any, List, Union -from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.data.data import InputData, OutputData -from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE -import torch.optim as optim -import torch.nn as nn -import torch.nn.functional as F -import torch -from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array -import pandas as pd -from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams -from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter -import torch.utils.data as data -from fedot_ind.core.architecture.settings.computational import default_device - -class SqueezeExciteBlock(nn.Module): - def __init__(self, input_channels, filters, reduce=4): - super().__init__() - self.filters = filters - self.pool = nn.AvgPool1d(input_channels) - self.bottleneck = max(self.filters // reduce, 4) - self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) - self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) - torch.nn.init.kaiming_normal_(self.fc1.weight.data) - torch.nn.init.kaiming_normal_(self.fc2.weight.data) - - def forward(self, x): - input_x = x - x = self.pool(x) - x = F.relu(self.fc1(x.view(-1, 1, self.filters))) - x = F.sigmoid(self.fc2(x)) - x = x.view(-1, self.filters, 1) * input_x - return x - -class MLSTM_module(nn.Module): - def __init__(self, input_size, input_channels, - inner_size, inner_channels, - output_size, num_layers, dropout=0.25): - super().__init__() - self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) - self.lstm = nn.LSTM(input_size, inner_size, num_layers, - batch_first=True, dropout=dropout) - self.conv_branch = nn.Sequential( - nn.Conv1d(input_channels, inner_channels, - padding='same', - kernel_size=9), - nn.BatchNorm1d(inner_channels), - nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels), - nn.Conv1d(inner_channels, inner_channels * 2, - padding='same', - kernel_size=5, - ), # c x l | n x c x l - nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l - nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels * 2), - nn.Conv1d(inner_channels * 2, inner_channels, - padding='same', - kernel_size=3, - ), # c x l | n x c x l - nn.BatchNorm1d(inner_channels), # n x c | n x c x l - nn.ReLU(), - ) - seq = next(iter(self.conv_branch.modules())) - idx = [0, 4, 8] - for i in idx: - torch.nn.init.kaiming_uniform_(seq[i].weight.data) - - def forward(self, x, hidden_state=None): + def forward(self, x, hidden_state=None, return_hidden_state=False): + # hidden_state = hidden_state or self.hidden_state + if not self.training: + print(x.shape) x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size x_conv = self.conv_branch(x) # n x inner_ch x len x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) x = F.softmax(self.proj(x)) + # self.hidden_state = hidden_state + if return_hidden_state: + return x, hidden_state return x#, hidden_state - def augment_zero_padding(self, X: torch.Tensor): - res = [] - for i in self.prediction_idx: - zeroed_X = X[...] - zeroed_X[..., i + 1:] = 0 - res.append(zeroed_X) - res = torch.concat(res, 0) - return res[torch.randperm(res.size(0)), ...] class MLSTM(BaseNeuralModel): def __init__(self, params: Optional[OperationParameters] = None): @@ -229,11 +101,19 @@ def __init__(self, params: Optional[OperationParameters] = None): self.hidden_size = params.get('hidden_size', 64) self.hidden_channels = params.get('hidden_channels', 32) self.num_layers = params.get('num_layers', 2) - # self.target = None - # self.task_type = None self.interval_percentage = params.get('interval_percentage', 10) self.min_ts_length = params.get('min_ts_length', 5) + def __repr__(self): + return 'MLSTM' + + @convert_to_3d_torch_array + def _predict_model(self, ts: InputData, output_mode='default'): + self.model.eval() + x_test = torch.Tensor(ts).to(self._device) + pred = self.model(x_test) + return self._convert_predict(pred, output_mode) + def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) prediction_idx = np.arange(0, n_idx, interval_length) @@ -241,7 +121,7 @@ def _compute_prediction_points(self, n_idx): return prediction_idx def _init_model(self, ts: InputData): - _, input_channels, input_size = ts.features.shape + *_, input_channels, input_size = ts.features.shape self.prediction_idx = self._compute_prediction_points(input_size) self.model = MLSTM_module(input_size, input_channels, self.hidden_size, self.hidden_channels, @@ -255,27 +135,40 @@ def _init_model(self, ts: InputData): if ts.num_classes == 2: loss_fn = CROSS_ENTROPY() else: - loss_fn = MULTI_CLASS_CROSS_ENTROPY() + loss_fn = CROSS_ENTROPY() return loss_fn, optimizer - def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): - return super()._train_loop(train_loader, val_loader, loss_fn, optimizer) + # @convert_to_3d_torch_array + # def predict(self, ts: InputData, output_mode: str = 'default'): + # return super().predict(ts, output_mode) + + # def predict_for_fit(self, ts: InputData, output_mode: str = 'default'): + # return super().predict_for_fit(ts, output_mode) @convert_to_3d_torch_array - def _fit_model(self, ts: InputData): - if isinstance(ts, torch.Tensor): - ts = self.augment_zero_padding(ts) - else: - print(type(ts)) + def _fit_model(self, ts: InputData, mode='zero_padding'): + self.epochs = 1 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 loss_fn, optimizer = self._init_model(ts) - train_loader, val_loader = self._prepare_data(ts, split_data=True) + + train_loader, val_loader = self._prepare_data(ts, split_data=False, + collate_fn=getattr(self, '_augment_zero_padding')) self._train_loop( train_loader=train_loader, val_loader=val_loader, loss_fn=loss_fn, - optimizer=optimizer + optimizer=optimizer, ) - - - + def _augment_zero_padding(self, batch,): + prediction_idx = self.prediction_idx + x, y = zip(*batch) + X, y = torch.stack(x), torch.stack(y) + y = np.tile(y, (len(prediction_idx), 1)) + res = [] + for i in prediction_idx: + zeroed_X = X[...] + zeroed_X[..., i + 1:] = 0 + res.append(zeroed_X) + res = np.concatenate(res, 0) + perm = np.random.permutation(res.shape[0]) + return torch.tensor(res[perm, ...]), torch.tensor(y[perm]) From 370eb28e1b5b77274d4c2039c07ca7f5f96b0356 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 01:27:24 +0300 Subject: [PATCH 27/43] all work, but need eval --- .../core/models/early_tc/base_early_tc.py | 3 + fedot_ind/core/models/early_tc/ecec.py | 61 +++++++++++++------ fedot_ind/core/models/early_tc/economy_k.py | 21 +++++-- .../core/models/early_tc/prob_threshold.py | 29 ++++++--- fedot_ind/core/models/early_tc/teaser.py | 19 ++++-- 5 files changed, 93 insertions(+), 40 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 5e180c2a9..38e5fd23f 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -71,6 +71,9 @@ def _select_estimators(self, X, training=False): elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) + elif 'last_available': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = [last_idx] else: raise ValueError('Unknown prediction mode') return estimator_indices, offset diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 45f7f9fe4..00962a233 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -9,6 +9,7 @@ class ECEC(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): super().__init__(params) + self.__cv = 5 def _init_model(self, X, y): super()._init_model(X, y) @@ -30,34 +31,39 @@ def predict(self, X): predicted_labels, _, non_confident, confidences = self._predict(X) predicted_labels = np.stack(predicted_labels) predicted_labels[non_confident] = -1 - return predicted_labels, confidences + if self.transform_score: + confidences = self._transform_score(confidences) + return self._remove_first_1d(predicted_labels, confidences) def predict_proba(self, X): _, predicted_probas, non_confident, confidences = self._predict(X) predicted_probas = np.stack(predicted_probas) predicted_probas[non_confident] = -1 - return predicted_probas, confidences + if self.transform_score: + confidences = self._transform_score(confidences) + return self._remove_first_1d(predicted_probas, confidences) + + def _fit_one_interval(self, X, y, i): + X_part = X[..., :self.prediction_idx[i] + 1] + X_part = self.scalers[i].fit_transform(X_part) + self.slave_estimators[i].fit(X_part, y) + labels = cross_val_predict(self.slave_estimators[i], X_part, y, cv=self.__cv) + return labels - def _score(self, X, y, alpha, training=False): - y = y.astype(int) - predicted_labels, *_ = super()._predict(X, training) # n_pred x n_inst - predicted_labels = np.stack(predicted_labels) - n = predicted_labels.shape[0] - accuracies = (predicted_labels == np.tile(y, (n, 1))) # n_pred x n_inst - confidences = np.ones((n, X.shape[0]), dtype='float32') - for i in range(n): - y_pred = predicted_labels[i] - reliability_i = confusion_matrix(y, y_pred, normalize='pred') - confidences[i] = 1 - reliability_i[y, y_pred] # n_inst - self._reliabilities[i] = reliability_i - confidences = 1 - np.cumprod(confidences, axis=0) # n_pred x n_inst + def _score(self, y, y_pred, alpha): + matches = (y_pred == np.tile(y, (self.n_pred, 1))) # n_pred x n_inst + n, n_inst = matches.shape[:2] + confidences = np.ones((n, n_inst), dtype='float32') + for i in range(self.n_pred): + confidences[i] = self._reliabilities[i, y, y_pred[i]] + confidences = 1 - np.cumprod(1 - confidences, axis=0) # n_pred x n_inst candidates = self._select_thrs(confidences) # n_candidates cfs = np.zeros((len(candidates), n)) for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst - accuracy_for_candidate = (accuracies * mask).sum(1) / mask.sum(1) # n_pred + accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) - self._best_estimator_idx = np.argmin(cfs.mean(0)) + self._chosen_estimator_idx = np.argmin(cfs.mean(0)) return candidates[np.argmin(cfs, axis=0)] # n_pred @staticmethod @@ -75,6 +81,23 @@ def cost_func(earliness, accuracies, alpha): return alpha * (1 - accuracies) + (1 - alpha) * earliness def fit(self, X, y): - super().fit(X, y) - self.confidence_thresholds = self._score(X, y, self.accuracy_importance, training=True) + y = np.array(y).flatten().astype(int) + self._init_model(X, y) + labels = [] + for i in range(self.n_pred): + labels.append(self._fit_one_interval(X, y, i)) + predicted_labels = np.stack(labels) + for i in range(self.n_pred): + y_pred = predicted_labels[i] + reliability_i = confusion_matrix(y, y_pred, normalize='pred') + self._reliabilities[i] = reliability_i + self.confidence_thresholds = self._score(y, predicted_labels, self.accuracy_importance) + + def _transform_score(self, confidences): + thr = self.confidence_thresholds[self._estimator_for_predict[-1]] + confidences = confidences - thr + positive = confidences > 0 + confidences[positive] *= 1 / (1 - thr) + confidences[~positive] *= 1 / thr + return confidences \ No newline at end of file diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 639e680cd..8eb207550 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -11,8 +11,10 @@ def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} super().__init__(params) + self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) self._cluster_factor = params.get('cluster_factor' , 1) + # self.confidence_mode = params.get('confidence_mode', 'time') # or 'confidence' self._random_state = 2104 self.__cv = 5 @@ -77,13 +79,20 @@ def _get_prediction_time(self, X, cluster_centroids, i): def predict_proba(self, X): probas, times, is_optimal = self._predict(X) is_optimal = np.stack(is_optimal) - idx = np.tile(np.arange(self.n_pred), (is_optimal.shape[1], 1)).T # n_pred x n_inst - idx[~is_optimal] = self.n_pred - idx = np.argmin(idx, 0) - probas = np.stack(probas) - return probas[idx], np.stack(times)[idx] + probas, times = np.stack(probas), np.stack(times) + if self.transform_score: + times = self._transform_score(times) + return self._remove_first_1d(probas, times) def predict(self, X): probas, times = self.predict_proba(X) labels = probas.argmax(-1) - return labels, times + return self._remove_first_1d(labels, times) + + def _transform_score(self, time): + idx = self._estimator_for_predict[-1] + scores = -(1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) + scores[scores == 0] = 1 # no posibility for lininterp when sure + return scores + + diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 51d169909..8dcb8828f 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -7,7 +7,7 @@ class ProbabilityThresholdClassifier(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} - super().__init__() + super().__init__(params) self.probability_threshold = params.get('probability_threshold', None) def _init_model(self, X, y): @@ -18,19 +18,23 @@ def _init_model(self, X, y): def predict_proba(self, X): _, predicted_probas, non_acceptance = self._predict(X) predicted_probas[non_acceptance] = 0 - return predicted_probas.squeeze() + scores = predicted_probas.max(-1) + if self.transform_score: + scores = self._transform_score(scores) + return self._remove_first_1d(predicted_probas, scores) def predict(self, X): - predicted_labels, _, non_acceptance = self._predict(X, training=False) + predicted_labels, predicted_probas, non_acceptance = self._predict(X, training=False) predicted_labels[non_acceptance] = -1 - # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] - return predicted_labels # prediction_points x n_instances + scores = predicted_probas.max(-1) + if self.transform_score: + scores = self._transform_score(scores) + return self._remove_first_1d(predicted_labels, scores) # (prediction_points x) n_instances def _predict(self, X, training=True): predicted_probas, predicted_labels = super()._predict(X, training) predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) - # print(predicted_labels.shape, predicted_probas.shape) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions double_check = predicted_probas.max(axis=-1) > self.probability_threshold non_acceptance[non_acceptance & double_check] = False @@ -38,10 +42,17 @@ def _predict(self, X, training=True): def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) - self._best_estimator_idx = np.argmax(scores) + self._chosen_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) - return self._score(X, y, self.accuracy_importance) - + self._score(X, y, self.accuracy_importance) + + def _transform_score(self, confidences): + thr = self.probability_threshold + confidences = confidences - thr + positive = confidences > 0 + confidences[positive] *= 1 / (1 - thr) + confidences[~positive] *= 1 / thr + return confidences diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index c03c5d1e6..10d36e5a0 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -54,31 +54,38 @@ def _predict(self, X): # for each point of estimation for i in range(predicted_labels.shape[0]): # find not accepted points - ith_point_to_oc = to_oc_check[to_oc_check[:, 0] == i, 1] - X_to_ith = X_ocs[i][ith_point_to_oc] + X_to_ith = X_ocs[i] # if they are not outliers final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject # mark as accepted - non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False + # non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False final_verdicts[i] = final_verdict + non_acceptance[non_acceptance & (final_verdict > 0)] = False return predicted_labels, predicted_probas, non_acceptance, final_verdicts def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] - return predicted_probas.squeeze() + if self.transform_score: + final_verdicts = self._transform_score(final_verdicts) + return self._remove_first_1d(predicted_probas, final_verdicts) def predict(self, X): predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) predicted_labels[non_acceptance] = -1 # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] - return predicted_labels # prediction_points x n_instances + if self.transform_score: + final_verdicts = self._transform_score(final_verdicts) + return self._remove_first_1d(predicted_labels, final_verdicts) # (prediction_points x) n_instances def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) - self._best_estimator_idx = np.argmax(scores) + self._chosen_estimator_idx = np.argmax(scores) return scores def fit(self, X, y): super().fit(X, y) return self._score(X, y, self.accuracy_importance) + + def _transform_score(self, scores): + return np.tanh(scores) From e51949b65ee4a047b6536e7eb15861faab6297e5 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 12:09:39 +0300 Subject: [PATCH 28/43] evth converged to one interface + refactored --- .../core/models/early_tc/base_early_tc.py | 53 ++-- fedot_ind/core/models/early_tc/ecec.py | 18 +- fedot_ind/core/models/early_tc/economy_k.py | 15 +- .../core/models/early_tc/prob_threshold.py | 14 +- fedot_ind/core/models/early_tc/teaser.py | 22 +- .../core/models/nn/network_impl/mlstm.py | 235 ++++++++++++------ .../data/default_operation_params.json | 14 +- .../data/industrial_model_repository.json | 22 +- fedot_ind/core/repository/model_repository.py | 8 +- fedot_ind/core/tuning/search_space.py | 32 ++- 10 files changed, 280 insertions(+), 153 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 38e5fd23f..9fbc162c5 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -11,10 +11,12 @@ def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} super().__init__() - self.prediction_mode = params.get('prediction_mode', 'best_by_harmonic_mean') self.interval_percentage = params.get('interval_percentage', 10) - self.consecutive_predictions = params.get('consecutive_predictions', 3) + self.consecutive_predictions = params.get('consecutive_predictions', 1) self.accuracy_importance = params.get('accuracy_importance', 1.) + + self.prediction_mode = params.get('prediction_mode', 'last_available') + self.transform_score = params.get('transform_score', True) self.min_ts_length = params.get('min_ts_step', 3) self.random_state = params.get('random_state', None) self.weasel_params = {} @@ -26,14 +28,15 @@ def _init_model(self, X, y): self.n_pred = len(self.prediction_idx) self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] self.scalers = [StandardScaler() for _ in range(self.n_pred)] - self._best_estimator_idx = -1 + self._chosen_estimator_idx = -1 self.classes_ = [np.unique(y)] + self._estimator_for_predict = [-1] @property def required_length(self): - if not hasattr(self, '_best_estimator_idx'): + if not hasattr(self, '_chosen_estimator_idx'): return None - return self.prediction_idx[self._best_estimator_idx] + return self.prediction_idx[self._chosen_estimator_idx] @property def n_classes(self): @@ -47,7 +50,7 @@ def fit(self, X, y=None): self._fit_one_interval(X, y, i) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i] + 1] # what's dimensionality of input? will it work in case of multivariate? + X_part = X[..., :self.prediction_idx[i] + 1] X_part = self.scalers[i].fit_transform(X_part) probas = self.slave_estimators[i].fit_predict_proba(X_part, y) return probas @@ -67,19 +70,21 @@ def _compute_prediction_points(self, n_idx): def _select_estimators(self, X, training=False): offset = 0 if not training and self.prediction_mode == 'best_by_harmonic_mean': - estimator_indices = [self._best_estimator_idx] + estimator_indices = [self._chosen_estimator_idx] + elif not training and self.prediction_mode == 'last_available': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = [last_idx] elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) - elif 'last_available': - last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) - estimator_indices = [last_idx] else: raise ValueError('Unknown prediction mode') return estimator_indices, offset def _predict(self, X, training=True): estimator_indices, offset = self._select_estimators(X, training) + if not training: + self._estimator_for_predict = estimator_indices prediction = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) @@ -94,20 +99,32 @@ def _consecutive_count(self, predicted_labels: List[np.array]): consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 return consecutive_labels # prediction_points x n_instances - def predict_proba(self, X): - raise NotImplementedError + def predict_proba(self, *args): + predicted_probas, scores, *_ = args + if self.transform_score: + scores = self._transform_score(scores) + scores = np.tile(scores[..., None], (1, 1, self.n_classes)) + prediction = np.stack([predicted_probas, scores], axis=0) + if prediction.shape[1] == 1: + prediction = prediction.squeeze(1) + return prediction def predict(self, X): - raise NotImplementedError + prediction = self.predict_proba(X) + labels = prediction[0:1].argmax(-1) + scores = prediction[1:2, ..., 0] + prediction = np.stack([labels, scores], 0) + if prediction.shape[1] == 1: + prediction = prediction.squeeze(1) + return prediction - def _score(self, X, y, hm_shift_to_acc=None): + def _score(self, X, y, accuracy_importance=None, training=True): y = np.array(y).flatten() - hm_shift_to_acc = hm_shift_to_acc or self.accuracy_importance - predictions = self._predict(X)[0] + accuracy_importance = accuracy_importance or self.accuracy_importance + predictions = self._predict(X, training)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + hm_shift_to_acc) * accuracies * self.earliness[:prediction_points] / (hm_shift_to_acc * accuracies + self.earliness[:prediction_points]) - + return (1 + accuracy_importance) * accuracies * self.earliness[:prediction_points] / (accuracy_importance * accuracies + self.earliness[:prediction_points]) def _get_applicable_index(self, last_available_idx): idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 00962a233..f6e163d25 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -3,7 +3,6 @@ from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.models.early_tc.base_early_tc import BaseETC from sklearn.model_selection import cross_val_predict -from sklearn.base import clone from sklearn.metrics import confusion_matrix class ECEC(BaseETC): @@ -25,23 +24,14 @@ def _predict(self, X, training=False): reliabilities = np.stack(reliabilities) confidences = 1 - np.cumprod(1 - reliabilities, axis=0) non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] - return predicted_labels, predicted_probas, non_confident, confidences - - def predict(self, X): - predicted_labels, _, non_confident, confidences = self._predict(X) predicted_labels = np.stack(predicted_labels) - predicted_labels[non_confident] = -1 - if self.transform_score: - confidences = self._transform_score(confidences) - return self._remove_first_1d(predicted_labels, confidences) + predicted_probas = np.stack(predicted_probas) + return predicted_labels, predicted_probas, non_confident, confidences def predict_proba(self, X): _, predicted_probas, non_confident, confidences = self._predict(X) - predicted_probas = np.stack(predicted_probas) predicted_probas[non_confident] = -1 - if self.transform_score: - confidences = self._transform_score(confidences) - return self._remove_first_1d(predicted_probas, confidences) + return super().predict_proba(predicted_probas, confidences) def _fit_one_interval(self, X, y, i): X_part = X[..., :self.prediction_idx[i] + 1] @@ -52,7 +42,7 @@ def _fit_one_interval(self, X, y, i): def _score(self, y, y_pred, alpha): matches = (y_pred == np.tile(y, (self.n_pred, 1))) # n_pred x n_inst - n, n_inst = matches.shape[:2] + n, n_inst, *_ = matches.shape confidences = np.ones((n, n_inst), dtype='float32') for i in range(self.n_pred): confidences[i] = self._reliabilities[i, y, y_pred[i]] diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 8eb207550..bf09acd4b 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -4,7 +4,7 @@ from fedot_ind.core.models.early_tc.base_early_tc import BaseETC from sklearn.cluster import KMeans from sklearn.metrics import confusion_matrix -from sklearn.model_selection import train_test_split, cross_val_predict +from sklearn.model_selection import cross_val_predict class EconomyK(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): @@ -14,7 +14,6 @@ def __init__(self, params: Optional[OperationParameters] = None): self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) self._cluster_factor = params.get('cluster_factor' , 1) - # self.confidence_mode = params.get('confidence_mode', 'time') # or 'confidence' self._random_state = 2104 self.__cv = 5 @@ -77,17 +76,9 @@ def _get_prediction_time(self, X, cluster_centroids, i): return time_optimal, is_optimal # n_inst def predict_proba(self, X): - probas, times, is_optimal = self._predict(X) - is_optimal = np.stack(is_optimal) + probas, times, _ = self._predict(X, training=False) probas, times = np.stack(probas), np.stack(times) - if self.transform_score: - times = self._transform_score(times) - return self._remove_first_1d(probas, times) - - def predict(self, X): - probas, times = self.predict_proba(X) - labels = probas.argmax(-1) - return self._remove_first_1d(labels, times) + return super().predict_proba(probas, times) def _transform_score(self, time): idx = self._estimator_for_predict[-1] diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 8dcb8828f..773f79d8e 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -16,20 +16,10 @@ def _init_model(self, X, y): self.probability_threshold = 1 / len(self.classes_[0]) def predict_proba(self, X): - _, predicted_probas, non_acceptance = self._predict(X) + _, predicted_probas, non_acceptance = self._predict(X, training=False) predicted_probas[non_acceptance] = 0 scores = predicted_probas.max(-1) - if self.transform_score: - scores = self._transform_score(scores) - return self._remove_first_1d(predicted_probas, scores) - - def predict(self, X): - predicted_labels, predicted_probas, non_acceptance = self._predict(X, training=False) - predicted_labels[non_acceptance] = -1 - scores = predicted_probas.max(-1) - if self.transform_score: - scores = self._transform_score(scores) - return self._remove_first_1d(predicted_labels, scores) # (prediction_points x) n_instances + return super().predict_proba(predicted_probas, scores) def _predict(self, X, training=True): predicted_probas, predicted_labels = super()._predict(X, training) diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 10d36e5a0..23d6c078d 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,9 +1,9 @@ from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np -from sklearn.svm import OneClassSVM -from sklearn.model_selection import GridSearchCV from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from sklearn.model_selection import GridSearchCV +from sklearn.svm import OneClassSVM class TEASER(BaseETC): @@ -40,13 +40,12 @@ def _form_X_oc(self, predicted_probas): d = d.min(axis=-1).reshape(-1, 1) return np.hstack([predicted_probas, d]) - def _predict(self, X): + def _predict(self, X, training=False): estimator_indices, offset = self._select_estimators(X) X_ocs, predicted_probas, predicted_labels = zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary ) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions - to_oc_check = np.argwhere(non_acceptance) X_ocs = np.stack(X_ocs) predicted_probas = np.stack(predicted_probas) predicted_labels = np.stack(predicted_labels) @@ -56,9 +55,8 @@ def _predict(self, X): # find not accepted points X_to_ith = X_ocs[i] # if they are not outliers - final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # 1 for accept -1 for reject + final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # mark as accepted - # non_acceptance[i, np.argwhere(final_verdict >= 0).flatten()] = False final_verdicts[i] = final_verdict non_acceptance[non_acceptance & (final_verdict > 0)] = False return predicted_labels, predicted_probas, non_acceptance, final_verdicts @@ -66,17 +64,7 @@ def _predict(self, X): def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] - if self.transform_score: - final_verdicts = self._transform_score(final_verdicts) - return self._remove_first_1d(predicted_probas, final_verdicts) - - def predict(self, X): - predicted_labels, _, non_acceptance, final_verdicts = self._predict(X) - predicted_labels[non_acceptance] = -1 - # predicted_labels[non_acceptance] = final_verdicts[non_acceptance] - if self.transform_score: - final_verdicts = self._transform_score(final_verdicts) - return self._remove_first_1d(predicted_labels, final_verdicts) # (prediction_points x) n_instances + return super().predict_proba(predicted_probas, final_verdicts) def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 612d2b11b..3e1d3c4b5 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -1,23 +1,20 @@ +import copy from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel from typing import Optional, Callable, Any, List, Union from fedot.core.operations.operation_parameters import OperationParameters from fedot.core.data.data import InputData, OutputData from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE import torch.optim as optim -from torch.optim import lr_scheduler +import torch.optim.lr_scheduler as lr_scheduler import torch.nn as nn import torch.nn.functional as F import torch -from tqdm import tqdm +from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array, fedot_data_type +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array import pandas as pd -from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter import torch.utils.data as data -from fedot_ind.core.architecture.settings.computational import default_device class SqueezeExciteBlock(nn.Module): def __init__(self, input_channels, filters, reduce=4): @@ -46,20 +43,22 @@ def __init__(self, input_size, input_channels, self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) self.lstm = nn.LSTM(input_size, inner_size, num_layers, batch_first=True, dropout=dropout) + + squeeze_excite_size = input_size #if not interval else interval self.conv_branch = nn.Sequential( nn.Conv1d(input_channels, inner_channels, padding='same', kernel_size=9), nn.BatchNorm1d(inner_channels), nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels), + SqueezeExciteBlock(squeeze_excite_size, inner_channels), nn.Conv1d(inner_channels, inner_channels * 2, padding='same', kernel_size=5, ), # c x l | n x c x l nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l nn.ReLU(), - SqueezeExciteBlock(input_size, inner_channels * 2), + SqueezeExciteBlock(squeeze_excite_size, inner_channels * 2), nn.Conv1d(inner_channels * 2, inner_channels, padding='same', kernel_size=3, @@ -72,103 +71,197 @@ def __init__(self, input_size, input_channels, for i in idx: torch.nn.init.kaiming_uniform_(seq[i].weight.data) - def forward(self, x, hidden_state=None, return_hidden_state=False): - # hidden_state = hidden_state or self.hidden_state - if not self.training: - print(x.shape) + def forward(self, x, hidden_state=None, return_hidden=False): x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size x_conv = self.conv_branch(x) # n x inner_ch x len x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) x = F.softmax(self.proj(x)) - # self.hidden_state = hidden_state - if return_hidden_state: + if return_hidden: return x, hidden_state - return x#, hidden_state - + return x + class MLSTM(BaseNeuralModel): def __init__(self, params: Optional[OperationParameters] = None): if params is None: params = {} super().__init__() - # self.num_classes = params.get('num_classes', None) - # self.epochs = params.get('epochs', 100) - # self.batch_size = params.get('batch_size', 16) - # self.activation = params.get('activation', 'ReLU') - # self.learning_rate = 0.001 - self.dropout = params.get('dropout', 0.25) self.hidden_size = params.get('hidden_size', 64) self.hidden_channels = params.get('hidden_channels', 32) self.num_layers = params.get('num_layers', 2) self.interval_percentage = params.get('interval_percentage', 10) self.min_ts_length = params.get('min_ts_length', 5) - + self.fitting_mode = params.get('fitting_mode', 'zero_padding') + self.proba_thr = params.get('proba_thr', None) + def __repr__(self): return 'MLSTM' - @convert_to_3d_torch_array - def _predict_model(self, ts: InputData, output_mode='default'): - self.model.eval() - x_test = torch.Tensor(ts).to(self._device) - pred = self.model(x_test) - return self._convert_predict(pred, output_mode) - def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) - prediction_idx = np.arange(0, n_idx, interval_length) + prediction_idx = np.arange(interval_length - 1, n_idx, interval_length) self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 - return prediction_idx + return prediction_idx, interval_length def _init_model(self, ts: InputData): - *_, input_channels, input_size = ts.features.shape - self.prediction_idx = self._compute_prediction_points(input_size) - self.model = MLSTM_module(input_size, input_channels, + _, input_channels, input_size = ts.features.shape + self.input_size = input_size + self.prediction_idx, self.interval = self._compute_prediction_points(input_size) + self.model = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, + input_channels, self.hidden_size, self.hidden_channels, self.num_classes, self.num_layers, self.dropout) - self.model_for_inference = MLSTM_module(input_size, input_channels, + self.model_for_inference = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, + input_channels, self.hidden_size, self.hidden_channels, self.num_classes, self.num_layers, self.dropout) optimizer = optim.Adam(self.model.parameters(), lr=0.001) - if ts.num_classes == 2: - loss_fn = CROSS_ENTROPY() - else: - loss_fn = CROSS_ENTROPY() + loss_fn = CROSS_ENTROPY() return loss_fn, optimizer - # @convert_to_3d_torch_array - # def predict(self, ts: InputData, output_mode: str = 'default'): - # return super().predict(ts, output_mode) - - # def predict_for_fit(self, ts: InputData, output_mode: str = 'default'): - # return super().predict_for_fit(ts, output_mode) - @convert_to_3d_torch_array - def _fit_model(self, ts: InputData, mode='zero_padding'): - self.epochs = 1 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 + def _fit_model(self, ts: InputData): + mode = self.fitting_mode loss_fn, optimizer = self._init_model(ts) + train_loader, val_loader = self._prepare_data(ts, split_data=True, + collate_fn=getattr(self, '_augment_with_zeros')) + if mode == 'zero_padding': + super()._train_loop( + train_loader=train_loader, + val_loader=val_loader, + loss_fn=loss_fn, + optimizer=optimizer + ) + elif mode == 'moving_window': + self._train_loop( + train_loader=train_loader, + val_loader=None, + loss_fn=loss_fn, + optimizer=optimizer + ) + else: + raise ValueError('Unknown fitting mode') + + def _moving_window_output(self, inputs): + hidden_state = None + output = -torch.ones((inputs.shape[0], self.num_classes)) + for i in self.prediction_idx: + if i >= inputs.shape[-1]: + break + batch_interval = inputs[..., i - self.prediction_idx[0] : i + 1] + output, hidden_state = self.model(batch_interval, hidden_state, return_hidden=True) + return output - train_loader, val_loader = self._prepare_data(ts, split_data=False, - collate_fn=getattr(self, '_augment_zero_padding')) - self._train_loop( - train_loader=train_loader, - val_loader=val_loader, - loss_fn=loss_fn, - optimizer=optimizer, - ) + def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): + early_stopping = EarlyStopping() + scheduler = lr_scheduler.OneCycleLR(optimizer=optimizer, + steps_per_epoch=len(train_loader), + epochs=self.epochs, + max_lr=self.learning_rate) + if val_loader is None: + print('Not enough class samples for validation') + + best_model = None + best_val_loss = float('inf') + val_interval = self.get_validation_frequency( + self.epochs, self.learning_rate) - def _augment_zero_padding(self, batch,): - prediction_idx = self.prediction_idx - x, y = zip(*batch) - X, y = torch.stack(x), torch.stack(y) - y = np.tile(y, (len(prediction_idx), 1)) - res = [] - for i in prediction_idx: - zeroed_X = X[...] - zeroed_X[..., i + 1:] = 0 - res.append(zeroed_X) - res = np.concatenate(res, 0) - perm = np.random.permutation(res.shape[0]) - return torch.tensor(res[perm, ...]), torch.tensor(y[perm]) + for epoch in range(1, self.epochs + 1): + training_loss = 0.0 + valid_loss = 0.0 + self.model.train() + total = 0 + correct = 0 + for batch in tqdm(train_loader): + optimizer.zero_grad() + inputs, targets = batch + output = self._moving_window_output(inputs) + loss = loss_fn(output, targets.float()) + loss.backward() + optimizer.step() + training_loss += loss.data.item() * inputs.size(0) + total += targets.size(0) + correct += (torch.argmax(output, 1) == + torch.argmax(targets, 1)).sum().item() + + accuracy = correct / total + training_loss /= len(train_loader.dataset) + print('Epoch: {}, Accuracy = {}, Training Loss: {:.2f}'.format( + epoch, accuracy, training_loss)) + + if val_loader is not None and epoch % val_interval == 0: + self.model.eval() + total = 0 + correct = 0 + for batch in val_loader: + inputs, targets = batch + + output = self.model(inputs) + + loss = loss_fn(output, targets.float()) + + valid_loss += loss.data.item() * inputs.size(0) + total += targets.size(0) + correct += (torch.argmax(output, 1) == + torch.argmax(targets, 1)).sum().item() + if valid_loss < best_val_loss: + best_val_loss = valid_loss + best_model = copy.deepcopy(self.model) + + early_stopping(training_loss, self.model, './') + adjust_learning_rate(optimizer, scheduler, + epoch + 1, self.learning_rate, printout=False) + scheduler.step() + + if early_stopping.early_stop: + print("Early stopping") + break + + if best_model is not None: + self.model = best_model + + @convert_to_3d_torch_array + def _predict_model(self, x_test: InputData, output_mode: str = 'default'): + self.model.eval() + if self.fitting_mode == 'zero_padding': + x_test = self._padding(x_test).to(self._device) + pred = self.model(x_test) + elif self.fitting_mode == 'moving_window': + pred = self._moving_window_output(torch.tensor(x_test).float()) + else: + raise ValueError('Unknown prediction mode') + return self._convert_predict(pred, output_mode) + + def _padding(self, ts: np.array): + if ts.shape[-1] == self.input_size: + return torch.tensor(ts).float() + n, ch, size = ts.shape + x = torch.zeros((n, ch, self.input_size)).float() + x[..., :size] = ts + return x + + def _augment_with_zeros(self, batch: np.array): + X, y = zip(*batch) + X, y = np.stack(X), np.stack(y) + X_res, y_res = [], [] + for i in self.prediction_idx: + x = X[...] + x[..., :i + i] = 0 + X_res.append(x) + y_res.append(y) + X_res = np.concatenate(X_res) + y_res = np.concatenate(y_res) + perm = np.random.permutation(X_res.shape[0]) + return torch.tensor(X_res[perm]), torch.tensor(y_res[perm]) + + def _transform_score(self, probas): + # linear interp + thr = self.proba_thr + probas = probas - thr + positive = probas > 0 + probas[positive] *= 1 / (1 - thr) + probas[~positive] *= 1 / thr + return probas diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index 98a7e2986..cfbf25e1c 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -124,15 +124,25 @@ "min_samples_leaf": 10, "bootstrap": false }, + "ecec": { + "interval_percentage": 10, + "accuracy_importance": 0.7 + }, + "economy_k": { + "interval_percentage": 10, + "accuracy_importance": 0.7, + "cluster_factor": 1, + "lambda": 1 + }, "teaser": { "interval_percentage": 10, "consecutive_predictions": 3, - "hm_shift_to_acc": 2 + "accuracy_importance": 2 }, "proba_threshold_etc": { "interval_percentage": 10, "consecutive_predictions": 3, - "hm_shift_to_acc": 2 + "accuracy_importance": 2 }, "dt": { "max_depth": 5, diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 42f58446a..9b624321f 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -735,10 +735,18 @@ "non_linear" ] }, - "teaser": { + "ecec": { + "meta": "sklearn_class", + "tags": [ + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" + }, + "economy_k": { "meta": "sklearn_class", "tags": [ - "simple", "interpretable", "non_lagged", "non_linear" @@ -755,6 +763,16 @@ ], "input_type": "[DataTypesEnum.table]" }, + "teaser": { + "meta": "sklearn_class", + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" + }, "xgboost": { "meta": "sklearn_class", "presets": ["*tree"], diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index c7283a1bc..c3e226dc7 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -45,8 +45,10 @@ from fedot_ind.core.models.detection.custom.stat_detector import StatisticalDetector from fedot_ind.core.models.detection.probalistic.kalman import UnscentedKalmanFilter from fedot_ind.core.models.detection.subspaces.sst import SingularSpectrumTransformation -from fedot_ind.core.models.early_tc.teaser import TEASER +from fedot_ind.core.models.early_tc.ecec import ECEC +from fedot_ind.core.models.early_tc.economy_k import EconomyK from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier +from fedot_ind.core.models.early_tc.teaser import TEASER from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR @@ -93,8 +95,10 @@ class AtomizedModel(Enum): # for detection 'one_class_svm': OneClassSVM, # Early classification + 'ecec': ECEC, + 'economy_k': EconomyK, + 'proba_threshold_etc': ProbabilityThresholdClassifier, 'teaser': TEASER, - 'proba_threshold_etc': ProbabilityThresholdClassifier } FEDOT_PREPROC_MODEL = { # data standartization diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 994db4e51..c19e7340e 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -65,15 +65,41 @@ 'selection_strategy': {'hyperopt-dist': hp.choice, 'sampling-scope': [['sum', 'pairwise']]} }, - 'teaser': + 'ecec': { + 'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'accuracy_importance': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[i / 10 for i in range(11)]]}, + }, + 'economy_k': { + 'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'lambda': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[1e-6, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4, 1e6]]}, + 'accuracy_importance': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[i / 10 for i in range(11)]]}, + }, + 'mlstm_model': { + 'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'dropout': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, + 'hidden_size': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(10, 101, 10))]}, + 'num_layers': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(1, 6))]}, + 'hidden_channels': {'hyperopt-dist': hp.choice, + 'sampling-scope': [8, 16, 32, 64, 96]}, + }, + 'proba_threshold_etc': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, - 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'accuracy_importance': {'hyperopt-dist': hp.choice, 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, }, - 'proba_threshold_etc': + 'teaser': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, From 926eb927102b068b2710858351bce05868f300ba Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 15:02:54 +0300 Subject: [PATCH 29/43] slight fixes --- fedot_ind/core/models/early_tc/base_early_tc.py | 2 +- fedot_ind/core/models/early_tc/economy_k.py | 9 ++++++--- .../core/repository/data/default_operation_params.json | 4 ++-- fedot_ind/core/tuning/search_space.py | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 9fbc162c5..cbed8c463 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -124,7 +124,7 @@ def _score(self, X, y, accuracy_importance=None, training=True): predictions = self._predict(X, training)[0] prediction_points = predictions.shape[0] accuracies = (predictions == np.tile(y, (prediction_points, 1))).sum(axis=1) / len(y) - return (1 + accuracy_importance) * accuracies * self.earliness[:prediction_points] / (accuracy_importance * accuracies + self.earliness[:prediction_points]) + return (1 - accuracy_importance) * self.earliness[:prediction_points] + accuracy_importance * accuracies def _get_applicable_index(self, last_available_idx): idx = np.searchsorted(self.prediction_idx, last_available_idx, side='right') diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index bf09acd4b..c39097189 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -65,7 +65,7 @@ def __expected_costs(self, X, cluster_centroids, i): np.sum(self.state[i:], axis=-1), axes=(0, 2, 1) ) * self._pyck_[None, ...], axis=1) costs = cluster_probas @ s_glob.T # n_inst x time_left - costs += self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? + costs -= self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? return costs def _get_prediction_time(self, X, cluster_centroids, i): @@ -82,8 +82,11 @@ def predict_proba(self, X): def _transform_score(self, time): idx = self._estimator_for_predict[-1] - scores = -(1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) - scores[scores == 0] = 1 # no posibility for lininterp when sure + scores = (1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) # [1 / n; 1 ] - 1 / n) * n /(n - 1) * 2 - 1 + n = self.n_pred + scores -= 1 / n + scores *= n / (n - 1) * 2 + scores -= 1 return scores diff --git a/fedot_ind/core/repository/data/default_operation_params.json b/fedot_ind/core/repository/data/default_operation_params.json index cfbf25e1c..a91a8a938 100644 --- a/fedot_ind/core/repository/data/default_operation_params.json +++ b/fedot_ind/core/repository/data/default_operation_params.json @@ -137,12 +137,12 @@ "teaser": { "interval_percentage": 10, "consecutive_predictions": 3, - "accuracy_importance": 2 + "accuracy_importance": 0.5 }, "proba_threshold_etc": { "interval_percentage": 10, "consecutive_predictions": 3, - "accuracy_importance": 2 + "accuracy_importance": 0.5 }, "dt": { "max_depth": 5, diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index c19e7340e..832eaad26 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -97,15 +97,15 @@ 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, 'accuracy_importance': {'hyperopt-dist': hp.choice, - 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, 'teaser': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, - 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, - 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + 'accuracy_importance': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, 'deepar_model': {'epochs': {'hyperopt-dist': hp.choice, From 49d0570330408700a89905475c03a71293cb4f3a Mon Sep 17 00:00:00 2001 From: leostre Date: Mon, 15 Jul 2024 12:30:35 +0300 Subject: [PATCH 30/43] refactored train loop + microfixes --- .../core/metrics/metrics_implementation.py | 122 +++++++++++++++++- fedot_ind/core/models/early_tc/ecec.py | 6 +- fedot_ind/core/models/early_tc/economy_k.py | 3 +- .../core/models/early_tc/prob_threshold.py | 3 +- fedot_ind/core/models/early_tc/teaser.py | 5 +- .../models/nn/network_impl/base_nn_model.py | 117 ++++++++++------- .../core/models/nn/network_impl/mlstm.py | 97 +++++--------- 7 files changed, 227 insertions(+), 126 deletions(-) diff --git a/fedot_ind/core/metrics/metrics_implementation.py b/fedot_ind/core/metrics/metrics_implementation.py index fea9c2877..8803f18dd 100644 --- a/fedot_ind/core/metrics/metrics_implementation.py +++ b/fedot_ind/core/metrics/metrics_implementation.py @@ -1,6 +1,7 @@ from typing import Optional from typing import Union +import matplotlib.pyplot as plt import numpy as np import pandas as pd from fedot.core.data.data import InputData @@ -220,6 +221,9 @@ def smape(a, f, _=None): return 1 / len(a) * np.sum(2 * np.abs(f - a) / (np.abs(a) + np.abs(f)) * 100) +def rmse(y_true, y_pred): + return np.sqrt(mean_squared_error(y_true, y_pred)) + def mape(A, F): return mean_absolute_percentage_error(A, F) @@ -232,9 +236,6 @@ def calculate_regression_metric(target, **kwargs): target = target.astype(float) - def rmse(y_true, y_pred): - return np.sqrt(mean_squared_error(y_true, y_pred)) - metric_dict = {'r2': r2_score, 'mse': mean_squared_error, 'rmse': rmse, @@ -261,9 +262,6 @@ def calculate_forecasting_metric(target, **kwargs): target = target.astype(float) - def rmse(y_true, y_pred): - return np.sqrt(mean_squared_error(y_true, y_pred)) - metric_dict = { 'rmse': rmse, 'mae': mean_absolute_error, @@ -345,10 +343,95 @@ def kl_divergence(solution: pd.DataFrame, return np.average(solution.sum(axis=1), weights=sample_weights) else: return np.average(solution.mean()) + +class ETSCPareto(QualityMetric, ParetoMetrics): + def __init__(self, + target, + predicted_labels, + predicted_probs=None, + weigths: tuple = None, + mode: str = 'robust', + reduce: bool = True, + metric_list: tuple = ( + 'f1', 'roc_auc', 'accuracy', 'logloss', 'precision'), + default_value: float = 0.0): + self.target = target.flatten() + self.predicted_labels = predicted_labels + self.predicted_probs = predicted_probs + self.metric_list = metric_list + self.default_value = default_value + self.weights = weigths + self.mode = mode + self.columns = ['robustness'] if self.mode == 'robust' else [] + self.columns.extend(metric_list) + self.reduce = reduce + def metric(self) -> float: + if len(self.predicted_labels.shape) == 1: + self.predicted_labels = self.predicted_labels[None, ...] + self.predicted_probs = self.predicted_probs[None, ...] + + n_metrics = len(self.metric_list) + (self.mode == 'robust') + n_est = self.predicted_labels.shape[0] + result = np.zeros((n_est, n_metrics)) + if self.mode == 'robust': + mask = self.predicted_probs >= 0 + if not mask.any(): + return result + robustness = mask.sum(-1) / self.predicted_probs.shape[-1] + result[:, 0] = robustness.flatten() + else: + mask = np.ones_like(self.predicted_probs, dtype=bool) + + for est in range(n_est): + for i, metric in enumerate(self.metric_list, 1): + assert metric in CLASSIFICATION_METRIC_DICT, f'{metric} is not found in available metrics' + metric_value = CLASSIFICATION_METRIC_DICT[metric](self.target[mask[est]], + self.predicted_labels[est][mask[est]]) + result[est, i] = metric_value + + if self.weights is None: + if self.reduce: + self.weights = np.empty(n_metrics) + self.weights.fill(1 / len(self.weights)) + else: + self.weights = np.eye(n_metrics) + else: + assert self.weights.shape[-1] == self.metrics.shape[-1], 'Metrics and weights size mismatch!' + self.weights /= self.weights.sum() + + result = result @ self.weights.T + if not self.reduce: + return pd.DataFrame(result, columns=self.columns) + else: + return result + + def plot_bicrit_metric(self, metrics, select=None, metrics_names=None): + if not metrics_names: + metrics_names = ('Robustness', 'Accuracy') + plt.figure(figsize=(10, 10)) + assert metrics.shape[-1] == 2, 'only 2 metrics can be plotted' + for i, metric in enumerate(metrics): + selection = metric[select] + sizes = ((np.arange(selection.shape[0]) * 2)[::-1]) ** 1.5 + 10 + plt.scatter(*(metric[select]).T, + s=sizes, + label=i) + plt.legend(loc="upper right", bbox_to_anchor=(1.5, 1)) + plt.ylabel(metrics_names[1]) + plt.xlabel(metrics_names[0]) + plt.xlim((-0.05, 1.05)) + plt.ylim((-0.05, 1.05)) + plt.xticks(np.linspace(0, 1, 11)) + plt.yticks(np.linspace(0, 1, 11)) + plt.grid(True) + + def select_pareto_front(self, metrics, maximize=True): + pareto_mask = self.pareto_metric_list(metrics, maximise=maximize) + return metrics[pareto_mask] -class AnomalyMetric(QualityMetric): +class AnomalyMetric(QualityMetric): def __init__(self, target, predicted_labels, @@ -617,3 +700,28 @@ def calculate_detection_metric( target=target, predicted_labels=labels).metric() return metric_dict + +REGRESSION_METRIC_DICT = {'r2': r2_score, + 'mse': mean_squared_error, + 'rmse': rmse, + 'mae': mean_absolute_error, + 'msle': mean_squared_log_error, + 'mape': mean_absolute_percentage_error, + 'median_absolute_error': median_absolute_error, + 'explained_variance_score': explained_variance_score, + 'max_error': max_error, + 'd2_absolute_error_score': d2_absolute_error_score} + +CLASSIFICATION_METRIC_DICT = {'accuracy': accuracy_score, + 'f1': f1_score, + 'roc_auc': roc_auc_score, + 'precision': precision_score, + 'logloss': log_loss} + +FORECASTING_METRICS_DICT = { + 'rmse': rmse, + 'mae': mean_absolute_error, + 'median_absolute_error': median_absolute_error, + 'smape': smape, + 'mase': mase + } diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index f6e163d25..792810ce6 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -1,9 +1,11 @@ from typing import Optional -from fedot_ind.core.architecture.settings.computational import backend_methods as np + from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.early_tc.base_early_tc import BaseETC -from sklearn.model_selection import cross_val_predict from sklearn.metrics import confusion_matrix +from sklearn.model_selection import cross_val_predict + class ECEC(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index c39097189..fae2f409b 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -1,6 +1,7 @@ from typing import Optional -from fedot_ind.core.architecture.settings.computational import backend_methods as np + from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.early_tc.base_early_tc import BaseETC from sklearn.cluster import KMeans from sklearn.metrics import confusion_matrix diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 773f79d8e..fd1455a36 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -1,6 +1,7 @@ from typing import Optional -from fedot_ind.core.architecture.settings.computational import backend_methods as np + from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.early_tc.base_early_tc import BaseETC class ProbabilityThresholdClassifier(BaseETC): diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 23d6c078d..2dc905508 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -1,6 +1,7 @@ from typing import Optional -from fedot_ind.core.architecture.settings.computational import backend_methods as np + from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.early_tc.base_early_tc import BaseETC from sklearn.model_selection import GridSearchCV from sklearn.svm import OneClassSVM @@ -43,7 +44,7 @@ def _form_X_oc(self, predicted_probas): def _predict(self, X, training=False): estimator_indices, offset = self._select_estimators(X) X_ocs, predicted_probas, predicted_labels = zip( - *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] ) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions X_ocs = np.stack(X_ocs) diff --git a/fedot_ind/core/models/nn/network_impl/base_nn_model.py b/fedot_ind/core/models/nn/network_impl/base_nn_model.py index e9d6c7274..f285853d0 100644 --- a/fedot_ind/core/models/nn/network_impl/base_nn_model.py +++ b/fedot_ind/core/models/nn/network_impl/base_nn_model.py @@ -113,6 +113,69 @@ def _prepare_data(self, ts, split_data: bool = True, collate_fn=None): self.label_encoder = train_dataset.label_encoder return train_loader, val_loader + def _train_one_batch(self, batch, optimizer, loss_fn): + optimizer.zero_grad() + inputs, targets = batch + output = self.model(inputs) + loss = loss_fn(output, targets.float()) + loss.backward() + optimizer.step() + training_loss = loss.data.item() * inputs.size(0) + total = targets.size(0) + correct = (torch.argmax(output, 1) == + torch.argmax(targets, 1)).sum().item() + return training_loss, total, correct + + def _eval_one_batch(self, batch, loss_fn): + inputs, targets = batch + output = self.model(inputs) + loss = loss_fn(output, targets.float()) + valid_loss = loss.data.item() * inputs.size(0) + total = targets.size(0) + correct = (torch.argmax(output, 1) == + torch.argmax(targets, 1)).sum().item() + return valid_loss, total, correct + + def _run_one_epoch(self, train_loader, val_loader, + optimizer, loss_fn, + epoch, val_interval, + early_stopping, scheduler, + best_val_loss): + training_loss = 0.0 + valid_loss = 0.0 + self.model.train() + total = 0 + correct = 0 + best_model = self.model + for batch in tqdm(train_loader): + training_loss_batch, total_batch, correct_batch = self._train_one_batch(batch, optimizer, loss_fn) + training_loss += training_loss_batch + total += total_batch + correct += correct_batch + accuracy = correct / total + training_loss /= len(train_loader.dataset) + print('Epoch: {}, Accuracy = {}, Training Loss: {:.2f}'.format( + epoch, accuracy, training_loss)) + + if val_loader is not None and epoch % val_interval == 0: + self.model.eval() + total = 0 + correct = 0 + for batch in val_loader: + valid_loss_batch, total_batch, correct_batch = self._eval_one_batch(batch, loss_fn) + valid_loss += valid_loss_batch + total += total_batch + correct += correct_batch + if valid_loss < best_val_loss: + best_val_loss = valid_loss + best_model = copy.deepcopy(self.model) + + early_stopping(training_loss, self.model, './') + adjust_learning_rate(optimizer, scheduler, + epoch + 1, self.learning_rate, printout=False) + scheduler.step() + return best_model, best_val_loss + def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): early_stopping = EarlyStopping() scheduler = lr_scheduler.OneCycleLR(optimizer=optimizer, @@ -127,53 +190,13 @@ def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): self.epochs, self.learning_rate) loss_prefix = 'RMSE' if self.is_regression_task else 'Accuracy' for epoch in range(1, self.epochs + 1): - training_loss = 0.0 - valid_loss = 0.0 - self.model.train() - total = 0 - correct = 0 - for batch in tqdm(train_loader): - optimizer.zero_grad() - inputs, targets = batch - output = self.model(inputs) - loss = loss_fn(output, targets.float()) - loss.backward() - optimizer.step() - training_loss += loss.data.item() / inputs.size(0) if self.is_regression_task \ - else loss.data.item() * inputs.size(0) - total += targets.size(0) - correct += (torch.argmax(output, 1) == torch.argmax(targets, 1)).sum().item() \ - if not self.is_regression_task else 0 - - training_loss = training_loss / len(train_loader.dataset) if not self.is_regression_task else training_loss - accuracy = correct / total if not self.is_regression_task else training_loss - print('Epoch: {}, {}= {}, Training Loss: {:.2f}'.format( - epoch, loss_prefix, accuracy, training_loss)) - - if val_loader is not None and epoch % val_interval == 0: - self.model.eval() - total = 0 - correct = 0 - for batch in val_loader: - inputs, targets = batch - output = self.model(inputs) - - loss = loss_fn(output, targets.float()) - - valid_loss += loss.data.item() / inputs.size(0) if self.is_regression_task \ - else loss.data.item() * inputs.size(0) - total += targets.size(0) - correct += (torch.argmax(output, 1) == torch.argmax(targets, 1)).sum().item() \ - if not self.is_regression_task else 0 - if valid_loss < best_val_loss: - best_val_loss = valid_loss - best_model = copy.deepcopy(self.model) - - early_stopping(training_loss, self.model, './') - adjust_learning_rate(optimizer, scheduler, - epoch + 1, self.learning_rate, printout=False) - scheduler.step() - + best_model, best_val_loss = self._run_one_epoch( + train_loader, val_loader, + optimizer, loss_fn, + epoch, val_interval, + early_stopping, scheduler, + best_val_loss + ) if early_stopping.early_stop: print("Early stopping") break diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 3e1d3c4b5..604f28660 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -154,74 +154,39 @@ def _moving_window_output(self, inputs): batch_interval = inputs[..., i - self.prediction_idx[0] : i + 1] output, hidden_state = self.model(batch_interval, hidden_state, return_hidden=True) return output - - def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): - early_stopping = EarlyStopping() - scheduler = lr_scheduler.OneCycleLR(optimizer=optimizer, - steps_per_epoch=len(train_loader), - epochs=self.epochs, - max_lr=self.learning_rate) - if val_loader is None: - print('Not enough class samples for validation') - - best_model = None - best_val_loss = float('inf') - val_interval = self.get_validation_frequency( - self.epochs, self.learning_rate) - - for epoch in range(1, self.epochs + 1): - training_loss = 0.0 - valid_loss = 0.0 - self.model.train() - total = 0 - correct = 0 - for batch in tqdm(train_loader): - optimizer.zero_grad() - inputs, targets = batch - output = self._moving_window_output(inputs) - loss = loss_fn(output, targets.float()) - loss.backward() - optimizer.step() - training_loss += loss.data.item() * inputs.size(0) - total += targets.size(0) - correct += (torch.argmax(output, 1) == + + def _train_one_batch(self, batch, optimizer, loss_fn): + if self.fitting_mode == 'zero_padding': + return super()._train_one_batch(batch, optimizer, loss_fn) + elif self.fitting_mode == 'moving_window': + optimizer.zero_grad() + inputs, targets = batch + output = self._moving_window_output(inputs) + loss = loss_fn(output, targets.float()) + loss.backward() + optimizer.step() + training_loss = loss.data.item() * inputs.size(0) + total = targets.size(0) + correct = (torch.argmax(output, 1) == torch.argmax(targets, 1)).sum().item() - - accuracy = correct / total - training_loss /= len(train_loader.dataset) - print('Epoch: {}, Accuracy = {}, Training Loss: {:.2f}'.format( - epoch, accuracy, training_loss)) - - if val_loader is not None and epoch % val_interval == 0: - self.model.eval() - total = 0 - correct = 0 - for batch in val_loader: - inputs, targets = batch - - output = self.model(inputs) - - loss = loss_fn(output, targets.float()) - - valid_loss += loss.data.item() * inputs.size(0) - total += targets.size(0) - correct += (torch.argmax(output, 1) == + return training_loss, total, correct + else: + raise ValueError('Unknown fitting mode!') + + def _eval_one_batch(self, batch, loss_fn): + if self.fitting_mode == 'zero_padding': + return super()._eval_one_batch(batch, loss_fn) + elif self.fitting_mode == 'moving_window': + inputs, targets = batch + output = self._moving_window_output(inputs) + loss = loss_fn(output, targets.float()) + valid_loss = loss.data.item() * inputs.size(0) + total = targets.size(0) + correct = (torch.argmax(output, 1) == torch.argmax(targets, 1)).sum().item() - if valid_loss < best_val_loss: - best_val_loss = valid_loss - best_model = copy.deepcopy(self.model) - - early_stopping(training_loss, self.model, './') - adjust_learning_rate(optimizer, scheduler, - epoch + 1, self.learning_rate, printout=False) - scheduler.step() - - if early_stopping.early_stop: - print("Early stopping") - break - - if best_model is not None: - self.model = best_model + return valid_loss, total, correct + else: + raise ValueError('Unknown fitting mode!') @convert_to_3d_torch_array def _predict_model(self, x_test: InputData, output_mode: str = 'default'): From 2e2ad91d0b3ed6e23cab0a8ca2e801c25f5fdd7b Mon Sep 17 00:00:00 2001 From: leostre Date: Wed, 17 Jul 2024 02:33:53 +0300 Subject: [PATCH 31/43] to pull req --- fedot_ind/core/models/early_tc/base_early_tc.py | 4 ++-- fedot_ind/core/models/early_tc/ecec.py | 3 --- fedot_ind/core/models/early_tc/economy_k.py | 1 - fedot_ind/core/models/early_tc/prob_threshold.py | 2 -- fedot_ind/core/models/early_tc/teaser.py | 7 ++----- 5 files changed, 4 insertions(+), 13 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index cbed8c463..9c8e12cb8 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -85,9 +85,9 @@ def _predict(self, X, training=True): estimator_indices, offset = self._select_estimators(X, training) if not training: self._estimator_for_predict = estimator_indices - prediction = zip( + prediction = (np.stack(array_list) for array_list in zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary - ) + )) return prediction # see the output in _predict_one_slave def _consecutive_count(self, predicted_labels: List[np.array]): diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 792810ce6..576b3a4ba 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -23,11 +23,8 @@ def _predict_one_slave(self, X, i, offset=0): def _predict(self, X, training=False): predicted_labels, predicted_probas, reliabilities = super()._predict(X, training) - reliabilities = np.stack(reliabilities) confidences = 1 - np.cumprod(1 - reliabilities, axis=0) non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] - predicted_labels = np.stack(predicted_labels) - predicted_probas = np.stack(predicted_probas) return predicted_labels, predicted_probas, non_confident, confidences def predict_proba(self, X): diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index fae2f409b..481cad97b 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -78,7 +78,6 @@ def _get_prediction_time(self, X, cluster_centroids, i): def predict_proba(self, X): probas, times, _ = self._predict(X, training=False) - probas, times = np.stack(probas), np.stack(times) return super().predict_proba(probas, times) def _transform_score(self, time): diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index fd1455a36..b72a927f1 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -24,8 +24,6 @@ def predict_proba(self, X): def _predict(self, X, training=True): predicted_probas, predicted_labels = super()._predict(X, training) - predicted_probas = np.stack(predicted_probas) - predicted_labels = np.stack(predicted_labels) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions double_check = predicted_probas.max(axis=-1) > self.probability_threshold non_acceptance[non_acceptance & double_check] = False diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 2dc905508..0350d0886 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -43,13 +43,10 @@ def _form_X_oc(self, predicted_probas): def _predict(self, X, training=False): estimator_indices, offset = self._select_estimators(X) - X_ocs, predicted_probas, predicted_labels = zip( + X_ocs, predicted_probas, predicted_labels = map(np.stack, zip( *[self._predict_one_slave(X, i, offset) for i in estimator_indices] - ) + )) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions - X_ocs = np.stack(X_ocs) - predicted_probas = np.stack(predicted_probas) - predicted_labels = np.stack(predicted_labels) final_verdicts = np.zeros((len(estimator_indices), X.shape[0])) # for each point of estimation for i in range(predicted_labels.shape[0]): From fa802695e1d1b855f468844d5301a0ec7b21ced7 Mon Sep 17 00:00:00 2001 From: autopep8 bot Date: Tue, 16 Jul 2024 23:52:38 +0000 Subject: [PATCH 32/43] Automated autopep8 fixes --- fedot_ind/core/metrics/interval_metrics.py | 62 +++++----- .../core/metrics/metrics_implementation.py | 55 ++++----- .../core/models/early_tc/base_early_tc.py | 36 +++--- fedot_ind/core/models/early_tc/ecec.py | 29 +++-- fedot_ind/core/models/early_tc/economy_k.py | 50 ++++---- fedot_ind/core/models/early_tc/metrics.py | 62 +++++----- .../core/models/early_tc/prob_threshold.py | 9 +- fedot_ind/core/models/early_tc/teaser.py | 32 +++--- .../models/nn/network_impl/base_nn_model.py | 12 +- .../core/models/nn/network_impl/mlstm.py | 108 +++++++++--------- fedot_ind/core/tuning/search_space.py | 50 ++++---- tests/unit/core/models/test_teaser.py | 5 +- 12 files changed, 262 insertions(+), 248 deletions(-) diff --git a/fedot_ind/core/metrics/interval_metrics.py b/fedot_ind/core/metrics/interval_metrics.py index f4a5f6544..f95586147 100644 --- a/fedot_ind/core/metrics/interval_metrics.py +++ b/fedot_ind/core/metrics/interval_metrics.py @@ -1,17 +1,18 @@ from sklearn.metrics import confusion_matrix import numpy as np -import pandas as pd -from fedot.core.data.data import InputData, OutputData -from typing import Tuple, List, Optional, Union, Literal +import pandas as pd +from typing import Union, Literal + def conf_matrix(actual, predicted): cm = confusion_matrix(actual, predicted) return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) + def average_delay(boundaries, prediction, - point, - use_idx=True, - window_placement='lefter'): + point, + use_idx=True, + window_placement='lefter'): cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) # statistics statistics = { @@ -29,14 +30,16 @@ def average_delay(boundaries, prediction, i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() } return detection_history, statistics - + + def tp_transform(tps): return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) + def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], - prediction: pd.DataFrame, - use_switch_point: bool = True, # if first anomaly dot is considered as changepoint - use_idx: bool = False): + prediction: pd.DataFrame, + use_switch_point: bool = True, # if first anomaly dot is considered as changepoint + use_idx: bool = False): if isinstance(boundaries, pd.DataFrame): boundaries = boundaries.values.T anomaly_tsp = prediction[prediction == 1].sort_index().index @@ -44,18 +47,18 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], if boundaries.shape[1]: - FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest + FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest for i, (b_low, b_up) in enumerate(boundaries): all_tsp_in_window = prediction[b_low: b_up].index anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp - if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? + if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? FNs.append(i if use_idx else all_tsp_in_window) - TPs[i] = [b_low, - anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, + TPs[i] = [b_low, + anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, b_up] if not use_idx: FNs.append(all_tsp_in_window - anomaly_tsp_in_window) - FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest + FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest else: FPs.append(anomaly_tsp) @@ -69,9 +72,11 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], ) # cognate of single_detecting_boundaries -def get_boundaries(idx, actual_timestamps, window_size:int = None, - window_placement: Literal['left', 'right', 'central'] = 'left', - intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', + + +def get_boundaries(idx, actual_timestamps, window_size: int = None, + window_placement: Literal['left', 'right', 'central'] = 'left', + intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', ): # idx = idx # cast everything to pandas object fir the subsequent comfort @@ -82,30 +87,30 @@ def get_boundaries(idx, actual_timestamps, window_size:int = None, else: idx = pd.Series(idx) td = window_size - else: + else: raise TypeError('Unexpected type of ts index') - + boundaries = np.tile(actual_timestamps, (2, 1)) - # [0, ...] - lower bound, [1, ...] - upper + # [0, ...] - lower bound, [1, ...] - upper if window_placement == 'left': boundaries[0] -= td elif window_placement == 'central': boundaries[0] -= td / 2 boundaries[1] += td / 2 elif window_placement == 'right': - boundaries[1] += td + boundaries[1] += td else: raise ValueError('Unknown mode') - + if not len(actual_timestamps): return boundaries - # intersection resolution + # intersection resolution for i in range(len(actual_timestamps) - 1): if not boundaries[0, i + 1] > boundaries[1, i]: continue - if intersection_mode == 'shift_to_left': + if intersection_mode == 'shift_to_left': boundaries[0, i + 1] = boundaries[1, i] elif intersection_mode == 'shift_to_right': boundaries[1, i] = boundaries[0, i + 1] @@ -120,6 +125,7 @@ def get_boundaries(idx, actual_timestamps, window_size:int = None, boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) return boundaries + def nab(boundaries, predictions, mode='standard', custom_coefs=None): inner_coefs = { 'low_FP': [1.0, -0.11, -1.0], @@ -128,11 +134,9 @@ def nab(boundaries, predictions, mode='standard', custom_coefs=None): } coefs = custom_coefs or inner_coefs[mode] confusion_matrix = extract_cp_cm(boundaries, predictions) - + tps = confusion_matrix['tps'] - score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], + score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], coefs) return score - - diff --git a/fedot_ind/core/metrics/metrics_implementation.py b/fedot_ind/core/metrics/metrics_implementation.py index 8803f18dd..e419023ef 100644 --- a/fedot_ind/core/metrics/metrics_implementation.py +++ b/fedot_ind/core/metrics/metrics_implementation.py @@ -221,6 +221,7 @@ def smape(a, f, _=None): return 1 / len(a) * np.sum(2 * np.abs(f - a) / (np.abs(a) + np.abs(f)) * 100) + def rmse(y_true, y_pred): return np.sqrt(mean_squared_error(y_true, y_pred)) @@ -343,7 +344,8 @@ def kl_divergence(solution: pd.DataFrame, return np.average(solution.sum(axis=1), weights=sample_weights) else: return np.average(solution.mean()) - + + class ETSCPareto(QualityMetric, ParetoMetrics): def __init__(self, target, @@ -387,7 +389,7 @@ def metric(self) -> float: for i, metric in enumerate(self.metric_list, 1): assert metric in CLASSIFICATION_METRIC_DICT, f'{metric} is not found in available metrics' metric_value = CLASSIFICATION_METRIC_DICT[metric](self.target[mask[est]], - self.predicted_labels[est][mask[est]]) + self.predicted_labels[est][mask[est]]) result[est, i] = metric_value if self.weights is None: @@ -399,13 +401,13 @@ def metric(self) -> float: else: assert self.weights.shape[-1] == self.metrics.shape[-1], 'Metrics and weights size mismatch!' self.weights /= self.weights.sum() - + result = result @ self.weights.T if not self.reduce: return pd.DataFrame(result, columns=self.columns) else: return result - + def plot_bicrit_metric(self, metrics, select=None, metrics_names=None): if not metrics_names: metrics_names = ('Robustness', 'Accuracy') @@ -414,8 +416,8 @@ def plot_bicrit_metric(self, metrics, select=None, metrics_names=None): for i, metric in enumerate(metrics): selection = metric[select] sizes = ((np.arange(selection.shape[0]) * 2)[::-1]) ** 1.5 + 10 - plt.scatter(*(metric[select]).T, - s=sizes, + plt.scatter(*(metric[select]).T, + s=sizes, label=i) plt.legend(loc="upper right", bbox_to_anchor=(1.5, 1)) plt.ylabel(metrics_names[1]) @@ -425,7 +427,7 @@ def plot_bicrit_metric(self, metrics, select=None, metrics_names=None): plt.xticks(np.linspace(0, 1, 11)) plt.yticks(np.linspace(0, 1, 11)) plt.grid(True) - + def select_pareto_front(self, metrics, maximize=True): pareto_mask = self.pareto_metric_list(metrics, maximise=maximize) return metrics[pareto_mask] @@ -701,27 +703,28 @@ def calculate_detection_metric( predicted_labels=labels).metric() return metric_dict + REGRESSION_METRIC_DICT = {'r2': r2_score, - 'mse': mean_squared_error, - 'rmse': rmse, - 'mae': mean_absolute_error, - 'msle': mean_squared_log_error, - 'mape': mean_absolute_percentage_error, - 'median_absolute_error': median_absolute_error, - 'explained_variance_score': explained_variance_score, - 'max_error': max_error, - 'd2_absolute_error_score': d2_absolute_error_score} + 'mse': mean_squared_error, + 'rmse': rmse, + 'mae': mean_absolute_error, + 'msle': mean_squared_log_error, + 'mape': mean_absolute_percentage_error, + 'median_absolute_error': median_absolute_error, + 'explained_variance_score': explained_variance_score, + 'max_error': max_error, + 'd2_absolute_error_score': d2_absolute_error_score} CLASSIFICATION_METRIC_DICT = {'accuracy': accuracy_score, - 'f1': f1_score, - 'roc_auc': roc_auc_score, - 'precision': precision_score, - 'logloss': log_loss} + 'f1': f1_score, + 'roc_auc': roc_auc_score, + 'precision': precision_score, + 'logloss': log_loss} FORECASTING_METRICS_DICT = { - 'rmse': rmse, - 'mae': mean_absolute_error, - 'median_absolute_error': median_absolute_error, - 'smape': smape, - 'mase': mase - } + 'rmse': rmse, + 'mae': mean_absolute_error, + 'median_absolute_error': median_absolute_error, + 'smape': smape, + 'mase': mase +} diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 9c8e12cb8..f49a98127 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -7,9 +7,9 @@ class BaseETC(ClassifierMixin, BaseEstimator): - def __init__(self, params: Optional[OperationParameters] = None): + def __init__(self, params: Optional[OperationParameters] = None): if params is None: - params = {} + params = {} super().__init__() self.interval_percentage = params.get('interval_percentage', 10) self.consecutive_predictions = params.get('consecutive_predictions', 1) @@ -26,7 +26,9 @@ def _init_model(self, X, y): max_data_length = X.shape[-1] self.prediction_idx = self._compute_prediction_points(max_data_length) self.n_pred = len(self.prediction_idx) - self.slave_estimators = [WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) for _ in range(self.n_pred)] + self.slave_estimators = [ + WEASEL(random_state=self.random_state, support_probabilities=True, **self.weasel_params) + for _ in range(self.n_pred)] self.scalers = [StandardScaler() for _ in range(self.n_pred)] self._chosen_estimator_idx = -1 self.classes_ = [np.unique(y)] @@ -37,7 +39,7 @@ def required_length(self): if not hasattr(self, '_chosen_estimator_idx'): return None return self.prediction_idx[self._chosen_estimator_idx] - + @property def n_classes(self): return len(self.classes_[0]) @@ -50,23 +52,23 @@ def fit(self, X, y=None): self._fit_one_interval(X, y, i) def _fit_one_interval(self, X, y, i): - X_part = X[..., :self.prediction_idx[i] + 1] + X_part = X[..., :self.prediction_idx[i] + 1] X_part = self.scalers[i].fit_transform(X_part) probas = self.slave_estimators[i].fit_predict_proba(X_part, y) return probas def _predict_one_slave(self, X, i, offset=0): - X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] + X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] X_part = self.scalers[i].transform(X_part) probas = self.slave_estimators[i].predict_proba(X_part) - return probas, np.argmax(probas, axis=-1) - + return probas, np.argmax(probas, axis=-1) + def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) prediction_idx = np.arange(n_idx - 1, -1, -interval_length)[::-1][1:] - self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 return prediction_idx - + def _select_estimators(self, X, training=False): offset = 0 if not training and self.prediction_mode == 'best_by_harmonic_mean': @@ -80,15 +82,15 @@ def _select_estimators(self, X, training=False): else: raise ValueError('Unknown prediction mode') return estimator_indices, offset - + def _predict(self, X, training=True): estimator_indices, offset = self._select_estimators(X, training) if not training: self._estimator_for_predict = estimator_indices prediction = (np.stack(array_list) for array_list in zip( - *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary + *[self._predict_one_slave(X, i, offset) for i in estimator_indices] # check boundary )) - return prediction # see the output in _predict_one_slave + return prediction # see the output in _predict_one_slave def _consecutive_count(self, predicted_labels: List[np.array]): n = len(predicted_labels[0]) @@ -97,10 +99,10 @@ def _consecutive_count(self, predicted_labels: List[np.array]): for i in range(1, prediction_points): equal = predicted_labels[i - 1] == predicted_labels[i] consecutive_labels[i, equal] = consecutive_labels[i - 1, equal] + 1 - return consecutive_labels # prediction_points x n_instances - + return consecutive_labels # prediction_points x n_instances + def predict_proba(self, *args): - predicted_probas, scores, *_ = args + predicted_probas, scores, *_ = args if self.transform_score: scores = self._transform_score(scores) scores = np.tile(scores[..., None], (1, 1, self.n_classes)) @@ -108,7 +110,7 @@ def predict_proba(self, *args): if prediction.shape[1] == 1: prediction = prediction.squeeze(1) return prediction - + def predict(self, X): prediction = self.predict_proba(X) labels = prediction[0:1].argmax(-1) diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 576b3a4ba..b210d6cfa 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -11,27 +11,27 @@ class ECEC(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): super().__init__(params) self.__cv = 5 - + def _init_model(self, X, y): super()._init_model(X, y) self._reliabilities = np.zeros((self.n_pred, self.n_classes, self.n_classes)) def _predict_one_slave(self, X, i, offset=0): - predicted_probas, predicted_labels = super()._predict_one_slave(X, i, offset) - reliabilities = self._reliabilities[i, predicted_labels, predicted_labels].flatten() # n_inst + predicted_probas, predicted_labels = super()._predict_one_slave(X, i, offset) + reliabilities = self._reliabilities[i, predicted_labels, predicted_labels].flatten() # n_inst return predicted_labels.astype(int), predicted_probas, reliabilities - + def _predict(self, X, training=False): predicted_labels, predicted_probas, reliabilities = super()._predict(X, training) confidences = 1 - np.cumprod(1 - reliabilities, axis=0) non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] return predicted_labels, predicted_probas, non_confident, confidences - + def predict_proba(self, X): _, predicted_probas, non_confident, confidences = self._predict(X) predicted_probas[non_confident] = -1 return super().predict_proba(predicted_probas, confidences) - + def _fit_one_interval(self, X, y, i): X_part = X[..., :self.prediction_idx[i] + 1] X_part = self.scalers[i].fit_transform(X_part) @@ -40,20 +40,20 @@ def _fit_one_interval(self, X, y, i): return labels def _score(self, y, y_pred, alpha): - matches = (y_pred == np.tile(y, (self.n_pred, 1))) # n_pred x n_inst + matches = (y_pred == np.tile(y, (self.n_pred, 1))) # n_pred x n_inst n, n_inst, *_ = matches.shape confidences = np.ones((n, n_inst), dtype='float32') for i in range(self.n_pred): confidences[i] = self._reliabilities[i, y, y_pred[i]] - confidences = 1 - np.cumprod(1 - confidences, axis=0) # n_pred x n_inst - candidates = self._select_thrs(confidences) # n_candidates + confidences = 1 - np.cumprod(1 - confidences, axis=0) # n_pred x n_inst + candidates = self._select_thrs(confidences) # n_candidates cfs = np.zeros((len(candidates), n)) for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst - accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred + accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) self._chosen_estimator_idx = np.argmin(cfs.mean(0)) - return candidates[np.argmin(cfs, axis=0)] # n_pred + return candidates[np.argmin(cfs, axis=0)] # n_pred @staticmethod def _select_thrs(confidences): @@ -62,13 +62,13 @@ def _select_thrs(confidences): pair_means = C[:-1] + difference / 2 difference_shifted = np.roll(difference, 1) difference_idx = np.argwhere(difference <= difference_shifted) - means_candidates = pair_means[difference_idx].flatten() + means_candidates = pair_means[difference_idx].flatten() return means_candidates if len(means_candidates) else C - + @staticmethod def cost_func(earliness, accuracies, alpha): return alpha * (1 - accuracies) + (1 - alpha) * earliness - + def fit(self, X, y): y = np.array(y).flatten().astype(int) self._init_model(X, y) @@ -89,4 +89,3 @@ def _transform_score(self, confidences): confidences[positive] *= 1 / (1 - thr) confidences[~positive] *= 1 / thr return confidences - \ No newline at end of file diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 8847c1e8c..68649f00e 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -7,14 +7,15 @@ from sklearn.metrics import confusion_matrix from sklearn.model_selection import cross_val_predict + class EconomyK(BaseETC): - def __init__(self, params: Optional[OperationParameters] = None): + def __init__(self, params: Optional[OperationParameters] = None): if params is None: - params = {} + params = {} super().__init__(params) self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) - self._cluster_factor = params.get('cluster_factor' , 1) + self._cluster_factor = params.get('cluster_factor', 1) self._random_state = 2104 self.__cv = 5 @@ -22,21 +23,23 @@ def _init_model(self, X, y): super()._init_model(X, y) self.n_clusters = int(self._cluster_factor * self.n_classes) self._clusterizer = KMeans(self.n_clusters, random_state=self._random_state) - self.state = np.zeros((self.n_pred, self.n_clusters, self.n_classes, self.n_classes)) + self.state = np.zeros((self.n_pred, self.n_clusters, self.n_classes, self.n_classes)) def fit(self, X, y): y = y.flatten().astype(int) self._init_model(X, y) - self._pyck_ = confusion_matrix(y, self._clusterizer.fit(X).labels_, normalize='true')[:self.n_classes, :self.n_clusters] + self._pyck_ = confusion_matrix( + y, self._clusterizer.fit(X).labels_, normalize='true')[ + :self.n_classes, :self.n_clusters] for i in range(self.n_pred): self._fit_one_interval(X, y, i) - + def _fit_one_interval(self, X, y, i): X_part = X[..., :self.prediction_idx[i] + 1] X_part = self.scalers[i].fit_transform(X_part) y_pred = cross_val_predict(self.slave_estimators[i], X_part, y, cv=self.__cv) self.slave_estimators[i].fit(X_part, y) - states_by_i = np.zeros(( self.n_clusters, self.n_classes, self.n_classes)) + states_by_i = np.zeros((self.n_clusters, self.n_classes, self.n_classes)) np.add.at(states_by_i, (self._clusterizer.labels_, y, y_pred), 1) states_by_i /= np.mean(states_by_i, -2, keepdims=True) states_by_i[np.isnan(states_by_i)] = 0 @@ -44,45 +47,46 @@ def _fit_one_interval(self, X, y, i): self.state[i] = states_by_i def _predict_one_slave(self, X, i, offset=0): - cluster_centers = self._clusterizer.cluster_centers_[:, :self.prediction_idx[i] + 1] # n_clust x len + cluster_centers = self._clusterizer.cluster_centers_[:, :self.prediction_idx[i] + 1] # n_clust x len X_part = X[..., max(0, offset - 1):self.prediction_idx[i] + 1] # n_inst x len X_part = self.scalers[i].transform(X_part) probas = self.slave_estimators[i].predict_proba(X_part) - optimal_time, is_optimal = self._get_prediction_time(X_part, cluster_centers, i) + optimal_time, is_optimal = self._get_prediction_time(X_part, cluster_centers, i) return probas, optimal_time, is_optimal - + def __cluster_probas(self, X, centroids): length = centroids.shape[-1] diffs = np.subtract.outer(X, centroids).swapaxes(1, 2) - diffs = diffs[..., np.eye(length).astype(bool)] # n_inst x n_clust x len - distances = np.linalg.norm(diffs, axis=-1) + diffs = diffs[..., np.eye(length).astype(bool)] # n_inst x n_clust x len + distances = np.linalg.norm(diffs, axis=-1) delta_k = 1. - distances / distances.mean(axis=-1)[:, None] - s = 1. / (1. + np.exp(-self.lambda_ * delta_k)) - return s / s.sum(axis=-1)[:, None] # n_inst x n_clust + s = 1. / (1. + np.exp(-self.lambda_ * delta_k)) + return s / s.sum(axis=-1)[:, None] # n_inst x n_clust def __expected_costs(self, X, cluster_centroids, i): - cluster_probas = self.__cluster_probas(X, cluster_centroids) # n_inst x n_clust + cluster_probas = self.__cluster_probas(X, cluster_centroids) # n_inst x n_clust s_glob = np.sum(np.transpose( - np.sum(self.state[i:], axis=-1), axes=(0, 2, 1) - ) * self._pyck_[None, ...], axis=1) - costs = cluster_probas @ s_glob.T # n_inst x time_left - costs -= self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? + np.sum(self.state[i:], axis=-1), axes=(0, 2, 1) + ) * self._pyck_[None, ...], axis=1) + costs = cluster_probas @ s_glob.T # n_inst x time_left + costs -= self.earliness[None, i:] * (1 - self.accuracy_importance) # subtract or add ? return costs def _get_prediction_time(self, X, cluster_centroids, i): - costs = self.__expected_costs(X, cluster_centroids, i) + costs = self.__expected_costs(X, cluster_centroids, i) min_costs = np.argmin(costs, axis=-1) is_optimal = min_costs == 0 time_optimal = self.prediction_idx[min_costs + i] - return time_optimal, is_optimal # n_inst - + return time_optimal, is_optimal # n_inst + def predict_proba(self, X): probas, times, _ = self._predict(X, training=False) return super().predict_proba(probas, times) def _transform_score(self, time): idx = self._estimator_for_predict[-1] - scores = (1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) # [1 / n; 1 ] - 1 / n) * n /(n - 1) * 2 - 1 + # [1 / n; 1 ] - 1 / n) * n /(n - 1) * 2 - 1 + scores = (1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) n = self.n_pred scores -= 1 / n scores *= n / (n - 1) * 2 diff --git a/fedot_ind/core/models/early_tc/metrics.py b/fedot_ind/core/models/early_tc/metrics.py index f4a5f6544..f95586147 100644 --- a/fedot_ind/core/models/early_tc/metrics.py +++ b/fedot_ind/core/models/early_tc/metrics.py @@ -1,17 +1,18 @@ from sklearn.metrics import confusion_matrix import numpy as np -import pandas as pd -from fedot.core.data.data import InputData, OutputData -from typing import Tuple, List, Optional, Union, Literal +import pandas as pd +from typing import Union, Literal + def conf_matrix(actual, predicted): cm = confusion_matrix(actual, predicted) return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) + def average_delay(boundaries, prediction, - point, - use_idx=True, - window_placement='lefter'): + point, + use_idx=True, + window_placement='lefter'): cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) # statistics statistics = { @@ -29,14 +30,16 @@ def average_delay(boundaries, prediction, i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() } return detection_history, statistics - + + def tp_transform(tps): return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) + def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], - prediction: pd.DataFrame, - use_switch_point: bool = True, # if first anomaly dot is considered as changepoint - use_idx: bool = False): + prediction: pd.DataFrame, + use_switch_point: bool = True, # if first anomaly dot is considered as changepoint + use_idx: bool = False): if isinstance(boundaries, pd.DataFrame): boundaries = boundaries.values.T anomaly_tsp = prediction[prediction == 1].sort_index().index @@ -44,18 +47,18 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], if boundaries.shape[1]: - FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest + FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest for i, (b_low, b_up) in enumerate(boundaries): all_tsp_in_window = prediction[b_low: b_up].index anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp - if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? + if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? FNs.append(i if use_idx else all_tsp_in_window) - TPs[i] = [b_low, - anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, + TPs[i] = [b_low, + anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, b_up] if not use_idx: FNs.append(all_tsp_in_window - anomaly_tsp_in_window) - FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest + FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest else: FPs.append(anomaly_tsp) @@ -69,9 +72,11 @@ def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], ) # cognate of single_detecting_boundaries -def get_boundaries(idx, actual_timestamps, window_size:int = None, - window_placement: Literal['left', 'right', 'central'] = 'left', - intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', + + +def get_boundaries(idx, actual_timestamps, window_size: int = None, + window_placement: Literal['left', 'right', 'central'] = 'left', + intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', ): # idx = idx # cast everything to pandas object fir the subsequent comfort @@ -82,30 +87,30 @@ def get_boundaries(idx, actual_timestamps, window_size:int = None, else: idx = pd.Series(idx) td = window_size - else: + else: raise TypeError('Unexpected type of ts index') - + boundaries = np.tile(actual_timestamps, (2, 1)) - # [0, ...] - lower bound, [1, ...] - upper + # [0, ...] - lower bound, [1, ...] - upper if window_placement == 'left': boundaries[0] -= td elif window_placement == 'central': boundaries[0] -= td / 2 boundaries[1] += td / 2 elif window_placement == 'right': - boundaries[1] += td + boundaries[1] += td else: raise ValueError('Unknown mode') - + if not len(actual_timestamps): return boundaries - # intersection resolution + # intersection resolution for i in range(len(actual_timestamps) - 1): if not boundaries[0, i + 1] > boundaries[1, i]: continue - if intersection_mode == 'shift_to_left': + if intersection_mode == 'shift_to_left': boundaries[0, i + 1] = boundaries[1, i] elif intersection_mode == 'shift_to_right': boundaries[1, i] = boundaries[0, i + 1] @@ -120,6 +125,7 @@ def get_boundaries(idx, actual_timestamps, window_size:int = None, boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) return boundaries + def nab(boundaries, predictions, mode='standard', custom_coefs=None): inner_coefs = { 'low_FP': [1.0, -0.11, -1.0], @@ -128,11 +134,9 @@ def nab(boundaries, predictions, mode='standard', custom_coefs=None): } coefs = custom_coefs or inner_coefs[mode] confusion_matrix = extract_cp_cm(boundaries, predictions) - + tps = confusion_matrix['tps'] - score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], + score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], coefs) return score - - diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index b72a927f1..03551dbc7 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -4,10 +4,11 @@ from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.early_tc.base_early_tc import BaseETC + class ProbabilityThresholdClassifier(BaseETC): def __init__(self, params: Optional[OperationParameters] = None): if params is None: - params = {} + params = {} super().__init__(params) self.probability_threshold = params.get('probability_threshold', None) @@ -15,7 +16,7 @@ def _init_model(self, X, y): super()._init_model(X, y) if self.probability_threshold is None: self.probability_threshold = 1 / len(self.classes_[0]) - + def predict_proba(self, X): _, predicted_probas, non_acceptance = self._predict(X, training=False) predicted_probas[non_acceptance] = 0 @@ -33,11 +34,11 @@ def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) self._chosen_estimator_idx = np.argmax(scores) return scores - + def fit(self, X, y): super().fit(X, y) self._score(X, y, self.accuracy_importance) - + def _transform_score(self, confidences): thr = self.probability_threshold confidences = confidences - thr diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 0350d0886..6d7c4470f 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -8,7 +8,7 @@ class TEASER(BaseETC): - def __init__(self, params: Optional[OperationParameters] = None): + def __init__(self, params: Optional[OperationParameters] = None): super().__init__(params) self._oc_svm_params = (100., 10., 5., 2.5, 1.5, 1., 0.5, 0.25, 0.1) @@ -18,29 +18,29 @@ def _init_model(self, X, y): def _fit_one_interval(self, X, y, i): probas = super()._fit_one_interval(X, y, i) - filtered_probas = self._filter_trues(probas, y) # + filtered_probas = self._filter_trues(probas, y) X_oc = self._form_X_oc(filtered_probas) self.oc_estimators[i] = GridSearchCV(OneClassSVM(), - param_grid={"gamma": self._oc_svm_params}, - scoring='accuracy', - cv=min(X.shape[0], 10) - ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ + param_grid={"gamma": self._oc_svm_params}, + scoring='accuracy', + cv=min(X.shape[0], 10) + ).fit(X_oc, np.ones((len(X_oc), 1))).best_estimator_ def _predict_one_slave(self, X, i, offset=0): probas, labels = super()._predict_one_slave(X, i, offset) X_oc = self._form_X_oc(probas) return X_oc, probas, labels - - def _filter_trues(self, predicted_probas, y): # different logic in sktime + + def _filter_trues(self, predicted_probas, y): # different logic in sktime predicted_labels = np.argmax(predicted_probas, axis=-1).flatten() return predicted_probas[predicted_labels == y] - + def _form_X_oc(self, predicted_probas): d = (predicted_probas.max() - predicted_probas) d[d == 0] = 1 d = d.min(axis=-1).reshape(-1, 1) return np.hstack([predicted_probas, d]) - + def _predict(self, X, training=False): estimator_indices, offset = self._select_estimators(X) X_ocs, predicted_probas, predicted_labels = map(np.stack, zip( @@ -48,30 +48,30 @@ def _predict(self, X, training=False): )) non_acceptance = self._consecutive_count(predicted_labels) < self.consecutive_predictions final_verdicts = np.zeros((len(estimator_indices), X.shape[0])) - # for each point of estimation + # for each point of estimation for i in range(predicted_labels.shape[0]): # find not accepted points X_to_ith = X_ocs[i] # if they are not outliers final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # mark as accepted - final_verdicts[i] = final_verdict + final_verdicts[i] = final_verdict non_acceptance[non_acceptance & (final_verdict > 0)] = False return predicted_labels, predicted_probas, non_acceptance, final_verdicts - + def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] return super().predict_proba(predicted_probas, final_verdicts) - + def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) self._chosen_estimator_idx = np.argmax(scores) return scores - + def fit(self, X, y): super().fit(X, y) return self._score(X, y, self.accuracy_importance) - + def _transform_score(self, scores): return np.tanh(scores) diff --git a/fedot_ind/core/models/nn/network_impl/base_nn_model.py b/fedot_ind/core/models/nn/network_impl/base_nn_model.py index f285853d0..d76737d6a 100644 --- a/fedot_ind/core/models/nn/network_impl/base_nn_model.py +++ b/fedot_ind/core/models/nn/network_impl/base_nn_model.py @@ -123,9 +123,9 @@ def _train_one_batch(self, batch, optimizer, loss_fn): training_loss = loss.data.item() * inputs.size(0) total = targets.size(0) correct = (torch.argmax(output, 1) == - torch.argmax(targets, 1)).sum().item() + torch.argmax(targets, 1)).sum().item() return training_loss, total, correct - + def _eval_one_batch(self, batch, loss_fn): inputs, targets = batch output = self.model(inputs) @@ -133,11 +133,11 @@ def _eval_one_batch(self, batch, loss_fn): valid_loss = loss.data.item() * inputs.size(0) total = targets.size(0) correct = (torch.argmax(output, 1) == - torch.argmax(targets, 1)).sum().item() + torch.argmax(targets, 1)).sum().item() return valid_loss, total, correct def _run_one_epoch(self, train_loader, val_loader, - optimizer, loss_fn, + optimizer, loss_fn, epoch, val_interval, early_stopping, scheduler, best_val_loss): @@ -172,7 +172,7 @@ def _run_one_epoch(self, train_loader, val_loader, early_stopping(training_loss, self.model, './') adjust_learning_rate(optimizer, scheduler, - epoch + 1, self.learning_rate, printout=False) + epoch + 1, self.learning_rate, printout=False) scheduler.step() return best_model, best_val_loss @@ -188,7 +188,7 @@ def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): best_val_loss = float('inf') val_interval = self.get_validation_frequency( self.epochs, self.learning_rate) - loss_prefix = 'RMSE' if self.is_regression_task else 'Accuracy' + 'RMSE' if self.is_regression_task else 'Accuracy' for epoch in range(1, self.epochs + 1): best_model, best_val_loss = self._run_one_epoch( train_loader, val_loader, diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 604f28660..627eaf8a1 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -1,69 +1,65 @@ -import copy from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel -from typing import Optional, Callable, Any, List, Union +from typing import Optional from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.data.data import InputData, OutputData -from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE +from fedot.core.data.data import InputData +from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY import torch.optim as optim -import torch.optim.lr_scheduler as lr_scheduler import torch.nn as nn import torch.nn.functional as F import torch -from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array -import pandas as pd -from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping -import torch.utils.data as data + class SqueezeExciteBlock(nn.Module): - def __init__(self, input_channels, filters, reduce=4): - super().__init__() - self.filters = filters - self.pool = nn.AvgPool1d(input_channels) - self.bottleneck = max(self.filters // reduce, 4) - self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) - self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) - torch.nn.init.kaiming_normal_(self.fc1.weight.data) - torch.nn.init.kaiming_normal_(self.fc2.weight.data) - - def forward(self, x): - input_x = x - x = self.pool(x) - x = F.relu(self.fc1(x.view(-1, 1, self.filters))) - x = F.sigmoid(self.fc2(x)) - x = x.view(-1, self.filters, 1) * input_x - return x + def __init__(self, input_channels, filters, reduce=4): + super().__init__() + self.filters = filters + self.pool = nn.AvgPool1d(input_channels) + self.bottleneck = max(self.filters // reduce, 4) + self.fc1 = nn.Linear(self.filters, self.bottleneck, bias=False) + self.fc2 = nn.Linear(self.bottleneck, self.filters, bias=False) + torch.nn.init.kaiming_normal_(self.fc1.weight.data) + torch.nn.init.kaiming_normal_(self.fc2.weight.data) + + def forward(self, x): + input_x = x + x = self.pool(x) + x = F.relu(self.fc1(x.view(-1, 1, self.filters))) + x = F.sigmoid(self.fc2(x)) + x = x.view(-1, self.filters, 1) * input_x + return x + class MLSTM_module(nn.Module): def __init__(self, input_size, input_channels, - inner_size, inner_channels, + inner_size, inner_channels, output_size, num_layers, dropout=0.25): super().__init__() self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) self.lstm = nn.LSTM(input_size, inner_size, num_layers, - batch_first=True, dropout=dropout) - - squeeze_excite_size = input_size #if not interval else interval + batch_first=True, dropout=dropout) + + squeeze_excite_size = input_size # if not interval else interval self.conv_branch = nn.Sequential( nn.Conv1d(input_channels, inner_channels, padding='same', - kernel_size=9), + kernel_size=9), nn.BatchNorm1d(inner_channels), nn.ReLU(), SqueezeExciteBlock(squeeze_excite_size, inner_channels), nn.Conv1d(inner_channels, inner_channels * 2, padding='same', kernel_size=5, - ), # c x l | n x c x l - nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels * 2), # n x c | n x c x l nn.ReLU(), SqueezeExciteBlock(squeeze_excite_size, inner_channels * 2), nn.Conv1d(inner_channels * 2, inner_channels, padding='same', kernel_size=3, - ), # c x l | n x c x l - nn.BatchNorm1d(inner_channels), # n x c | n x c x l + ), # c x l | n x c x l + nn.BatchNorm1d(inner_channels), # n x c | n x c x l nn.ReLU(), ) seq = next(iter(self.conv_branch.modules())) @@ -72,8 +68,8 @@ def __init__(self, input_size, input_channels, torch.nn.init.kaiming_uniform_(seq[i].weight.data) def forward(self, x, hidden_state=None, return_hidden=False): - x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size - x_conv = self.conv_branch(x) # n x inner_ch x len + x_lstm, hidden_state = self.lstm(x, hidden_state) # n x input_ch x inner_size + x_conv = self.conv_branch(x) # n x inner_ch x len x = torch.cat([torch.flatten(x_lstm, start_dim=1), torch.flatten(x_conv, start_dim=1)], dim=-1) x = F.softmax(self.proj(x)) if return_hidden: @@ -84,7 +80,7 @@ def forward(self, x, hidden_state=None, return_hidden=False): class MLSTM(BaseNeuralModel): def __init__(self, params: Optional[OperationParameters] = None): if params is None: - params = {} + params = {} super().__init__() self.dropout = params.get('dropout', 0.25) self.hidden_size = params.get('hidden_size', 64) @@ -94,34 +90,34 @@ def __init__(self, params: Optional[OperationParameters] = None): self.min_ts_length = params.get('min_ts_length', 5) self.fitting_mode = params.get('fitting_mode', 'zero_padding') self.proba_thr = params.get('proba_thr', None) - + def __repr__(self): return 'MLSTM' - + def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) prediction_idx = np.arange(interval_length - 1, n_idx, interval_length) - self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 return prediction_idx, interval_length def _init_model(self, ts: InputData): _, input_channels, input_size = ts.features.shape self.input_size = input_size self.prediction_idx, self.interval = self._compute_prediction_points(input_size) - self.model = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, - input_channels, - self.hidden_size, self.hidden_channels, - self.num_classes, self.num_layers, - self.dropout) - self.model_for_inference = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, + self.model = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, input_channels, self.hidden_size, self.hidden_channels, self.num_classes, self.num_layers, self.dropout) + self.model_for_inference = MLSTM_module(input_size if self.fitting_mode != 'moving_window' else self.interval, + input_channels, + self.hidden_size, self.hidden_channels, + self.num_classes, self.num_layers, + self.dropout) optimizer = optim.Adam(self.model.parameters(), lr=0.001) loss_fn = CROSS_ENTROPY() return loss_fn, optimizer - + @convert_to_3d_torch_array def _fit_model(self, ts: InputData): mode = self.fitting_mode @@ -144,17 +140,17 @@ def _fit_model(self, ts: InputData): ) else: raise ValueError('Unknown fitting mode') - + def _moving_window_output(self, inputs): hidden_state = None output = -torch.ones((inputs.shape[0], self.num_classes)) for i in self.prediction_idx: if i >= inputs.shape[-1]: break - batch_interval = inputs[..., i - self.prediction_idx[0] : i + 1] + batch_interval = inputs[..., i - self.prediction_idx[0]: i + 1] output, hidden_state = self.model(batch_interval, hidden_state, return_hidden=True) return output - + def _train_one_batch(self, batch, optimizer, loss_fn): if self.fitting_mode == 'zero_padding': return super()._train_one_batch(batch, optimizer, loss_fn) @@ -168,11 +164,11 @@ def _train_one_batch(self, batch, optimizer, loss_fn): training_loss = loss.data.item() * inputs.size(0) total = targets.size(0) correct = (torch.argmax(output, 1) == - torch.argmax(targets, 1)).sum().item() + torch.argmax(targets, 1)).sum().item() return training_loss, total, correct else: raise ValueError('Unknown fitting mode!') - + def _eval_one_batch(self, batch, loss_fn): if self.fitting_mode == 'zero_padding': return super()._eval_one_batch(batch, loss_fn) @@ -183,7 +179,7 @@ def _eval_one_batch(self, batch, loss_fn): valid_loss = loss.data.item() * inputs.size(0) total = targets.size(0) correct = (torch.argmax(output, 1) == - torch.argmax(targets, 1)).sum().item() + torch.argmax(targets, 1)).sum().item() return valid_loss, total, correct else: raise ValueError('Unknown fitting mode!') @@ -199,7 +195,7 @@ def _predict_model(self, x_test: InputData, output_mode: str = 'default'): else: raise ValueError('Unknown prediction mode') return self._convert_predict(pred, output_mode) - + def _padding(self, ts: np.array): if ts.shape[-1] == self.input_size: return torch.tensor(ts).float() @@ -221,7 +217,7 @@ def _augment_with_zeros(self, batch: np.array): y_res = np.concatenate(y_res) perm = np.random.permutation(X_res.shape[0]) return torch.tensor(X_res[perm]), torch.tensor(y_res[perm]) - + def _transform_score(self, probas): # linear interp thr = self.proba_thr diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 832eaad26..7be4c626d 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -67,29 +67,29 @@ }, 'ecec': { 'interval_percentage': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[5, 10, 20, 25]]}, + 'sampling-scope': [[5, 10, 20, 25]]}, 'accuracy_importance': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[i / 10 for i in range(11)]]}, + 'sampling-scope': [[i / 10 for i in range(11)]]}, }, 'economy_k': { 'interval_percentage': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[5, 10, 20, 25]]}, + 'sampling-scope': [[5, 10, 20, 25]]}, 'lambda': {'hyperopt-dist': hp.choice, 'sampling-scope': [[1e-6, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4, 1e6]]}, 'accuracy_importance': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[i / 10 for i in range(11)]]}, + 'sampling-scope': [[i / 10 for i in range(11)]]}, }, 'mlstm_model': { 'interval_percentage': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[5, 10, 20, 25]]}, + 'sampling-scope': [[5, 10, 20, 25]]}, 'dropout': {'hyperopt-dist': hp.choice, 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, 'hidden_size': {'hyperopt-dist': hp.choice, - 'sampling-scope': [list(range(10, 101, 10))]}, + 'sampling-scope': [list(range(10, 101, 10))]}, 'num_layers': {'hyperopt-dist': hp.choice, - 'sampling-scope': [list(range(1, 6))]}, + 'sampling-scope': [list(range(1, 6))]}, 'hidden_channels': {'hyperopt-dist': hp.choice, - 'sampling-scope': [8, 16, 32, 64, 96]}, + 'sampling-scope': [8, 16, 32, 64, 96]}, }, 'proba_threshold_etc': {'interval_percentage': {'hyperopt-dist': hp.choice, @@ -97,32 +97,32 @@ 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, 'accuracy_importance': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, - }, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, + }, 'teaser': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, 'sampling_scope': [[1, 2, 3, 4, 5]]}, 'accuracy_importance': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, - }, - 'deepar_model': - {'epochs': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, + }, + 'deepar_model': + {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}, - 'dropout': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, - 'rnn_layers':{'hyperopt-dist': hp.choice, - 'sampling-scope': [list(range(1, 6))]}, - 'hidden_size':{'hyperopt-dist': hp.choice, - 'sampling-scope': [list(range(10, 101, 10))]}, - 'cell_type':{'hyperopt-dist': hp.choice, - 'sampling-scope': [['GRU', 'LSTM', 'RNN']]}, - 'expected_distribution': {'hyperopt-dist': hp.choice, - 'sampling-scope': [['normal', 'cauchy']]} - }, + 'dropout': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5]]}, + 'rnn_layers': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(1, 6))]}, + 'hidden_size': {'hyperopt-dist': hp.choice, + 'sampling-scope': [list(range(10, 101, 10))]}, + 'cell_type': {'hyperopt-dist': hp.choice, + 'sampling-scope': [['GRU', 'LSTM', 'RNN']]}, + 'expected_distribution': {'hyperopt-dist': hp.choice, + 'sampling-scope': [['normal', 'cauchy']]} + }, 'patch_tst_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}, diff --git a/tests/unit/core/models/test_teaser.py b/tests/unit/core/models/test_teaser.py index 2bc19b8de..1cf847f67 100644 --- a/tests/unit/core/models/test_teaser.py +++ b/tests/unit/core/models/test_teaser.py @@ -8,10 +8,12 @@ def teaser(): teaser = TEASER.TEASER({'interval_length': 10, 'prediction_mode': ''}) return teaser + @pytest.fixture(scope='module') def xy(): return np.random.randn((2, 23)), np.random.randint(0, 2, size=(2, 1)) + def test_get_applicable_index(teaser): teaser._init_model(23) idx, offset = teaser._get_last_applicable_idx(100) @@ -21,6 +23,7 @@ def test_get_applicable_index(teaser): assert offset == 100 - teaser.prediction_idx[idx], 'Wrong offset estimation in the middle' assert idx == len(teaser.prediction_idx) - 1 + def test_compute_prediction_points(teaser): indices = teaser._compute_prediction_points(23) assert 2 in indices @@ -31,5 +34,3 @@ def test_compute_prediction_points(teaser): # pass # def test_score(teaser): - - From c2126eda06f46441cf3314055d40d8ce91c0ee73 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 28 Jun 2024 14:11:17 +0300 Subject: [PATCH 33/43] both etc models are registered, available via api --- .../repository/data/industrial_model_repository.json | 10 ++++++++++ fedot_ind/core/repository/model_repository.py | 1 + fedot_ind/core/tuning/search_space.py | 8 ++++++++ 3 files changed, 19 insertions(+) diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 9b624321f..145aa979c 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -773,6 +773,16 @@ ], "input_type": "[DataTypesEnum.table]" }, + "proba_threshold_etc": { + "meta": "sklearn_class", + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" + }, "xgboost": { "meta": "sklearn_class", "presets": ["*tree"], diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index c3e226dc7..e79eea250 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -49,6 +49,7 @@ from fedot_ind.core.models.early_tc.economy_k import EconomyK from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier from fedot_ind.core.models.early_tc.teaser import TEASER +from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 832eaad26..2f1889a96 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -107,6 +107,14 @@ 'accuracy_importance': {'hyperopt-dist': hp.choice, 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, + 'proba_threshold_etc': + {'interval_percentage': {'hyperopt-dist': hp.choice, + 'sampling-scope': [[5, 10, 20, 25]]}, + 'acceptance_threshold': {'hyperopt-dist': hp.choice, + 'sampling_scope': [[1, 2, 3, 4, 5]]}, + 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, + 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, + }, 'deepar_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, From 143b7a2580b8b5459eb15eaf7a69173109166e57 Mon Sep 17 00:00:00 2001 From: leostre Date: Tue, 9 Jul 2024 14:33:24 +0300 Subject: [PATCH 34/43] fitting w augmentation --- fedot_ind/core/models/nn/network_impl/mlstm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 604f28660..9a22d7108 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -11,7 +11,7 @@ import torch from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array, fedot_data_type import pandas as pd from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping import torch.utils.data as data From ded4f22944f1a9c7b4e4635e3bb2fb2a6b28a6c5 Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 01:27:24 +0300 Subject: [PATCH 35/43] all work, but need eval --- fedot_ind/core/models/early_tc/base_early_tc.py | 3 +++ fedot_ind/core/models/early_tc/economy_k.py | 1 + 2 files changed, 4 insertions(+) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 9c8e12cb8..d488e22b0 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -77,6 +77,9 @@ def _select_estimators(self, X, training=False): elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) + elif 'last_available': + last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) + estimator_indices = [last_idx] else: raise ValueError('Unknown prediction mode') return estimator_indices, offset diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 481cad97b..9958d408d 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -15,6 +15,7 @@ def __init__(self, params: Optional[OperationParameters] = None): self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) self._cluster_factor = params.get('cluster_factor' , 1) + # self.confidence_mode = params.get('confidence_mode', 'time') # or 'confidence' self._random_state = 2104 self.__cv = 5 From 40afba90572107649b8316128b64fc5e433c173d Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 12:09:39 +0300 Subject: [PATCH 36/43] evth converged to one interface + refactored --- fedot_ind/core/models/early_tc/base_early_tc.py | 3 --- fedot_ind/core/models/early_tc/ecec.py | 2 ++ fedot_ind/core/models/early_tc/economy_k.py | 1 - fedot_ind/core/models/nn/network_impl/mlstm.py | 2 +- .../repository/data/industrial_model_repository.json | 11 ++++++++++- fedot_ind/core/repository/model_repository.py | 1 - fedot_ind/core/tuning/search_space.py | 2 +- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index d488e22b0..9c8e12cb8 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -77,9 +77,6 @@ def _select_estimators(self, X, training=False): elif training or self.prediction_mode == 'all': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) estimator_indices = np.arange(last_idx + 1) - elif 'last_available': - last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) - estimator_indices = [last_idx] else: raise ValueError('Unknown prediction mode') return estimator_indices, offset diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index 576b3a4ba..b83cc3254 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -25,6 +25,8 @@ def _predict(self, X, training=False): predicted_labels, predicted_probas, reliabilities = super()._predict(X, training) confidences = 1 - np.cumprod(1 - reliabilities, axis=0) non_confident = confidences < self.confidence_thresholds[:len(predicted_labels), None] + predicted_labels = np.stack(predicted_labels) + predicted_probas = np.stack(predicted_probas) return predicted_labels, predicted_probas, non_confident, confidences def predict_proba(self, X): diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 9958d408d..481cad97b 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -15,7 +15,6 @@ def __init__(self, params: Optional[OperationParameters] = None): self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda', 1.) self._cluster_factor = params.get('cluster_factor' , 1) - # self.confidence_mode = params.get('confidence_mode', 'time') # or 'confidence' self._random_state = 2104 self.__cv = 5 diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 9a22d7108..604f28660 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -11,7 +11,7 @@ import torch from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array, fedot_data_type +from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array import pandas as pd from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping import torch.utils.data as data diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index 145aa979c..b05cb3c95 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -766,7 +766,6 @@ "teaser": { "meta": "sklearn_class", "tags": [ - "simple", "interpretable", "non_lagged", "non_linear" @@ -783,6 +782,16 @@ ], "input_type": "[DataTypesEnum.table]" }, + "teaser": { + "meta": "sklearn_class", + "tags": [ + "simple", + "interpretable", + "non_lagged", + "non_linear" + ], + "input_type": "[DataTypesEnum.table]" + }, "xgboost": { "meta": "sklearn_class", "presets": ["*tree"], diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index e79eea250..c3e226dc7 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -49,7 +49,6 @@ from fedot_ind.core.models.early_tc.economy_k import EconomyK from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier from fedot_ind.core.models.early_tc.teaser import TEASER -from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor from fedot_ind.core.models.nn.network_impl.dummy_nn import DummyOverComplicatedNeuralNetwork from fedot_ind.core.models.nn.network_impl.deepar import DeepAR diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 2f1889a96..05b844998 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -107,7 +107,7 @@ 'accuracy_importance': {'hyperopt-dist': hp.choice, 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, - 'proba_threshold_etc': + 'teaser': {'interval_percentage': {'hyperopt-dist': hp.choice, 'sampling-scope': [[5, 10, 20, 25]]}, 'acceptance_threshold': {'hyperopt-dist': hp.choice, From d6ad8fda9a704823a1fda17267b7681a86b5d1cb Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 12 Jul 2024 15:02:54 +0300 Subject: [PATCH 37/43] slight fixes --- fedot_ind/core/tuning/search_space.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 05b844998..832eaad26 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -107,14 +107,6 @@ 'accuracy_importance': {'hyperopt-dist': hp.choice, 'sampling-scope': [[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1,]]}, }, - 'teaser': - {'interval_percentage': {'hyperopt-dist': hp.choice, - 'sampling-scope': [[5, 10, 20, 25]]}, - 'acceptance_threshold': {'hyperopt-dist': hp.choice, - 'sampling_scope': [[1, 2, 3, 4, 5]]}, - 'hm_shift_to_acc': {'hyperopt-dist': hp.choice, - 'sampling-scope': [0.01, 0.1, 0.5, 1, 5, 10, 100]}, - }, 'deepar_model': {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, From 8fec47f82fe63417a37e6fca04f9701210387816 Mon Sep 17 00:00:00 2001 From: leostre Date: Mon, 22 Jul 2024 17:32:52 +0300 Subject: [PATCH 38/43] added tests and notebook --- .../early_classification_example.ipynb | 588 ++++++++++++++++++ fedot_ind/core/metrics/interval_metrics.py | 3 +- .../core/metrics/metrics_implementation.py | 12 +- .../core/models/early_tc/base_early_tc.py | 50 +- fedot_ind/core/models/early_tc/ecec.py | 12 +- fedot_ind/core/models/early_tc/economy_k.py | 25 +- .../core/models/early_tc/prob_threshold.py | 21 +- fedot_ind/core/models/early_tc/teaser.py | 30 +- .../models/nn/network_impl/base_nn_model.py | 15 +- .../core/models/nn/network_impl/mlstm.py | 30 +- .../models/nn/network_impl/transformer.py | 2 + .../models/quantile/quantile_extractor.py | 2 +- .../data/industrial_model_repository.json | 4 +- .../unit/core/models/model_impl/test_mlstm.py | 37 ++ tests/unit/core/models/test_etc.py | 98 +++ 15 files changed, 866 insertions(+), 63 deletions(-) create mode 100644 examples/real_world_examples/industrial_examples/early_classification_example.ipynb create mode 100644 tests/unit/core/models/model_impl/test_mlstm.py create mode 100644 tests/unit/core/models/test_etc.py diff --git a/examples/real_world_examples/industrial_examples/early_classification_example.ipynb b/examples/real_world_examples/industrial_examples/early_classification_example.ipynb new file mode 100644 index 000000000..bd6f5fab0 --- /dev/null +++ b/examples/real_world_examples/industrial_examples/early_classification_example.ipynb @@ -0,0 +1,588 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For the following Early time series classification models let's load some univariate data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot_ind.api.utils.path_lib import PROJECT_PATH\n", + "import sys\n", + "import os\n", + "\n", + "if not os.getcwd() == PROJECT_PATH:\n", + " os.chdir(PROJECT_PATH)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot.core.data.data import InputData\n", + "from fedot.core.repository.dataset_types import DataTypesEnum\n", + "from fedot.core.repository.tasks import Task, TaskTypesEnum\n", + "from fedot_ind.api.utils.path_lib import PROJECT_PATH\n", + "from fedot_ind.core.architecture.settings.computational import backend_methods as np\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "from tqdm.autonotebook import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-07-22 13:42:16,908 - PyTorch version 2.2.2 available.\n" + ] + } + ], + "source": [ + "from fedot_ind.tools.loader import DataLoader\n", + "\n", + "def load_univariate_classification():\n", + " dl = DataLoader('Lightning7')\n", + " (train_series, train_labels), (test_series, test_labels) = dl.load_data()\n", + " train_data = InputData(idx=np.arange(test_series.shape[1]),\n", + " features=train_series.values,\n", + " target=train_labels,\n", + " task=Task(TaskTypesEnum.classification),\n", + " data_type=DataTypesEnum.table)\n", + " test_data = InputData(idx=np.arange(test_series.shape[1]),\n", + " features=test_series.values,\n", + " target=test_labels,\n", + " task=Task(TaskTypesEnum.classification),\n", + " data_type=DataTypesEnum.table)\n", + " return train_data, test_data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Shape of train_data.features: (70, 319)\n", + "test_data shape: (73, 319)\n", + "Number of classes: 7\n" + ] + } + ], + "source": [ + "train_data, test_data = load_univariate_classification()\n", + "print(f'Shape of train_data.features: {train_data.features.shape}\\ntest_data shape: {test_data.features.shape}')\n", + "print('Number of classes:', len(np.unique(train_data.target)))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAG2CAYAAACKxwc0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAArCElEQVR4nO3dfVSVdb7//9cWtlsxQBFvIAGxo3lPjXfH7BRMHhkyzFPTdGMOR6eaCjSlsYYpE5oMayZjMkazs0abdRbZOdNgZuZN3jEe9SgSpd1401A6mpKZbIHa7h/s3x9828YBFHDDdX30+Vhrr9V1y4v32kteXfvaezt8Pp9PAAAAhupgdQAAAICLQZkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEaztMzk5uZq1KhRCg0NVc+ePTV58mTt37+/3j7fffed0tPT1b17d11xxRW6/fbbdeLECYsSAwAAu7G0zGzdulXp6enauXOnNmzYIK/XqwkTJqiqqsq/z+zZs/X222/rv//7v7V161YdO3ZMt912m4WpAQCAnTjs9EWTX331lXr27KmtW7fqhhtuUEVFhXr06KGCggL99Kc/lSR9+umnGjRokHbs2KF//ud/tjgxAACwWrDVAX6ooqJCkhQRESFJ2rNnj7xer8aPH+/fZ+DAgYqNjW2yzHg8Hnk8Hv9ybW2tTp06pe7du8vhcLTxbwAAAALB5/PpzJkzio6OVocO538hyTZlpra2VrNmzdK4ceM0dOhQSdLx48fVsWNHde3atd6+vXr10vHjxxs9T25urnJycto6LgAAaAdHjhxRnz59zruPbcpMenq69u3bp23btl3UebKyspSZmelfrqioUGxsrMrKyhQaGnqxMQEAQDs4c+aM4uPjm/W32xZlJiMjQ6tXr1ZRUVG99tW7d2+dPXtWp0+frnd15sSJE+rdu3ej53K5XHK5XA3WR0REKCwsLODZAQBA4DmdTklq1i0ilr6byefzKSMjQ4WFhdq0aZPi4+PrbR8xYoScTqc2btzoX7d//34dPnxYY8eObe+4AADAhiy9MpOenq6CggK99dZbCg0N9d8HEx4ers6dOys8PFy/+MUvlJmZ6b+yMmPGDI0dO5Z3MgEAAEkWvzW7qUtHy5Yt07//+79LqvvQvEcffVSvv/66PB6PkpOT9cc//rHJl5n+L7fbrfDwcFVUVPAyEwAAhmjJ329bfc5MW6DMAABgnpb8/ea7mQAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwWrDVAQAAl76+v37H6ggB9/mCiVZHwP/DlRkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARrO0zBQVFSk1NVXR0dFyOBxauXJlve2VlZXKyMhQnz591LlzZw0ePFhLliyxJiwAALAlS8tMVVWVEhISlJ+f3+j2zMxMrV27Vv/5n/+pTz75RLNmzVJGRoZWrVrVzkkBAIBdBVv5w1NSUpSSktLk9u3btystLU2JiYmSpAceeECvvPKKdu3apUmTJrVTSgAAYGeWlpkLue6667Rq1SpNnz5d0dHR2rJliw4cOKAXX3yxyWM8Ho88Ho9/2e12S5K8Xq+8Xm+bZwYANOQK8lkdIeD4m9K2WjJfW5eZRYsW6YEHHlCfPn0UHBysDh066NVXX9UNN9zQ5DG5ubnKyclpsH79+vUKCQlpy7gAgCY8P9rqBIG3Zs0aqyNc0qqrq5u9r+3LzM6dO7Vq1SrFxcWpqKhI6enpio6O1vjx4xs9JisrS5mZmf5lt9utmJgYTZgwQWFhYe0VHQDwA0Oz11kdIeD2ZSdbHeGS9v0rK81h2zLz7bff6je/+Y0KCws1ceJESdLw4cNVWlqq3//+902WGZfLJZfL1WC90+mU0+ls08wAgMZ5ahxWRwg4/qa0rZbM17afM/P9PS4dOtSPGBQUpNraWotSAQAAu7H0ykxlZaUOHTrkXy4rK1NpaakiIiIUGxurG2+8UXPmzFHnzp0VFxenrVu36s9//rMWLlxoYWoAAGAnlpaZ4uJiJSUl+Ze/v9clLS1Ny5cv14oVK5SVlaUpU6bo1KlTiouL0/z58/Xggw9aFRkAANiMpWUmMTFRPl/Tb9fr3bu3li1b1o6JAACAaWx7zwwAAEBzUGYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRLC0zRUVFSk1NVXR0tBwOh1auXNlgn08++USTJk1SeHi4unTpolGjRunw4cPtHxYAANiSpWWmqqpKCQkJys/Pb3T7Z599puuvv14DBw7Uli1b9OGHH2ru3Lnq1KlTOycFAAB2FWzlD09JSVFKSkqT25944gndfPPNev755/3rrrrqqvaIBgAADGFpmTmf2tpavfPOO3rssceUnJys999/X/Hx8crKytLkyZObPM7j8cjj8fiX3W63JMnr9crr9bZ1bABAI1xBPqsjBBx/U9pWS+Zr2zJTXl6uyspKLViwQM8884yee+45rV27Vrfddps2b96sG2+8sdHjcnNzlZOT02D9+vXrFRIS0taxAQCNeH601QkCb82aNVZHuKRVV1c3e1+Hz+ezRV12OBwqLCz0X3U5duyYrrzySt19990qKCjw7zdp0iR16dJFr7/+eqPnaezKTExMjE6ePKmwsLA2/R0AAI0bmr3O6ggBty872eoIlzS3263IyEhVVFRc8O+3ba/MREZGKjg4WIMHD663ftCgQdq2bVuTx7lcLrlcrgbrnU6nnE5nwHMCAC7MU+OwOkLA8TelbbVkvrb9nJmOHTtq1KhR2r9/f731Bw4cUFxcnEWpAACA3Vh6ZaayslKHDh3yL5eVlam0tFQRERGKjY3VnDlzdOedd+qGG25QUlKS1q5dq7fffltbtmyxLjQAALAVS8tMcXGxkpKS/MuZmZmSpLS0NC1fvlz/9m//piVLlig3N1czZ87U1VdfrTfffFPXX3+9VZEBAIDNWFpmEhMTdaH7j6dPn67p06e3UyIAAGAa294zAwAA0ByUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDRLy0xRUZFSU1MVHR0th8OhlStXNrnvgw8+KIfDoby8vHbLBwAA7M/SMlNVVaWEhATl5+efd7/CwkLt3LlT0dHR7ZQMAACYItjKH56SkqKUlJTz7nP06FHNmDFD69at08SJE9spGQAAMIWlZeZCamtrNXXqVM2ZM0dDhgxp1jEej0cej8e/7Ha7JUler1der7dNcgIAzs8V5LM6QsDxN6VttWS+ti4zzz33nIKDgzVz5sxmH5Obm6ucnJwG69evX6+QkJBAxgMANNPzo61OEHhr1qyxOsIlrbq6utn72rbM7NmzR3/4wx9UUlIih8PR7OOysrKUmZnpX3a73YqJidGECRMUFhbWFlEBABcwNHud1RECbl92stURLmnfv7LSHLYtM3/7299UXl6u2NhY/7qamho9+uijysvL0+eff97ocS6XSy6Xq8F6p9Mpp9PZVnEBAOfhqWn+/5Sagr8pbasl87VtmZk6darGjx9fb11ycrKmTp2qadOmWZQKAADYjaVlprKyUocOHfIvl5WVqbS0VBEREYqNjVX37t3r7e90OtW7d29dffXV7R0VAADYlKVlpri4WElJSf7l7+91SUtL0/Llyy1KBQAATGJpmUlMTJTP1/y36zV1nwwAALh88d1MAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNEvLTFFRkVJTUxUdHS2Hw6GVK1f6t3m9Xj3++OMaNmyYunTpoujoaP385z/XsWPHrAsMAABsx9IyU1VVpYSEBOXn5zfYVl1drZKSEs2dO1clJSX661//qv3792vSpEkWJAUAAHYVbOUPT0lJUUpKSqPbwsPDtWHDhnrrXn75ZY0ePVqHDx9WbGxse0QEAAA2Z2mZaamKigo5HA517dq1yX08Ho88Ho9/2e12S6p72crr9bZ1RABAI1xBPqsjBBx/U9pWS+ZrTJn57rvv9Pjjj+vuu+9WWFhYk/vl5uYqJyenwfr169crJCSkLSMCAJrw/GirEwTemjVrrI5wSauurm72vg6fz2eLuuxwOFRYWKjJkyc32Ob1enX77bfrH//4h7Zs2XLeMtPYlZmYmBidPHnyvMcBCKyh2eusjhBw+7KTrY5gLJ4PaCm3263IyEhVVFRc8O+37a/MeL1e/exnP9MXX3yhTZs2XfAXcrlccrlcDdY7nU45nc62igng//DUOKyOEHD8G9J6PB/QUi2Zr63LzPdF5uDBg9q8ebO6d+9udSQAAGAzlpaZyspKHTp0yL9cVlam0tJSRUREKCoqSj/96U9VUlKi1atXq6amRsePH5ckRUREqGPHjlbFBgAANmJpmSkuLlZSUpJ/OTMzU5KUlpam7OxsrVq1SpJ0zTXX1Dtu8+bNSkxMbK+YAADAxiwtM4mJiTrf/cc2uTcZAADYGN/NBAAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRWlVm+vXrp6+//rrB+tOnT6tfv34XHQoAAKC5WlVmPv/8c9XU1DRY7/F4dPTo0YsOBQAA0FzBLdl51apV/v9et26dwsPD/cs1NTXauHGj+vbtG7BwAAAAF9KiMjN58mRJksPhUFpaWr1tTqdTffv21QsvvBCwcAAAABfSojJTW1srSYqPj9fu3bsVGRnZJqEAAACaq0Vl5ntlZWWBzgEAANAqrSozkrRx40Zt3LhR5eXl/is23/vTn/500cEAAACao1VlJicnR08//bRGjhypqKgoORyOQOcCAABollaVmSVLlmj58uWaOnVqoPMAAAC0SKs+Z+bs2bO67rrrAp0FAACgxVpVZu677z4VFBQEOgsAAECLteplpu+++05Lly7Ve++9p+HDh8vpdNbbvnDhwoCEAwAAuJBWlZkPP/xQ11xzjSRp37599bZxMzAAAGhPrSozmzdvDnQOAACAVmnVPTOBUlRUpNTUVEVHR8vhcGjlypX1tvt8Pj311FOKiopS586dNX78eB08eNCasAAAwJZadWUmKSnpvC8nbdq0qVnnqaqqUkJCgqZPn67bbrutwfbnn39eL730kl577TXFx8dr7ty5Sk5O1scff6xOnTq1JjoAALjEtKrMfH+/zPe8Xq9KS0u1b9++Bl9AeT4pKSlKSUlpdJvP51NeXp6efPJJ3XrrrZKkP//5z+rVq5dWrlypu+66qzXRAQDAJaZVZebFF19sdH12drYqKysvKtD3ysrKdPz4cY0fP96/Ljw8XGPGjNGOHTuaLDMej0cej8e/7Ha7JdUVLq/XG5BsAC7MFeSzOkLA8W9I6/F8QEu1ZL6t/m6mxtx7770aPXq0fv/731/0uY4fPy5J6tWrV731vXr18m9rTG5urnJychqsX79+vUJCQi46F4DmeX601QkCb82aNVZHMBbPB7RUdXV1s/cNaJnZsWOH5feyZGVlKTMz07/sdrsVExOjCRMmKCwszMJkwOVlaPY6qyME3L7sZKsjGIvnQx3m0Hzfv7LSHK0qM//3Zl2fz6cvv/xSxcXFmjt3bmtO2UDv3r0lSSdOnFBUVJR//YkTJxrcs/NDLpdLLperwXqn09ngw/0AtB1PzaX3mVP8G9J6PB/qMIe2OW+r3podHh5e7xEREaHExEStWbNG8+bNa80pG4iPj1fv3r21ceNG/zq3263//d//1dixYwPyMwAAgPladWVm2bJlAfnhlZWVOnTokH+5rKxMpaWlioiIUGxsrGbNmqVnnnlG/fv39781Ozo6WpMnTw7IzwcAAOa7qHtm9uzZo08++USSNGTIEF177bUtOr64uFhJSUn+5e/vdUlLS9Py5cv12GOPqaqqSg888IBOnz6t66+/XmvXrrX8vhwAAGAfrSoz5eXluuuuu7RlyxZ17dpVknT69GklJSVpxYoV6tGjR7POk5iYKJ+v6bfrORwOPf3003r66adbExMAAFwGWnXPzIwZM3TmzBl99NFHOnXqlE6dOqV9+/bJ7XZr5syZgc4IAADQpFZdmVm7dq3ee+89DRo0yL9u8ODBys/P14QJEwIWDgAA4EJadWWmtra20bdMOZ1O1dbWXnQoAACA5mpVmfnxj3+sRx55RMeOHfOvO3r0qGbPnq2bbropYOEAAAAupFVl5uWXX5bb7Vbfvn111VVX6aqrrlJ8fLzcbrcWLVoU6IwAAABNatU9MzExMSopKdF7772nTz/9VJI0aNCgel8KCQAA0B5adGVm06ZNGjx4sNxutxwOh/71X/9VM2bM0IwZMzRq1CgNGTJEf/vb39oqKwAAQAMtKjN5eXm6//77G/3CxvDwcP3yl7/UwoULAxYOAADgQlpUZj744AP95Cc/aXL7hAkTtGfPnosOBQAA0FwtKjMnTpw477dYBgcH66uvvrroUAAAAM3VojJz5ZVXat++fU1u//DDDxUVFXXRoQAAAJqrRWXm5ptv1ty5c/Xdd9812Pbtt99q3rx5uuWWWwIWDgAA4EJa9NbsJ598Un/96181YMAAZWRk6Oqrr5Ykffrpp8rPz1dNTY2eeOKJNgkKAADQmBaVmV69emn79u166KGHlJWV5f/Ga4fDoeTkZOXn56tXr15tEhQAAKAxLf7QvLi4OK1Zs0bffPONDh06JJ/Pp/79+6tbt25tkQ8AAOC8WvUJwJLUrVs3jRo1KpBZAAAAWqxV380EAABgF5QZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0W5eZmpoazZ07V/Hx8ercubOuuuoq/fa3v/V/wSUAAECrv5upPTz33HNavHixXnvtNQ0ZMkTFxcWaNm2awsPDNXPmTKvjAQAAG7B1mdm+fbtuvfVWTZw4UZLUt29fvf7669q1a5fFyQAAgF3Yusxcd911Wrp0qQ4cOKABAwbogw8+0LZt27Rw4cImj/F4PPJ4PP5lt9stSfJ6vfJ6vW2eGUAdV9Cl93Iw/4a0Hs+HOsyhbc7r8Nn4BpTa2lr95je/0fPPP6+goCDV1NRo/vz5ysrKavKY7Oxs5eTkNFhfUFCgkJCQtowLAAACpLq6Wvfcc48qKioUFhZ23n1tXWZWrFihOXPm6He/+52GDBmi0tJSzZo1SwsXLlRaWlqjxzR2ZSYmJkYnT5684DBaY2j2uoCf02r7spNbfAxzqMMczmEW+CGeD3WYQ/O53W5FRkY2q8zY+mWmOXPm6Ne//rXuuusuSdKwYcP0xRdfKDc3t8ky43K55HK5Gqx3Op1yOp0Bz+ipcQT8nFZrzZyYQx3mcA6zwA/xfKjDHNrmvLZ+a3Z1dbU6dKgfMSgoSLW1tRYlAgAAdmPrKzOpqamaP3++YmNjNWTIEL3//vtauHChpk+fbnU0AABgE7YuM4sWLdLcuXP18MMPq7y8XNHR0frlL3+pp556yupoAADAJmxdZkJDQ5WXl6e8vDyrowAAAJuy9T0zAAAAF0KZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARrN9mTl69Kjuvfdede/eXZ07d9awYcNUXFxsdSwAAGATwVYHOJ9vvvlG48aNU1JSkt5991316NFDBw8eVLdu3ayOBgAAbMLWZea5555TTEyMli1b5l8XHx9vYSIAAGA3ti4zq1atUnJysu644w5t3bpVV155pR5++GHdf//9TR7j8Xjk8Xj8y263W5Lk9Xrl9XoDntEV5Av4Oa3WmjkxhzrM4RxmgR/i+VCHObTNeR0+n8+2k+3UqZMkKTMzU3fccYd2796tRx55REuWLFFaWlqjx2RnZysnJ6fB+oKCAoWEhLRpXgAAEBjV1dW65557VFFRobCwsPPua+sy07FjR40cOVLbt2/3r5s5c6Z2796tHTt2NHpMY1dmYmJidPLkyQsOozWGZq8L+Dmtti87ucXHMIc6zOEcZoEf4vlQhzk0n9vtVmRkZLPKjK1fZoqKitLgwYPrrRs0aJDefPPNJo9xuVxyuVwN1judTjmdzoBn9NQ4An5Oq7VmTsyhDnM4h1ngh3g+1GEObXNeW781e9y4cdq/f3+9dQcOHFBcXJxFiQAAgN3YuszMnj1bO3fu1LPPPqtDhw6poKBAS5cuVXp6utXRAACATdi6zIwaNUqFhYV6/fXXNXToUP32t79VXl6epkyZYnU0AABgE7a+Z0aSbrnlFt1yyy1WxwAAADZl6yszAAAAF0KZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACj2f67mQDAZH1//Y7VEQLu8wUTrY4A1MOVGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjGVVmFixYIIfDoVmzZlkdBQAA2IQxZWb37t165ZVXNHz4cKujAAAAGzGizFRWVmrKlCl69dVX1a1bN6vjAAAAGwm2OkBzpKena+LEiRo/fryeeeaZ8+7r8Xjk8Xj8y263W5Lk9Xrl9XoDns0V5Av4Oa3WmjkxhzrM4RxmUYc51GEOdZhD25zX4fP5bD3ZFStWaP78+dq9e7c6deqkxMREXXPNNcrLy2t0/+zsbOXk5DRYX1BQoJCQkDZOCwAAAqG6ulr33HOPKioqFBYWdt59bV1mjhw5opEjR2rDhg3+e2UuVGYauzITExOjkydPXnAYrTE0e13Az2m1fdnJLT6GOdRhDucwizrMoQ5zqMMcms/tdisyMrJZZcbWLzPt2bNH5eXl+tGPfuRfV1NTo6KiIr388svyeDwKCgqqd4zL5ZLL5WpwLqfTKafTGfCMnhpHwM9ptdbMiTnUYQ7nMIs6zKEOc6jDHNrmvLYuMzfddJP27t1bb920adM0cOBAPf744w2KDAAAuPzYusyEhoZq6NCh9dZ16dJF3bt3b7AeAABcnox4azYAAEBTbH1lpjFbtmyxOgIAALARrswAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRbF9mcnNzNWrUKIWGhqpnz56aPHmy9u/fb3UsAABgE7YvM1u3blV6erp27typDRs2yOv1asKECaqqqrI6GgAAsIFgqwNcyNq1a+stL1++XD179tSePXt0ww03WJQKAADYhe3LzP9VUVEhSYqIiGh0u8fjkcfj8S+73W5JktfrldfrDXgeV5Av4Oe0WmvmxBzqMIdzmEUd5lCHOdRhDm1zXofP5zNmsrW1tZo0aZJOnz6tbdu2NbpPdna2cnJyGqwvKChQSEhIW0cEAAABUF1drXvuuUcVFRUKCws7775GlZmHHnpI7777rrZt26Y+ffo0uk9jV2ZiYmJ08uTJCw6jNYZmrwv4Oa22Lzu5xccwhzrM4RxmUYc51GEOdZhD87ndbkVGRjarzBjzMlNGRoZWr16toqKiJouMJLlcLrlcrgbrnU6nnE5nwHN5ahwBP6fVWjMn5lCHOZzDLOowhzrMoQ5zaJvz2r7M+Hw+zZgxQ4WFhdqyZYvi4+OtjgQAAGzE9mUmPT1dBQUFeuuttxQaGqrjx49LksLDw9W5c2eL0wEAAKvZ/nNmFi9erIqKCiUmJioqKsr/eOONN6yOBgAAbMD2V2YMuj8ZAABYwPZXZgAAAM6HMgMAAIxGmQEAAEajzAAAAKNRZgAAgNEoMwAAwGiUGQAAYDTKDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZAABgNMoMAAAwGmUGAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxmRJnJz89X37591alTJ40ZM0a7du2yOhIAALAJ25eZN954Q5mZmZo3b55KSkqUkJCg5ORklZeXWx0NAADYgO3LzMKFC3X//fdr2rRpGjx4sJYsWaKQkBD96U9/sjoaAACwgWCrA5zP2bNntWfPHmVlZfnXdejQQePHj9eOHTsaPcbj8cjj8fiXKyoqJEmnTp2S1+sNeMbg/68q4Oe02tdff93iY5hDHeZwDrOowxzqMIc6zKH5zpw5I0ny+XwX3tlnY0ePHvVJ8m3fvr3e+jlz5vhGjx7d6DHz5s3zSeLBgwcPHjx4XAKPI0eOXLAv2PrKTGtkZWUpMzPTv1xbW6tTp06pe/fucjgcFiZrPbfbrZiYGB05ckRhYWFWx7EMcziHWdRhDnWYwznMos6lMAefz6czZ84oOjr6gvvausxERkYqKChIJ06cqLf+xIkT6t27d6PHuFwuuVyueuu6du3aVhHbVVhYmLFPykBiDucwizrMoQ5zOIdZ1DF9DuHh4c3az9Y3AHfs2FEjRozQxo0b/etqa2u1ceNGjR071sJkAADALmx9ZUaSMjMzlZaWppEjR2r06NHKy8tTVVWVpk2bZnU0AABgA7YvM3feeae++uorPfXUUzp+/LiuueYarV27Vr169bI6WrtxuVyaN29eg5fPLjfM4RxmUYc51GEO5zCLOpfbHBw+X3Pe8wQAAGBPtr5nBgAA4EIoMwAAwGiUGQAAYDTKDAAAMBplxgD5+fnq27evOnXqpDFjxmjXrl1WR2p3RUVFSk1NVXR0tBwOh1auXGl1pHaXm5urUaNGKTQ0VD179tTkyZO1f/9+q2NZYvHixRo+fLj/A8HGjh2rd9991+pYlluwYIEcDodmzZpldZR2lZ2dLYfDUe8xcOBAq2NZ5ujRo7r33nvVvXt3de7cWcOGDVNxcbHVsdoUZcbm3njjDWVmZmrevHkqKSlRQkKCkpOTVV5ebnW0dlVVVaWEhATl5+dbHcUyW7duVXp6unbu3KkNGzbI6/VqwoQJqqq69L647kL69OmjBQsWaM+ePSouLtaPf/xj3Xrrrfroo4+sjmaZ3bt365VXXtHw4cOtjmKJIUOG6Msvv/Q/tm3bZnUkS3zzzTcaN26cnE6n3n33XX388cd64YUX1K1bN6ujta2L/zpItKXRo0f70tPT/cs1NTW+6OhoX25uroWprCXJV1hYaHUMy5WXl/sk+bZu3Wp1FFvo1q2b7z/+4z+sjmGJM2fO+Pr37+/bsGGD78Ybb/Q98sgjVkdqV/PmzfMlJCRYHcMWHn/8cd/1119vdYx2x5UZGzt79qz27Nmj8ePH+9d16NBB48eP144dOyxMBjuoqKiQJEVERFicxFo1NTVasWKFqqqqLtuvOUlPT9fEiRPr/VtxuTl48KCio6PVr18/TZkyRYcPH7Y6kiVWrVqlkSNH6o477lDPnj117bXX6tVXX7U6VpujzNjYyZMnVVNT0+DTjnv16qXjx49blAp2UFtbq1mzZmncuHEaOnSo1XEssXfvXl1xxRVyuVx68MEHVVhYqMGDB1sdq92tWLFCJSUlys3NtTqKZcaMGaPly5dr7dq1Wrx4scrKyvQv//IvOnPmjNXR2t3f//53LV68WP3799e6dev00EMPaebMmXrttdesjtambP91BgAaSk9P1759+y7b+wIk6eqrr1ZpaakqKir0l7/8RWlpadq6detlVWiOHDmiRx55RBs2bFCnTp2sjmOZlJQU/38PHz5cY8aMUVxcnP7rv/5Lv/jFLyxM1v5qa2s1cuRIPfvss5Kka6+9Vvv27dOSJUuUlpZmcbq2w5UZG4uMjFRQUJBOnDhRb/2JEyfUu3dvi1LBahkZGVq9erU2b96sPn36WB3HMh07dtQ//dM/acSIEcrNzVVCQoL+8Ic/WB2rXe3Zs0fl5eX60Y9+pODgYAUHB2vr1q166aWXFBwcrJqaGqsjWqJr164aMGCADh06ZHWUdhcVFdWg0A8aNOiSf9mNMmNjHTt21IgRI7Rx40b/utraWm3cuPGyvTfgcubz+ZSRkaHCwkJt2rRJ8fHxVkeyldraWnk8HqtjtKubbrpJe/fuVWlpqf8xcuRITZkyRaWlpQoKCrI6oiUqKyv12WefKSoqyuoo7W7cuHENPrLhwIEDiouLsyhR++BlJpvLzMxUWlqaRo4cqdGjRysvL09VVVWaNm2a1dHaVWVlZb3/yyorK1NpaakiIiIUGxtrYbL2k56eroKCAr311lsKDQ313zcVHh6uzp07W5yufWVlZSklJUWxsbE6c+aMCgoKtGXLFq1bt87qaO0qNDS0wT1TXbp0Uffu3S+re6l+9atfKTU1VXFxcTp27JjmzZunoKAg3X333VZHa3ezZ8/Wddddp2effVY/+9nPtGvXLi1dulRLly61OlrbsvrtVLiwRYsW+WJjY30dO3b0jR492rdz506rI7W7zZs3+yQ1eKSlpVkdrd009vtL8i1btszqaO1u+vTpvri4OF/Hjh19PXr08N10002+9evXWx3LFi7Ht2bfeeedvqioKF/Hjh19V155pe/OO+/0HTp0yOpYlnn77bd9Q4cO9blcLt/AgQN9S5cutTpSm3P4fD6fRT0KAADgonHPDAAAMBplBgAAGI0yAwAAjEaZAQAARqPMAAAAo1FmAACA0SgzAADAaJQZALbncDi0cuVKq2MAsCnKDADLHT9+XDNmzFC/fv3kcrkUExOj1NTUet9LBgBN4buZAFjq888/17hx49S1a1f97ne/07Bhw+T1erVu3Tqlp6fr008/tToiAJvjygwASz388MNyOBzatWuXbr/9dg0YMEBDhgxRZmamdu7c2egxjz/+uAYMGKCQkBD169dPc+fOldfr9W//4IMPlJSUpNDQUIWFhWnEiBEqLi6WJH3xxRdKTU1Vt27d1KVLFw0ZMkRr1qxpl98VQNvgygwAy5w6dUpr167V/Pnz1aVLlwbbu3bt2uhxoaGhWr58uaKjo7V3717df//9Cg0N1WOPPSZJmjJliq699lotXrxYQUFBKi0tldPplFT37eNnz55VUVGRunTpoo8//lhXXHFFm/2OANoeZQaAZQ4dOiSfz6eBAwe26Lgnn3zS/999+/bVr371K61YscJfZg4fPqw5c+b4z9u/f3///ocPH9btt9+uYcOGSZL69et3sb8GAIvxMhMAy/h8vlYd98Ybb2jcuHHq3bu3rrjiCj355JM6fPiwf3tmZqbuu+8+jR8/XgsWLNBnn33m3zZz5kw988wzGjdunObNm6cPP/zwon8PANaizACwTP/+/eVwOFp0k++OHTs0ZcoU3XzzzVq9erXef/99PfHEEzp79qx/n+zsbH300UeaOHGiNm3apMGDB6uwsFCSdN999+nvf/+7pk6dqr1792rkyJFatGhRwH83AO3H4Wvt/xoBQACkpKRo79692r9/f4P7Zk6fPq2uXbvK4XCosLBQkydP1gsvvKA//vGP9a623HffffrLX/6i06dPN/oz7r77blVVVWnVqlUNtmVlZemdd97hCg1gMK7MALBUfn6+ampqNHr0aL355ps6ePCgPvnkE7300ksaO3Zsg/379++vw4cPa8WKFfrss8/00ksv+a+6SNK3336rjIwMbdmyRV988YX+53/+R7t379agQYMkSbNmzdK6detUVlamkpISbd682b8NgJm4ARiApfr166eSkhLNnz9fjz76qL788kv16NFDI0aM0OLFixvsP2nSJM2ePVsZGRnyeDyaOHGi5s6dq+zsbElSUFCQvv76a/385z/XiRMnFBkZqdtuu005OTmSpJqaGqWnp+sf//iHwsLC9JOf/EQvvvhie/7KAAKMl5kAAIDReJkJAAAYjTIDAACMRpkBAABGo8wAAACjUWYAAIDRKDMAAMBolBkAAGA0ygwAADAaZQYAABiNMgMAAIxGmQEAAEajzAAAAKP9/5Ie4+1FNbFfAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.bar(*(np.unique(train_data.target, return_counts=True)))\n", + "plt.ylabel('Count')\n", + "plt.xlabel('Class')\n", + "plt.yticks(np.arange(0, 21, 2));\n", + "plt.grid(axis='y');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Standalone models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All the models presented below share the same principle of functioning:\n", + "\n", + "Since there's no way for many classifiers to support inference for different data sizes, the basic Early ETSC class implements fitting of multiple slave estimators according to the specified intervals (by *interval_percentage* key word) on time series instance. Depending on the length of features passed and *prediction_mode* parameter the appropriate subset if classifiers is selected and inference is committed. The details of fitting and inference vary drastically. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot_ind.core.models.early_tc import base_early_tc as BASE_ETC\n", + "from importlib import reload\n", + "\n", + "Xtr, ytr = train_data.features.squeeze(), train_data.target\n", + "Xte, yte = test_data.features.squeeze(), test_data.target\n", + "\n", + "INTEVAL_PERCENTAGE = 10\n", + "earliness = np.round((1 - np.arange(0, Xtr.shape[0], int(INTEVAL_PERCENTAGE * Xtr.shape[0] / 100)) / Xtr.shape[0]) * 100)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 198, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot_ind.core.metrics.metrics_implementation import ETSCPareto\n", + "\n", + "def eval_param_influence(model, prm_name, options, **model_kw):\n", + " r = {}\n", + " for option in tqdm(options, desc='Options'):\n", + " model_ = model({prm_name: option, **model_kw})\n", + " model_.fit(Xtr, ytr)\n", + " labels, scores = model_.predict(Xte)\n", + " r[option] = ETSCPareto(yte, labels.astype(int), scores, reduce=False, metric_list=('accuracy',)).metric().copy()\n", + " return r\n", + "\n", + "def plot_changes(result_metrics: dict, param_name='', height=3):\n", + " fig, axes = plt.subplots(1, len(result_metrics), figsize=(len(result_metrics) * height, height * 1.2))\n", + " for i, (param_val, values) in enumerate(result_metrics.items()):\n", + " n = len(values.accuracy)\n", + " earliness = np.round((1 - np.arange(n)/ n) * 100)\n", + " axes[i].plot(values.robustness, \n", + " values.accuracy, c='k', alpha=0.4)\n", + " scatter = axes[i].scatter(x=values.robustness, \n", + " y=values.accuracy,\n", + " c=earliness,\n", + " cmap='plasma'\n", + " )\n", + " axes[i].set_xlim((-0.05, 1.05))\n", + " axes[i].set_ylim((-0.05, 1.05))\n", + " axes[i].set_xticks(np.linspace(0, 1, 6))\n", + " axes[i].set_yticks(np.linspace(0, 1, 6))\n", + " axes[i].grid('all')\n", + " axes[i].set_title(f'{param_name} = {param_val}')\n", + " if i == 0: \n", + " legend1 = axes[i].legend(*scatter.legend_elements(alpha=0.6),\n", + " loc='best',\n", + " title=\"earliness, %\",\n", + " ncols=2,\n", + " borderaxespad=0\n", + " )\n", + " axes[i].add_artist(legend1)\n", + " \n", + " fig.supylabel('accuracy')\n", + " fig.supxlabel('robustness')\n", + " fig.tight_layout()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Probability Thresholding" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Probability Thresholding executes the simpliest mode of prediction:\n", + "\n", + "Firstly, the number of matching consecutive predictions is avaluated. If number of classifiers predicted the same label exceeds the specified *consecutive_predictions* parameter, the classification is done confidently. \n", + "\n", + "Otherwise the predicted probability is compared to the *probability_threshold*. And if it is not below it, the prediction is accepted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot_ind.core.models.early_tc import prob_threshold as PROB_THR\n", + "\n", + "cons_preds = [1, 3, 5, 7]\n", + "r_pthr = eval_param_influence(PROB_THR.ProbabilityThresholdClassifier,\n", + " 'consecutive_predictions', cons_preds, probability_threshold=0.8,\n", + " prediction_mode='all', interval_percentage=INTEVAL_PERCENTAGE) " + ] + }, + { + "cell_type": "code", + "execution_count": 199, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABWAAAAGkCAYAAAC/wYxsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACmbUlEQVR4nOzdd3gVZfr/8ffMSXLSG0lIgNB7L6EpIFVsKCp2xVVc10Ws665iQ3+KuOtXxbru2hU7FlwrTTpSBBSlQ+iEJJBG+jkzvz8ikZiAaScnJ/m8ritXcmaembmfOXDn5J6Z5zFs27YRERERERERERERkVpnejsAERERERERERERkYZKBVgRERERERERERERD1EBVkRERERERERERMRDVIAVERERERERERER8RAVYEVEREREREREREQ8RAVYEREREREREREREQ9RAVZERERERERERETEQ1SAFREREREREREREfEQFWBFREREREREREREPEQFWBEfYBgGDz30kLfDEBHxWcqjIiI1ozwqIlIzyqONmwqwIlJt06dP5/zzz6dp06b6ZSIiUkUHDx7k6quvplOnToSFhREZGcmAAQN48803sW3b2+GJiNR7u3fvxjCMCr/ef/99b4cnIlLvPfTQQyfNo4ZhsHz5cm+H2GD4eTsAEfFd999/P/Hx8fTp04dvv/3W2+GIiPiU9PR09u/fz4QJE2jZsiXFxcXMmzePP/3pT2zdupXHHnvM2yGKiPiEK664gnPOOafMssGDB3spGhER33HRRRfRvn37csvvvfdejh07Rv/+/b0QVcOkAqyIB+Xl5REcHOztMDwmOTmZ1q1bk56eTmxsrLfDEZEGqCHn0Z49e7Jo0aIyy6ZMmcK4ceN49tlneeSRR3A4HN4JTkQajIacR4/r27cvV199tbfDEJEGqiHn0Z49e9KzZ88yy/bt28f+/fu54YYbCAgI8FJkDY+GIJCTOnDgAJMmTaJZs2Y4nU7atGnDX//6V4qKikrb7Nq1i0suuYTo6GiCg4MZNGgQX375ZZn9LFq0CMMw+PDDD5k+fTotWrQgMDCQUaNGsWPHjjJtt2/fzsUXX0x8fDyBgYG0aNGCyy+/nKysrDLtZs2aRb9+/QgKCiI6OprLL7+cffv2levDqlWrOOecc4iKiiIkJISePXvyzDPPVPocHI/9gw8+4N577yU+Pp6QkBDOP//8cscbPnw43bt354cffmDYsGEEBwdz7733AlBYWMi0adNo3749TqeTxMRE/vGPf1BYWFhmH4WFhdxxxx3ExsYSFhbG+eefz/79+8vFlZOTw+23307r1q1xOp3ExcUxZswY1q1bV+m+1YbWrVvX6fFEfI3yqPJodbRu3Zq8vLwy/05EGivlUeXRysrNzVXeFKmA8qjyaFW999572LbNVVdd5dU4GhrdASsVOnjwIAMGDCAzM5Mbb7yRzp07c+DAAWbPnk1eXh4BAQEcPnyY0047jby8PG699VaaNGnCm2++yfnnn8/s2bO58MILy+zz8ccfxzRN7rrrLrKysvjXv/7FVVddxapVqwAoKipi7NixFBYWcssttxAfH8+BAwf44osvyMzMJCIiAigZd/SBBx7g0ksv5YYbbiAtLY3nnnuOYcOGsX79eiIjIwGYN28e5513HgkJCdx2223Ex8ezefNmvvjiC2677bYqnY/p06djGAZ33303qampzJw5k9GjR7NhwwaCgoJK2x05coSzzz6byy+/nKuvvpqmTZtiWRbnn38+y5Yt48Ybb6RLly5s3LiRp59+mm3btvHZZ5+Vbn/DDTcwa9YsrrzySk477TQWLlzIueeeWy6em266idmzZzNlyhS6du3KkSNHWLZsGZs3b6Zv374n7UdxcXG5X3onEx0djWnqGo1IdSmPlqU8enL5+fnk5uZy7NgxFi9ezOuvv87gwYPLnBeRxkh5tCzl0ZN7+OGH+fvf/45hGPTr14/p06dz5plnVuoYIg2Z8mhZyqOV884775CYmMiwYcOqtJ38AVukAhMnTrRN07TXrFlTbp1lWbZt2/btt99uA/bSpUtL1+Xk5Nht2rSxW7dubbvdbtu2bfu7776zAbtLly52YWFhadtnnnnGBuyNGzfatm3b69evtwH7o48+Omlcu3fvth0Ohz19+vQyyzdu3Gj7+fmVLne5XHabNm3sVq1a2RkZGRXGXxnHY2/evLmdnZ1duvzDDz+0AfuZZ54pXXbGGWfYgP3SSy+V2cfbb79tm6ZZ5jzZtm2/9NJLNmAvX77ctm3b3rBhgw3YkydPLtPuyiuvtAF72rRppcsiIiLsm2++udL9+H1/KvOVnJxc6f2mpaWVi1GksVMetcvErjx6cjNmzCiz3ahRo+y9e/dWOTaRhkZ51C4Tu/JoeXv27LHPPPNM+9///rf9+eef2zNnzrRbtmxpm6Zpf/HFF1WOTaShUR61y8SuPPrHfv75Zxuw//GPf1Q5Ljk13QEr5ViWxWeffca4ceNISkoqt94wDAC++uorBgwYwJAhQ0rXhYaGcuONNzJ16lQ2bdpE9+7dS9ddd911ZcYPGTp0KFDyuEP37t1Lr4R9++23nHPOORWOsfLJJ59gWRaXXnop6enppcvj4+Pp0KED3333Hffeey/r168nOTmZp59+uvTK2e/jr4qJEycSFhZW+nrChAkkJCTw1Vdfceutt5YudzqdXHfddWW2/eijj+jSpQudO3cuE/PIkSMB+O677zjttNP46quvAMrsD+D222/n3XffLbMsMjKSVatWcfDgQZo1a1bpfvTq1Yt58+ZVqm18fHyl9ysiZSmPlqc8enJXXHEFSUlJpKWl8cUXX3D48GHy8/MrHZNIQ6Q8Wp7yaHktW7YsNxHsNddcQ9euXfnb3/5W4R1nIo2F8mh5yqN/7J133gHQ8AMeoAKslJOWlkZ2dnaZJFuRPXv2MHDgwHLLu3TpUrr+xH20bNmyTLuoqCgAMjIyAGjTpg133nknTz31FO+88w5Dhw7l/PPP5+qrry5N4tu3b8e2bTp06FBhTP7+/gDs3LkT4A/7UFm/P55hGLRv357du3eXWd68efNyg1Rv376dzZs3n3SSqtTUVKDkfJmmSbt27cqs79SpU7lt/vWvf3HttdeSmJhIv379OOecc5g4cSJt27Y9ZT+ioqIYPXr0KduISM0pj5anPHpyrVq1olWrVkBJMfbGG29k9OjRbN26VcMQSKOlPFqe8mjlREdHc9111/H444+zf/9+WrRo4bFjidRnyqPlKY+emm3bvPvuu3Tv3r3cxFxScyrASp052UzOtm2X/vzkk0/ypz/9iTlz5jB37lxuvfVWZsyYwffff0+LFi2wLAvDMPj6668r3F9oaKjH4q+Miv5QtiyLHj168NRTT1W4TWJiYpWPc+mllzJ06FA+/fRT5s6dyxNPPME///lPPvnkE84+++yTbldUVMTRo0crdYzY2FjNvi1SzyiPNo48OmHCBF5++WWWLFnC2LFjq7y9iJyc8mjjyKPH+3P06FEVYEVqmfJow82jy5cvZ8+ePcyYMaNS7aVqVICVcmJjYwkPD+fnn38+ZbtWrVqxdevWcsu3bNlSur46evToQY8ePbj//vtZsWIFp59+Oi+99BKPPvoo7dq1w7Zt2rRpQ8eOHU+6j+NXm37++edauTK0ffv2Mq9t22bHjh2VuirUrl07fvzxR0aNGnXKxyRatWqFZVns3LmzzNWxis4xQEJCApMnT2by5MmkpqbSt29fpk+ffspEvWLFCkaMGPGHMQMkJyfTunXrSrUVkbKUR8tTHq2848MPVHZyBZGGSHm0POXRytu1axfASe9UE2kMlEfLUx49tXfeeQfDMLjyyisr1V6qRlOcSzmmaTJ+/Hj+97//sXbt2nLrj1/ZOuecc1i9ejUrV64sXZebm8t///tfWrduTdeuXat03OzsbFwuV5llPXr0wDRNCgsLAbjoootwOBw8/PDDZa6wHY/ryJEjAPTt25c2bdowc+ZMMjMzK4y/Kt566y1ycnJKX8+ePZtDhw6dMiked+mll3LgwAFefvnlcuuOz3wNlO7r2WefLdNm5syZZV673e5yf5THxcXRrFmz0vN0MsfHiqnMl8aAFak+5dHylEfLS0tLq3D5q6++imEYp5z9VqShUx4tT3m0vIry6IEDB3jttdfo2bMnCQkJp9xepCFTHi1PefTkiouL+eijjxgyZEi5YSakdugOWKnQY489xty5cznjjDO48cYb6dKlC4cOHeKjjz5i2bJlREZGcs899/Dee+9x9tlnc+uttxIdHc2bb75JcnIyH3/8MaZZtfr+woULmTJlCpdccgkdO3bE5XLx9ttv43A4uPjii4GSq06PPvooU6dOZffu3YwfP56wsDCSk5P59NNPufHGG7nrrrswTZN///vfjBs3jt69e3PdddeRkJDAli1b+OWXX8oN1v9HoqOjGTJkCNdddx2HDx9m5syZtG/fnj//+c9/uO0111zDhx9+yE033cR3333H6aefjtvtZsuWLXz44Yd8++23JCUl0bt3b6644gpefPFFsrKyOO2001iwYAE7duwos7+cnBxatGjBhAkT6NWrF6GhocyfP581a9bw5JNPnjKW2h4r5u2332bPnj3k5eUBsGTJEh599NHSflf3aqlIQ6A8WpbyaHnTp09n+fLlnHXWWbRs2ZKjR4/y8ccfs2bNGm655Rbat29fK8cR8VXKo2Upj5b3j3/8g507dzJq1CiaNWvG7t27+c9//kNubi7PPPNMrRxDxJcpj5alPHpy3377LUeOHNHkW55ki5zEnj177IkTJ9qxsbG20+m027Zta9988812YWFhaZudO3faEyZMsCMjI+3AwEB7wIAB9hdffFFmP999950N2B999FGZ5cnJyTZgv/7667Zt2/auXbvs66+/3m7Xrp0dGBhoR0dH2yNGjLDnz59fLraPP/7YHjJkiB0SEmKHhITYnTt3tm+++WZ769atZdotW7bMHjNmjB0WFmaHhITYPXv2tJ977rlKn4Pjsb/33nv21KlT7bi4ODsoKMg+99xz7T179pRpe8YZZ9jdunWrcD9FRUX2P//5T7tbt2620+m0o6Ki7H79+tkPP/ywnZWVVdouPz/fvvXWW+0mTZrYISEh9rhx4+x9+/bZgD1t2jTbtm27sLDQ/vvf/2736tWrtF+9evWyX3zxxUr3q7acccYZNlDh13fffVfn8YjUN8qjyqOnMnfuXPu8886zmzVrZvv7+9thYWH26aefbr/++uu2ZVl1GotIfaU8qjx6Ku+++649bNgwOzY21vbz87NjYmLsCy+80P7hhx/qNA6R+kx5VHm0Mi6//HLb39/fPnLkiFeO3xgYtl2N+7ZFGolFixYxYsQIPvroIyZMmODtcEREfI7yqIhIzSiPiojUjPKo1AcaA1ZERERERERERETEQzQGrDRKRUVFHD169JRtIiIi6igaERHfozwqIlIzyqMiIjWjPCq+RAVYaZRWrFjBiBEjTtnm9ddfp3Xr1nUTkIiIj1EeFRGpGeVREZGaUR4VX6IxYKVRysjI4Icffjhlm27dupGQkFBHEYmI+BblURGRmlEeFRGpGeVR8SUqwIqIiIiIiIiIiIh4iCbhEhEREREREREREfGQRjkGrGVZHDx4kJCQEFwuFy6Xy9shSSPkdDrx82uU/wXlV7Ztk5OTQ7NmzTBN37oedjyPhoWFYRiGt8MRkUZKeVREpGaUR0VEaqayebRRVn8OHjzImDFjeOyxx0hMTPS5XzTSMBQVFfH4448zZ84cb4ciXrZv3z5atGjh7TCq5ODBgyQmJno7DBERQHlURKSmlEdFRGrmj/JooxwDNiMjg02bNtGkSRPi4uIICAio1BUzt9uNw+GogwjrjvrkHbZtk5aWRlZWFi1btjzlnbDFxcXMnTuXM888E39//zqM0nMaWp+q25/s7GwSExPJzMwkIiLCgxHWvqysLCIjI9m3bx/h4eGV2qahve+gPvkK9an+Ux5VHlWf6jf1qf5THlUeVZ/qN/Wp/vN0Hm2Ud8AWFBQQFBRE8+bNCQsLq9Q2tm2XFvYayuMN6pN3GYZBTk4ODofjlB8YiouLCQ4OJjw8vEEkNWh4fappf+r7v9WKHI85PDy8Sh94G9L7DuqTr1Cf6j/lUeVR9al+U5/qP+VR5VH1qX5Tn+o/T+fRRv3svYYeEG/yxQ85IiIiIiIiIiJSNapAioiIiIiIiIiIiHiICrAiIiIiIiIiIiIiHqICbCPUvHlzHnnkkdLXpmkya9YsL0YkIiIiIiIiIiLSMKkAK+zZs4eLL77Y22FU2rRp04iOjiY6OpqHHnqozLpFixbRq1cviouLvROciIiIiIiIiIjICfy8HYDUnYKCAgIDA8stT0xM9JkJoVatWsU///lPPvroI2zb5tJLL+Wcc85hwIABFBcXM3nyZF588cUGMQOfiIiIiIiIiIj4Pt0BW0+53W7uvfdemjdvTmBgIJ06deKNN94oXe9yubjssstK17dp04ZHH320zD4mTJjAmDFjuOeee4iLi6Ndu3YVHuvEIQi2bt2KYRi89dZbDBw4sPTYCxYsKLPN3LlzSUpKIjAwkPj4eK677jqys7NL1//zn/+kVatWOJ1OmjRpwllnnVW67o033qBjx44EBgYSGRnJaaedVmbbU/nll1/o1KkT48aN4/zzz6djx4788ssvQMmdsYMHD2bo0KGV2peIiIiIiIiIiIineb0Au2TJEsaNG0ezZs0wDIPPPvvsD7dZtGgRffv2xel00r59+zKFyYbivvvu4/333+e5555j/fr1TJkyhb/85S98/fXXQEmBtlmzZrz//vts2LCBu+++m+nTp/Paa6+V2c+KFSvYtm0bX3/9NZ9//nmlj//QQw9x5513smbNGtq2bcvEiRNLH+vftGkT48eP54ILLmDt2rXMmjWLVatWcf311wOwdOlS7rvvPu677z42btzI//73v9Ki6J49e7jhhhu4+uqr+fHHH5k7dy4XXHABtm1XKq4+ffqwe/dutm/fzrZt29i9eze9e/dm06ZNvPvuuzz55JOV7qNIQ6E8KiJSM8qjIiI1ozwqInJqXi/A5ubm0qtXL1544YVKtU9OTubcc89lxIgRbNiwgdtvv50bbriBb7/91sOR1p38/HyeeeYZXn75ZS666CK6dOnCLbfcwvjx4/n3v/8NgNPp5Omnn2bo0KF07tyZm266iUsvvZQPP/ywzL6CgoJ499136devH/369at0DLfeeiuXXXYZPXr0YPr06Rw8eLD0TtOHH36Y8ePH88ADD9C9e3dGjx7NzJkz+fTTT8nLyyM5OZmgoCAuvfRSOnbsyGmnncZ9990HwP79+3G73Vx++eV06tSJAQMGcPfddxMREVGpuPr06cP999/PmWeeydixY3nggQfo06cPN9xwA48++ihz5syhS5cudO3alW+++abS/RXxZcqjIiI1ozwqIlIzyqMiIqfm9TFgzz77bM4+++xKt3/ppZdo06ZN6Z2OXbp0YdmyZTz99NOMHTvWU2HWqU2bNlFQUMC4cePKLC8uLqZLly6lrx9//HHefvttDh48SGFhYbn1AJ06dapw3Nc/0rdv39KfExMTAUhJSQFKhgHYtm1bmauatm1jWRZbt27l/PPP59FHH6VNmzYMHz6csWPHctVVVxEWFsbAgQMZPHgwffv2ZejQoYwZM4ZrrrmG2NjYSsf297//nb///e+lr59//nlCQ0MZPnx46b+Hffv2MXHiRPbs2UNQUFCV+y/iS5RHRURqRnlURKRmlEdFRE7N6wXYqlq5ciWjR48us2zs2LHcfvvtJ92msLCQwsLC0td5eXmlP1f20fcTVWebqjg+Hurs2bNp1apVmXVOpxPbtnnllVd46KGHePjhhxk6dCgRERFMnz6ddevWlYkvODi40vGe2M7f37/09fEJutxuN7Ztk5eXx1VXXcVdd91Vbh/t2rXD6XTy888/8/XXX/P1118zffp0ZsyYwQ8//ECTJk1YtmwZCxYs4KuvvuKll17i0UcfZcWKFXTq1KlqJ4qSovDjjz/O4sWLWbJkCa1bt6Zbt2707NmT4uJiNm7cSP/+/au837pw/Py6XK7S4R0qcnzdqdr4mobWp+r2x1v9r408ejxPFRcXV7ofDe19B/XJV6hP9Z/yqPKo+lS/qU/1n/Ko8qj6VL+pT/Wfp/OozxVgU1JSaNq0aZllTZs2JTs7m/z8/ArvdpwxYwYPP/xw6evTTjuN5557DrfbjdvtrtLxq9q+Onr27ElAQADJycmceeaZFcawbNky+vTpw9/+9rfS5bt37y4To23b2LZdYcy/X378tWVZAFiWVbr+xP253W569OjB1q1bT1owdbvdmKbJueeey7nnnsuMGTOIjY3lyy+/5KqrrgJgxIgRjBgxgscff5yWLVvy/vvvc//991f1VPHXv/6Vv/71r7Rq1Yrly5fjcrlKY3C73RQXF9fJe1YdlmVh2zZr1qwp80HiZObNm1cHUdWthtanqvbnxItBdak28uhxc+fOJTg4uErHb2jvO6hPvkJ9qv+URyunob3voD75CvWp/lMerZyG9r6D+uQr1Kf6z1N51OcKsNUxdepU7rzzztLXaWlpZGVl4XA4cDgcld6P2+2uUvvqio6O5qabbuL+++/Htm1GjhxJRkYGixYtIiIigptvvpmOHTvyySef8Pnnn9O+fXtee+01Nm7cSPPmzUtjNAwDwzAqjPn48uPFyeOvTbNkWGDTNEu3O3F/DoeDe++9lzPOOINJkyZx0003ERYWVjqh1htvvMEHH3zAzp07GTlyJE2aNOGzzz7Dtm26devG0qVLmTt3Lueccw7x8fEsW7aMjIwMunXrVuVzO2fOHHbt2sXHH3+MaZqcfvrpJCcn88knn7B3714cDge9evWqk/esOkzTxDAM+vfvT3h4+EnbFRcXM2/ePMaMGYO/v38dRug5Da1P1e3P8av2vuD3eTQ7O5vExETOPPPMU/77PVFDe99BffIV6lP9pzyqPKo+1W/qU/2nPKo8qj7Vb+pT/efpPOpzBdj4+HgOHz5cZtnhw4cJDw8/6VifTqcTp9NZ+jo3N5esrCzgt8fr/8iJj+dXdpuaePrpp4mNjeXJJ5/kjjvuICwsjG7dunHfffdhGAZ33nkn69ev59prr8UwDC644AKuvfZa5s+fXy6+ysb7R9sdL+gOHDiQuXPncu+99zJ69Ghs2yYxMZGLLroIwzCIjo7mqaee4l//+heFhYW0atWKl19+maSkJNavX8+yZcv4z3/+Q25uLs2aNePhhx/mkksuAeCrr77i3HPPZcuWLacckiA3N5c77riDd999t7TA2q5dO2bMmMHkyZMJCAjgpZdeIjQ0tFJ994bj59fPz69S/7n9/f0bRFI7UUPrU1X7462+10YePa4672FDe99BffIV6lP9pzxaOQ3tfQf1yVeoT/Wf8mjlNLT3HdQnX6E+1X+eyqM+V4AdPHgwX331VZll8+bNY/DgwV6KyDNM0+T+++8/6WP5QUFBzJ49+5T7ONn6AwcOAL8VlS3LKi0GdurUqdyYsTExMeWWDRs2jGXLllW4/zPPPLPCoRMA+vTpw9KlS08a844dO2jZsiWtW7c+aRuAkJAQkpOTyy2//fbbueWWW3A4HHVSKBfxRY0lj4qIeIryqIhIzSiPikhjY3o7gGPHjrFhwwY2bNgAQHJyMhs2bGDv3r1AyWMGEydOLG1/0003sWvXLv7xj3+wZcsWXnzxRT788EPuuOMOb4Qvtezrr7/moYceqvDKpohUTHlURKRmlEdFRGpGeVRE5NS8fgfs2rVrGTFiROnr42O6XHvttbzxxhscOnSoNGkDtGnThi+//JI77riDZ555hhYtWvDKK68wduzYOo9dat/XX3/t7RBEfI7yqIhIzSiPiojUjPKoiMipeb0AO3z48HKPt5/ojTfeqHCb9evXezAqERHfoTwqIlIzyqMiIjWjPCoicmpeH4JAREREREREREREpKFSAVZERERERERERETEQ1SAFREREREREREREfEQFWBFREREREREREREPEQFWBEREREREREREREPUQFWRERERERERERExEP8vB2AL3O7LbZsTiczo4DIqEA6d4nB4VBNuyYst5ui/SlYuXmYIcEEtIjHdDi8HZaIiIiIiIiIiEi1qFpYTau+38+tN3/DHbfO5d57FnLHrXO59eZvWPX9fo8e95tvvmHkyJHExcVhGAazZs0q1+bxxx+nefPmOJ1OevbsyeLFiz0aU23J27aLwy+/x+HXPiB11qccfu0DDr/8Hnnbdnn82MnJyYwfP57IyEgCAwPp2LEjS5cuLV3vq+dURERERERERES8SwXYalj1/X4ee2QZG386TGSkkzZtIomMdLLxp8M89sgyjxZhjx07Ro8ePXjyyScrXP/qq6/y4IMPcvfdd7Ny5Uq6devGuHHjOHDggMdiqg1523Zx5MMvKdxzAEdIMP5Nm+AICaZwzwGOfPilR4uwaWlpDBkyBD8/P+bMmcOGDRv45z//SZMmTQDfPaciIiIiIiIiIuJ9KsBWkWXZvPXGT2Rk5NO+fTRh4U4cfiZh4U7at48mIyOft9/4Cbfb8sjxJ0yYwDPPPMM111xT4fpnnnmGK664gltvvZW+ffsya9YsAgMDeeGFFzwST22w3G6yFq7AnZuHf0IsZpATwzQxg5z4J8Tizs0j67uVWG63R47/0EMPkZCQwOzZsznjjDPo3LkzF154IV27dgV885yKiIiIiIiIiEj9oAJsFW3ZnMb2bUeJjw/FMI0y6wzTID4+lG3bjrJlc3qdx1ZQUMCmTZsYM2ZM6TKHw8HQoUNZvXp1ncdTWUX7Uyg6lIpfVASG8btzahj4RUVQdPAwRftTPHL8b775ht69e3P22WcTHR1Nly5deOqppwDfPaciIiIiIiIiIlI/qABbRZkZhRQWuggO9q9wfVCwP4WFLjIzCuo4MkhJScHtdpOQkFBmeVxcHKmpqXUeT2VZuXnYxS4MZ8Xn1Ajwxy52YeXmeeT4+/fv5+2336Zdu3Z88cUX3HDDDdx77708//zzPntORURERERERESkfvDzdgC+JjLKidPpR15eMWHhznLr8/OKcTr9iIwK9EJ0vskMCcbw98MuLMYIKn9O7aJiDH8/zJBgjxzfsiy6d+/O888/D8Bpp53Gzz//zCuvvMJ5553nkWOKiIiIiIiIiEjjoDtgq6hzl1g6dIwmJeUYtmWXWWdbNikpx+jYMZrOXWLqPLb4+HgcDgeHDh0qszw1NZW4uLg6j6eyAlrEE5AQhysjC9v+3Tm1bVwZWQQ0a0pAi3iPHD82NpaOHTuWWdalSxcOHjzos+dURERERERERETqBxVgq8g0DSb+qSdRUUHs2HGUnOxCXC6LnOxCduw4SlRUENf8qScOR92f2sDAQLp27cr8+fNLl7ndbpYtW8aAAQPqPJ7KMh0OIkaehiMkmOJDaVj5hdhuCyu/kOJDaThCQ4gYMRjT4fDI8ZOSkti5c2eZZdu2baN58+Y+e05FRERERERERKR+0BAE1TBwUAvufWAIb73xE9u3HeXw4VycTj969mzKNX/qycBBLTx27KysLDZt2lT6eteuXaxcuZKYmBg6dOjAbbfdxuTJk0lKSuL000/niSeeID8/n8mTJ3ssptoQ3LEtXHouWQtXUHQoFTvTheHvh7N1CyJGDC5Z7yF33XUXI0eOZOrUqVx99dUsW7aMd955h6effhrAZ8+piIiIiIiIiIh4nwqw1TRwUAuS+jdjy+Z0MjMKiIwKpHOXGI/f+bp8+XLOPffc0tfTpk1j2rRpXHzxxcyePZtJkyaRmprKY489Rnp6Op07d2bOnDm0aOG5onBtCe7YlsB2rSjan4KVm4cZEkxAi3iP3fl63LBhw5g1axYPPvggTz31FM2bN2f69OncdNNNAD59TkVERERERERExLtUgK0Bh8OkW/e6HQf0nHPOKTdO6u9NnTqVqVOn1lFEtct0OAhs1bzOj3v55Zdz+eWXn3S9L59TERERERERERHxHo0BKyIiIiIiIiIiIuIhKsCKiIiIiIiIiIiIeIgKsCIiIiIiIiIiIiIeogKsiIiIiIiIiIiIiIeoACsiIiIiIiIiIiLiISrAioiIiIiIiIiIiHiICrAiIiIiIiIiIiIiHqICrIiIiIiIiIiIiIiHqAArIiIiIiIiIiIi4iF+3g7Al7ndFns3ZXIso4jQqABado3E4VBNuyYstxsrdT92wTGMwFDMuBaYDoe3wxIREREREREREakWVQuradPKVJ65aQXP3vw9//n7Gp69+XueuWkFm1amevS49957L927dyckJITo6GjGjBnDTz/9VKbN448/TvPmzXE6nfTs2ZPFixd7NKbaUrxnK3mf/Ye8z18m76u3Sr5/9h+K92z16HGbN2+OYRjlviZOnFjaxlfPqYiIiIiIiIiIeJcKsNWwaWUqbz20np0bMgiLCiChbRhhUQHs3JDBWw+t92gRdunSpfzlL39hyZIlfPXVVxQXF3PWWWeRnZ0NwKuvvsqDDz7I3XffzcqVK+nWrRvjxo3jwIEDHoupNhTv2Ur+gg9wH9qNERiCGdUUIzAE96Hd5C/4wKNF2DVr1rB3797Sr88++wyAyy+/HPDdcyoiIiIiIiIiIt6nAmwVWZbNN69tI+doES06hhEc5o/DYRAc5k+LjmHkHC3im9e343ZbHjn+0qVLueWWW+jXrx+DBg3i3Xff5dChQ6xYsQKAZ555hiuuuIJbb72Vvn37MmvWLAIDA3nhhRc8Ek9tsNxuCtcuwM47htkkAcMZhGGaGM4gzCYJ2HnHKFy7EMvt9sjxmzVrRmJiYunX559/TmJiImeddRbgm+dURERERERERETqBxVgq2jPL5ns25pNk2ZBGIZRZp1hGDRpFsS+LVns3ZRZJ/FkZGQAEBMTQ0FBAZs2bWLMmDGl6x0OB0OHDmX16tV1Ek91WKn7sdIPYoZFVXhOzbAorPQDWKn7PR5LQUEBn3zyCVdddRWmafrsORURERERERERkfpBBdgqOpZZRHGhG2dwxfOXBQQ7KC50cyyjyOOxuN1upkyZQt++fUlKSiIlJQW3201CQkKZdnFxcaSmenZs2pqwC45hu4rB31lxA38ntqsYu+CYx2N59913ycnJ4S9/+QuAz55TERERERERERGpHyquIspJhUYG4O90UJjnIjjMv9z6ojw3/k4HoVEBHo/l2muvZdu2bSxdutTjx/IkIzAUw88figvBGVS+QXEhhp8/RmCox2N5/fXXGTZsGK1bt/b4sUREREREREREpOHTHbBV1KpbJImdwjlyMB/btsuss22bIwfzSewcQcuukR6N49prr2X+/PksWLCAtm3bAhAfH4/D4eDQoUNl2qamphIXF+fReGrCjGuBGdMMKyejwnNq5WRgxjTHjGvh0Ti2bdvGihUruOGGG0qX+eo5FRERERERERGR+kEF2CoyTYOzru9IWHQA+7flkJdTjMttkZdTzP5tOYQ3CeCs6zrgcHjm1FqWxbXXXsvXX3/N/Pnz6dy5c+m6wMBAunbtyvz580uXud1uli1bxoABAzwST20wHQ6cSaMwgkOxjhzCLszHtizswnysI4cwgkNxJo3EdDg8Gsd//vMfoqOjueSSS0qX+eo5FRERERERERGR+kFDEFRD18FxTHyoD9+8to19W7MpTikZdqBdn2jOuq4DXQd77s7Ia6+9ls8++4wPP/yQiIgI9u3bB0B0dDQhISHcdtttTJ48maSkJE4//XSeeOIJ8vPzmTx5ssdiqg3+rTrBqMsoXLsAK/0g9rFMDD9/HAltcCaNLFnvQW63m/fee49LL70Uf/+yQ0v46jkVERERERERERHvUwG2mroOjqPTgBj2bsrkWEYRoVEBtOwa6bE7X4+bNWsWAOecc06Z5c8++yy33HILkyZNIjU1lccee4z09HQ6d+7MnDlzaNHCs4/v1wb/Vp1wtGiPlbofu+AYRmBoyfAEHr7zFeDzzz/n0KFD3HTTTeXW+fI5FRERERERERER71IBtgYcDpM2PaLr9Ji/HyO1IlOnTmXq1Kl1EE3tMx0OzIRWdX7cCy+88JTn1pfPqYiIiIiIiIiIeI/GgBURERERERERERHxEBVgRURERERERERERDxEBVgRERERERERERERD1EBVkRERERERERERMRDVIAVERERERERERER8RAVYEVEREREREREREQ8RAVYEREREREREREREQ+pFwXYF154gdatWxMYGMjAgQNZvXr1KdvPnDmTTp06ERQURGJiInfccQcFBQV1FK2ISP2jPCoiUjPKoyIiNaM8KiJycl4vwH7wwQfceeedTJs2jXXr1tGrVy/Gjh1Lampqhe3fffdd7rnnHqZNm8bmzZt59dVX+eCDD7j33nvrOHIRkfpBeVREpGaUR0VEakZ5VETk1Py8HcBTTz3Fn//8Z6677joAXnrpJb788ktee+017rnnnnLtV6xYwemnn86VV14JQOvWrbniiitYtWrVSY9RWFhIYWFh6eu8vLzSn23brnLMx7exLJvDv2SRn1FEUFQATbtFYJpGlfdXH1TnPHiCbVuQmQxFORAQBpFtMIzqXSeoL306mePxuVwuiouLT9ru+LpTtfE1Da1P1e1PbfXfG3k0Ozu7tA+V7UdDe99BffIVja1Pc17ZxIszV7MtLQuADrER3HRLEhf9pXudxlgVyqPKo+pT/eYrfcrLK2LOp9v4cUMKDofJ8JGtGTmqNQ5H+b8nfKVPlaU8qjyqPtVvvtIn5VHP5VHD9mKVqqioiODgYGbPns348eNLl1977bVkZmYyZ86cctu8++67TJ48mblz5zJgwAB27drFueeeyzXXXHPSq2UPPfQQDz/8cOnr0047jeeee4727dsTHBxcrdj3fH+ENa/vJm3bMVyFbvycDmI7htL/uta0GtSkWvusjCeffJJXX32VgwcPAtC+fXvuvfdeLrrootI2TzzxBM8//zzp6el06tSJZ555hqFDh3osplqT9gvs/BKy9oJVDKY/RLSEdudCbDePHdbtdvOPf/yD2bNnc+TIEWJjY7n88st57LHHMIySgronzml+fj67du3i0KFDZT5ISOORl5fHlVdeSVZWFuHh4dXah7fy6In7qm4eFRGpKeVREZGaUR4VEamZyuZRr94Bm56ejtvtpmnTpmWWN23alC1btlS4zZVXXkl6ejpDhgzBtm1cLhc33XTTKR9VmDp1KnfeeWfp67S0NLKysnA4HDgcjkrH63a7cTgc7Fl5hHn/bzN5RwsJbxZMQLCDojw3B3/MZN7/28xZD3Wn1WDPFGFbtmzJo48+SteuXbEsi5dffpmrrrqKlStX0q9fP1599VWmTZvG//3f/zFkyBCeeOIJLrjgAjZv3kyzZs1O2idvs1M3Yv30OhRlQVAM+AWBKx8ytsNPr2P2vh4jrkel9lXVPj3wwAO89dZbvPTSS/Tu3ZuVK1dy8803ExkZyb333lvlc1pZpmliGAb9+/c/5X/S4uJi5s2bx5gxY/D396/28eqThtan6vbn+FX7mvBWHs3OziYxMZEzzzyz0h/WG9r7DuqTr2gsfVo4ewfX3Py/U2732sxzGXtFx7oIsUqUR5VH1af6rb73acf2o4wZOQuXy8Kyyt5f5HAYNGsWxqJlEwkM/C32433q23Ywr563gvyMImz3r9v++lBji35RTJozhICQ+tfn31MeVR5Vn+q3+t6nquZR27b55cv97DU38tM/bPwcDrqPb8GgP7clsoVvXhDxdB71+hAEVbVo0SIee+wxXnzxRQYOHMiOHTu47bbbeOSRR3jggQcq3MbpdOJ0Oktf5+bmkpVV8lje8Tsc/8iJww6sem0XeUcLie0YVrp9YJgfzo5hpG3LYdXru0gcGF3hLdo1dcUVV5R5/dxzz/HWW2+xbNkykpKSePbZZ7niiiu49dZbAZg1axYJCQm88MILPPbYYxX2CSp/HjzBstxY278sKb6GtfwtFv8QbL9gyNmLteNLzNhumOapC6vV6dP333/PmWeeyWWXXQZAp06deO+991izZg2GYVTpnFbF8fj8/Pwq9Z/b39+/Xibqmmhofapqf7zV99rIo8dV5z1saO87qE++oqH3aeajqynMs7B//fVn2TkYODGMAAAMG56dsZrzJnruqZKaUh6tnIb+b7mhUJ/qzksvric314XbVfK3gG27MYzf/m7Yvi2Tz+fs5Kqry9/Q8eH1P5B7sAjLXf7B0L0rMvn2gc1c9Fw/zwVfy5RHK6e+/luuCfXJN9TXPlUlj1qWzft/WsWGj/dw+ruB5OwvxJ0Pi5/Yzornd3HT3DNoNSjGW12pMU/lUa9OwhUTE4PD4eDw4cNllh8+fJj4+PgKt3nggQe45ppruOGGG+jRowcXXnghjz32GDNmzMCyLI/HfPiXLFK35hDeLLhcgc8wDMKbBZO6JYfDm2p+JfGPuFwuXnnlFfLz8xk2bBgFBQVs2rSJMWPGlLZxOBwMHTr0D2eg9KrMZMjeC0ExFZ5TgmJKhiXITPbI4QcNGsSyZcvYuHEjUFKQXbt2LWeddZbvnlNpNHwxj4qIZ2zae6S0+Grbxbjcmyl2b8Bllfz+tA3YvO+oFyOsn5RHRXzfpx9vOaFoYOGyfsFlJWPbLgBME/43Z1uF2x7elF2m+GrZ7tKfbbfN6teSKchpGOMbeoryqIjvq0oeXfnSDn6YtafcPmy3TXGei1fHLaW4wF1ufWPn1QJsQEAA/fr1Y8GCBaXLLMtiwYIFDB48uMJt8vLyMM2yYR9/3LwuhrPNzyjCVegmILjiOzEDgh24Ct3kZxR5LIbVq1cTHBxMYGAgd9xxB++88w59+/YlJSUFt9tNQkJCmfZxcXEnnX2yXijKAXdRybADFXEElqwvyvHI4R999FHGjx9Pr1698PPz47TTTuMvf/kLN910k++eU2k0fDGPiohnOMpcxPztj1fLSqPItRrbdv2ujYDyqEhDkJ//W4HUtrOw7QIsKw2X9ROWlYZlQW5uxX+fGY7f8mKavYdfWEyGnVK6rDjfzeFfsjwXfAOgPCri+yqbR23bZsnMbaVDtZS0/+1zp21B7pEifpq9ry7D9wleH4Lgzjvv5NprryUpKYkBAwYwc+ZMcnNzS2dPnDhxIs2bN2fGjBkAjBs3jqeeeoo+ffqUPqrwwAMPMG7cuDoZyzQoKgA/Z8mYr4Fh5U9fUV7JhFxBUQEei6Fnz56sWbOGjIwM3n//fW688UbatWtHdHS0x47pUQFh4AgoGfPVP6T8endByfqAMI8c/vXXX+fjjz/mv//9Lz179uSHH35g6tSpNGvWjPPOO88jxxSpTb6WR0XEMwb2aMZXPyRjG2AYThxmIm7rtw+/Lvc6unYa5sUI6y/lURHf1rlLDBt/SsWybEwzCj+jC25rN7adj8tOxs84Qrt2nSve+IRiXx7Z2FgcYAvBdjhOo2QcQ9PPq/ct+QTlURHfVtk8mne0iPQdx0q3S0tL4xf3DzjtCFrRA4fhh+lnkLw8nX5Xt/Zeh+ohrxdgL7vsMtLS0njwwQdJSUmhd+/efPPNN6UDeO/du7fMlbH7778fwzC4//77OXDgALGxsYwbN47p06fXSbxNu0UQ1ymMAxsycJ4wBiyUXKnLPphH8z5RNO1avRkkKyMwMJBu3UrGbxsyZAjr1q3j//7v/3jttddwOBwcOnSoTPvU1FTi4uI8Fk+NRbaB8JaQsQ3bL7jcOSU/HaI7lrTzgPvvv5/bb7+dG264AYABAwaQnJzMk08+yQ033OCb51QaFV/LoyLiGXc/PpRvztyN27bBANOIwzIOYv/6OK0B9BldyOzZs5kwYYJ3g61nlEdFfNtf/tqPyX/5qvS1aYRhmN2w7BQs6yAudzYxccmsXbuW3r174+f325/BdgVPu1u42MvPtLeTCGkSSELPiLrohk9THhXxbZXNo+t/XIdluzENB1lWKhs2bMWNi2McIZkNtLF7Yxr+GKaeuvo9rxdgAaZMmcKUKVMqXLdo0aIyr/38/Jg2bRrTpk2rg8jKM02DgZPa8s20n0nbVjIWbEBwyR2x2QfzCIl2MvD6th6ZgOtkLMuiqKiIwMBAunbtyvz587n66qsBcLvdLFu2jEmTJtVZPFVlmg7oeB7W+lchZy92UMyvww4UlBRfAyIwO5z3hxNwVVdBQUG5x1/8/Pywbdtnz6k0Pr6UR0XEM3oPbcb/3T+Sux5diGXbYDhKirD2IQwcXHRWe1p1iuLo0aP897//5bLLLiMiQkWF45RHRXzX5Vd254vPt/H1VztKb2g1DBN/R3Pc7iZcfnU4cU1DWLduHTt27GDIkCGlhcH2I+LY/m1auUm48skmxdjJ9XdchF+A7sisDOVREd9V2Tz6y/afSGuZjGt/IJnWIdpaDsKMJhwjizwy2cU62hT1ocNI3bD2e3qWohpaD47hrIe707x3FPkZRRzZdYz8jCKa94li7MPdaT3Yc7O9TZkyhW+++YatW7eyevVqpkyZwurVq0uLg7fddhvvvfcezz//POvXr+eaa64hPz+fyZMneyym2mDG9cDsMwmiOkLRMTh2sOR7dEfMPpMw48rPWFpbRo8ezZNPPskHH3zA1q1befvtt/n3v//NOeecA/juOZW6kZ9VxNpZuwFYO2s3+VmeG/9ZROSPTLqvP6u/n8SlwzrRJiyMtuHtSeoQzz13D+L/Xr6N0NDQ0rYffPABa9eu9WK0IiK1w8/PZNb7FzF9xkhaJP72JGKfvvHMeu8yXn7lDsaMGUNwcDDZ2dl89dVXLFy4kMLCQi59ZQAJPUouRhm//nUc5mgCQPCgXDpeHVrueCIiDc3J8mi39tHcPbw7F6R2JGRhKAW7C2k5Mpwj1gFsLKKjo2lj9qIt/fAjgAIzh8MxP9P2TB8dItOD6sUdsL6o9eAYEgdEc3hTNvkZRQRFBdC0a7jH73xNS0tj0qRJpKWlERoaSufOnfnkk08YP348AJMmTSI1NZXHHnuM9PR0OnfuzJw5c2jRooVH46oNZlwPiOkKmcklE24FhEFkG4/d+XrcK6+8wt/+9jfuuOMOjh49SmxsLBMnTuSf//wn4NvnVDzHtm0WP7WVrx/YiG1anP5OIJ/eso7Pbv6Rc6b34Iw7Onk7RBFppDr0iuHluReWvl68eDFbt25l48aNXHnllWzdupXFixcDsG7dOtatW8f1119f5pFcERFf4+dnMuW2Adx8a3+OHsnHz98kIiKwdH2bNm1o0aIFa9as4ZdffmHXrl1s3ryZTp06cev3o/nh9V288a8dFGf6061VW7pcNYic0MMsWbqECU0nEBgYeIqji4j4vhPzaOrBHJbfsYKMJYcw0nM47M7GcBjkzD/GsdYpdBzdlB1LDpOdnc0x+yCRNKet0ZeDoRvpd3MCX339Jeeddx7BwcHe7la9oU/aNeBwmDTrEVmnx/zggw/+sM3UqVOZOnVqHURT+0zTAdHt6/SYkZGRvPrqq6ds48vnVDxj+Ys7+N/ffwTAEfTrQhtcBW4+/9sG/IMcnHZT3f5bFhGpSM+ePdm6dSvJyclkZ2fTqVMnWrZsydtvv13a5rXXXuPcc8+lefPmXoxURKTmDMOgSUzFf/D7+/tz2mmn0bFjR7777jtcLhfLli3j60e+JmhZMMHFGURY+fjvPUzBcwEUXFCA0c1g0aJFnHXWWXXcExER7zAMgx3P/ULGspK5cOxfh2hJKUxhe9E22AHth8bRfdogNu7/gZyEPZjOY5x/9VgGXH0+C5bNJTMzk88//5zzzjuvzBNYjZmGIBARqSJXoZtvHtxY+tq27ZIJ407wzYMbcRW56zo0EZFyoqKiaNmyJQAbN5bkrqCgIG688UbatWtX2u7LL7/km2++8UqMIiJ1KSYmhgsuuIAuXbpwcF4Km+ZtZl3eOtKK07DdJbNy2QU2vGeS8UsGe/fu5eeff/Zy1CIidaPgaAHb3tkGJ0xSmOJKYUvRZmxs4s14mq1rzlV/uoQuXbpw1n29GHBHAofjNpJ8cAdnn302YWFhZGdn8/nnn5Odne29ztQjKsCKiFTR9oWp5GcUAyXF15/cC5g7d26ZImxuehE7F6d5K0QRkTJ69uwJwNatWykoKChdPmrUKM4///zS13v37uW///0veXl5dR6jiEhdMgyDli1bkvhjC+LMOGx++xyX7k4HINQRSsBSJ7Zt8/3333PkyBFvhSsiUmcOrz6MVfxb9TXNncbWoi0AJPg1o4N/R3BD6ppUWrZsyYQJE2jXrh22bbNx40a+/vprevToQUREBMeOHePzzz8nMzPTS72pP1SAFRGporyjhaU/uyiu8OeSdpqQS0Tqh2bNmhETE4PL5WLTpk1l1sXHxzNp0iQMwyhdNmvWLH755Ze6DlNEpM4FEEAXZ1d6OHuWLst0Z/Jz4UYK3PmEH4ygiaMJlmWxYMECXC6XF6MVEfE8y/XbBalCu5BtRdsAaObXjI4BHUs/M1pWSbvg4GBGjRrFOeecQ3h4OLm5uaxYsYKAgAD8/PzIy8vjf//7H0ePHq37ztQjKsCKiFRRTLvKjWHTpG2IhyMREam843fB/vLLL7jdZYdIcTgc/PnPf2bgwIGly5YvX84bb7yBZf12B0TG4Xx+XnaYHeuPlH7oFhHxZYZZUkiIdkQTZP42duwR9xHWFq5lb/Fe+nXqR3BwMJmZmaxYscJboYqI1ImYXjHw63X5HUXbcdnFhJqhtPMvO8dJTI8mZV63aNGCCRMm0LdvX0zTJC0trfSiVX5+Pv/73/9IT0+vkz7UR5qES0SkiloObEJspzDSt+dABcO8GibEdQ4nMSm67oMTETmJtm3bsuTrJWz6cjPP/PNZ4qymRHWOouufOtP6vNYYpkGvXr1o27Yt7733HgBFRUW88sorDB14Jh/PSGb5p3uxfp2IITYxmKsf6M1Z13f0ZrdERGrk+OQyWe4s8q08TMOkZ0Avkl3JZLkzSXbtYtmW5XTr15W1a9eyesUGlr97lE3ziinIddG6WxTj/tqZIRe1wjSNPziaiEj9F5YYSuKYRH749gfS3ekYmHQK6IxplNzDaTgMmg1NIKxVGPzugSk/Pz+SkpJo3749S5cu5dChQ6XrCgsL+eKLL+jf8wxWfniExR8lN6o8qjtgRUSqyDAMLvlPEqbDwHCU/QVhOMD0M5nwUlKZx3lFRLwtfcMRDv4rhYOLD7Jt/zYKjhRweNVhFt64iEWTF2P9OvFMWFgYN954I/Hx8QDkZhXxp5GP8b9P5pUWXwHS9uXx9I0r+OCfP3mlPyIitSEgygkGHHIfBCDWEUeEI4Lezt50DupC0+5NybfzWLt2LYd35zLr//3Iu698wcHdR8hOL+TnZYeZfvki/nnNEtxu6w+OJiLiG/o90of9YfvAhFb+rQg1f30K1ITQ5iEMnTn0lNtHRkYybtw4hg8fTmBgYOnyvVuPcPWQGbzz1HJS9+Q2qjyqAqyISDW0GxbHzYtH0mpQ2btc25wey5QlI2k7JNZLkYmIlOcudDPvmvnEueJw2A7yrDyOWkdL7/zaNSeZza9vKbPN+eefz9ixY1n91X5ys4s55jrIfmsRll12vOvXH1jPkYOatEtEfNNpMwbjwkWaVTJ5ajO/ZkDJHV6JUYn8/dW76Ny5M65iizkvbMJVZGFZLo6yGdu2Sy9MLfowmf+9uOWkxxER8SU/bP2Bbrd0pfeEXnRt1xW/IAdhLUPpd3dfLph7PiHxwX+8E6Bjx45ceumlpXn08xe3UFxUTJrrRwrsDIBGk0dVgBURqaZWg2L467wRjHmwOwB3/DCGmxeNpOWAJn+wpYhI3dr95W4KjhTgsB3EOxIA2O/a91sDG37+zy/YdtlxXZsltCBnTafSQi3AQXs5+fZvM4EbwPy3d3g0fhERT0kc1YK2M9oQ0jKEUDOMcDMcw2HQ6qyWXPDNOOI6xzFs2DDizD4UHwvA/nX86yI7ixz2/LYjGz6ZualcHhUR8TU7duxg9+7dBIQE8Kd//YnLV1/KtckTuXT1JfS+rRfOSGeV9hcYGFguj9pYpNsby3ymbOh5VGPAiojUUEh0AByAyOaVuwooIlLXDq9OxfQzsVwWzf2ac8B1gEx3JrlWLiFmyYSBx/YdoyC9gKDYoNLtso8UUlxg0MIcTqa9k2N2SdH2iL2RFsZwoGTc65Tdx+q6SyIitcK2bY4EptNrSk/6depH65g2hCQEExgdWKZdyiabZv5JZBbvI8fejY1Ftr2HYBLwM0qKEYf3HCMztYCopkEVHUpEpN7Ly8tj+fLlAPTt25fo6Nqb16SiPHrE/pl4BuBnlOTNhpxHdQdsDVhuiyObjnJo5SGObDpaOnaaVJ9lubHytmJlryn5blUww5FIPZGfX8zbb/7I88+uBmDC+A958/UfKShweTkyEZGyDIeBTcndBIFmIIFmSWGh2C4q2+53Ex+EhPuXToYQabSjqZEEQLjRuky7sOiq3QkhIlJfHDhwgOzsbAICAuh1ei+adIsuV3wFSsb+N0zCjZY0NfoTaMTgZwTz+xH/TYfmABAR37Vs2TIKCwuJiYmhd+/etbrvivKog0CM35UmG2oeVQG2mlLWpLL8rhWsuGclqx9aw4p7VrL8rhWkrEmtsxjuvfdeDMNg0qRJZZY//vjjNG/eHKfTSc+ePVm8eHGdxVQTVs562P0A7HkI9j1e8n33AyXLPSgzM5NJkybRrFkzAgMD6dOnD0uWLCnTxlfPqXjO0aP5jDrjbe66Yx779mYD8OOPh7ll8teMHv42mZkFXo5QROQ3zYYmYLt+e5yryC4EIODXu7YwILJjJM7fFVIDQ/wZfH5i6QdhfyOUFubwMgVYt8tmxOVtPdsBEREP2bx5M1AyTqGf38kfEO09MgF3cUke9TOCiDG6E2/0x/FrHjUMaNk1gvAmuiAlIr7p+NADpmkyfPhwTLN2S4YV5dEEc2CjyaMqwFZDyppU1j+xjiM/H8UZEUBYqzCcEQEc+fko659YVydF2CVLlvDmm2/SsWPHMstfffVVHnzwQe6++25WrlxJt27dGDduHAcOHPB4TDVh5ayHA89C/mZwRIAzseR7/mY48KxHi7BXXXUVixcv5rXXXuOHH35g5MiRnHvuuSQnJwO+e07Fs26b8g2bN6Vh23C8pGH9ehP8Lz+ncset33otNhGR30scnUhYqzAMh4HbduO2S54wCTACShrY0PPm7hhG+TsOrrq/Fw4/A6OCT42GCWdc2po2PaI8Gb6IiEfk5+ezd+9eALp06XLKtgPPaUF829CT3pll23DpXT0qzKMiIvWdJ4ceOK6x51EVYKvItmy2v7uNwowiwtuF4R8agOEw8Q8NILxdGIUZRWx/b5tHhyPIyspi4sSJvPjii0RERJRZ98wzz3DFFVdw66230rdvX2bNmkVgYCAvvPCCx+KpKctyQ9pH4M4EZxtwhIDhKPnubFOyPH22R4YjyM3N5dtvv2X69OmcddZZdOvWjSeffJKWLVsyc+ZMwDfPqXjW/n3ZfP7ZVtzuigcHd7ttPv14C4cO5tRxZCIiFTP9TM58ZwyBMYEUUQQGOAwH/n7+APSY3J32l7avcNt2vZsw4+szadKsZJzrksfHSoqvZ/6pPXe9PrTO+iEiUpsOHDiAbdskJCQQFXXqC0kOP5NH/zeGyLjAktkHjePLS3645G/dGX1NOw9HLCLiGUuXLvXY0APHNfY8qkm4qihjayaZO7MIjg/C+N2tIIZhEhwfROaOLDK2ZtKka+1fMQC4/vrrGT16NBdccAHTp08vXV5QUMCmTZv4xz/+UbrM4XAwdOhQVq9e7ZFYakXBDihMBv+mJfecn8gwSpYX7CppF9ypVg9dXFyM2+0mKKjsAM+BgYGsXLnSd8+peNSq7w/w28SMv11ssbEAR8lSy2b1qgNccGHnOo9PRKQike0jmLDsIla+vJKtb20mwOWk3ci2dLm2M3H94k65bY9h8by1cwLr5h5k96ZMAkP8GHReIrEtQuooehGR2mVZFvv376djx4507dq1UtskdorglV8uZP7bO1k6ezd5OcW06RHFeX/pRJdBp86jIiL11fbt29mzZ4/Hhh44UWPOoyrAVlFRViFWoRu/oIpPnSPIgZXqpiir0CPHf+WVV9i4cSMbNmwoty4lJQW3201CQkKZ5XFxcWzfvt0j8dQKVzZYReBffrB7AMxAKE4raVfLIiMj6d27N48++ig9e/akRYsWvPzyy2zYsIGWLVv67jkVj6rsExEN9dEJEfFdAWEBJI5PpEdoD5o2bcoZFwyr9LYOh0n/s1vQ/+wWHoxQRKRu7Nmzh8LCQgIDA2nTpk2ltwsJD+CCm7twwc2nHrJARMQX5OXlsWLFCgD69evnkaEHfq+x5lENQVBFARFOTKcDV37Fs5y7892YTgcBEbU/aPDOnTu5++67efvttwkODq71/XuNXziYAWCdZNIiq6BkvV+4Rw7/zjvvYNs2bdq0ITAwkH//+9+MGzdOxTM5qcGntSg34+3vGcDAQc3rIhwRkSrJy8sDaFifJUREquj45FudO3f26N1eIiL12YlDD/Tq1cvb4TRo+k1TRVGdIolsF0FeSj62XXacV9u2yEvJJ7J9BFGdImv92CtXruTo0aOcfvrp+Pn54efnx5o1a3j99dfx8/OjadOmOBwODh06VGa71NRU4uLq8a3cge1LxnotPswJz3WXsO2S5YFtS9p5QNeuXVmzZg1ZWVns3LmTn376ieLiYlq2bEl8fLxvnlPxKPcxi2gr8LfZt37PhiZWIO5cz40FLSJSXfn5+YAKsCLSeGVlZXHw4EGgpAArItIY1eXQA6ICbJUZpkGHKzvijAoge2cOxceKsNxuio8Vkb0zB2dUAB2u6IjpqP1Te95557FmzRq+//770q9u3bpxwQUX8P333xMUFETXrl2ZP39+6TZut5tly5YxYMCAWo+ntpimA2IvAUdkyViw7lyw3SXfC5PBLxJiJpS086Dw8HBatWpFWloaS5YsYdy4cQQGBvrkORXP2r7uCG3d4YTYvw5FcrwQ++v3UNufNu5wdqw/6pX4RERORXfAikhjt2nTJgBiY2MJDQ31cjQiInUvLy+P5cuXA3U39EBjpzFgqyG+fxx9/t6X7e9uI3NnFlZqybADTXpE0+GKjsT398ydkZGRkSQlJZVZFhwcTHR0dOny2267jcmTJ5OUlMTpp5/OE088QX5+PpMnT/ZITLXFDOuD1fxWSPuopOhanFYy7EBw15Lia1gfjx37k08+wbZtunXrxpYtW7jnnnto27YtU6ZMAXz3nIrn+AWY+GHS3d2EdAI5EFBywSXCDiTCFUoTOxATA78AXeMSkfpHBVgRacxcLhfbtm0DIDEx0cvRiIh4x5IlSygqKiI2NlZDD9QRFWCrKb5/HHF9Y8jYmklRViEBEU6iOkV65M7Xqpg0aRKpqak89thjpKen07lzZ+bMmUOLFvV/wgwzrA9WSE8o2FEy4ZZfOAS29/idr5mZmUybNo3Dhw8TERHBOeecw1NPPYXTWTKOry+fU/GMXsMT8AswcRVZxNhBuK2SWcA7uaNx2yX/Xv2dJj2HNfVmmCIiFTpegA0KCvJyJCIidW/Xrl0UFhYSGhpKWFiYt8MREalz27ZtY+/evRp6oI6pAFsDpsOkSVfv3qa9evXqcsumTp3K1KlTvRBNzZmmA4I71ekxr7/+eq6//vpTtvHlcyq1LzzayTl/7sj//r0F3OXXGyac95dOhEbW/mR8IiI1pTtgRaQxOz78QOfOnUvHgRURaSzy8vJYsWIFAElJSURFRXk5osZDZW4RkWr487/6M3hcS6BkbGgAh1/J99MuaMWkx5NOuq2IiLfYtk1BQQGgAqyIND7p6emkpqZimiadOtXtTR8iIvXBiUMP9OzZ09vhNCq6A1ZEpBoCnA4enD2CHxd34rGHtgC5DL+iDWdd042ug2MxDMPbIYqIlJOfn49t2xiGoSEIRKTROX73a9u2bZUDRaTR0dAD3qWzLSJSTYZh0H1IU0Zc1haAyU8NottpcSq+iki9deL4r8pVItKYFBUVsWPHDgC6du3q5WhEROpWbm6uhh7wMhVgRURqwHJZZG7PAuDg8kNYLsvLEYmInJwm4BKRxmrbtm24XC6io6OJj4/3djgiInVq6dKlFBUVERcXp6EHvEQFWBGRatr56S4+SPqQLW9uBmDhpIW83/dDdn22y8uRiYhUTBNwiUhjdXz4Ad39KiKNzYlDD5xxxhkaesBLNAasiEg17PxsF4v+uhi37S6zPD81n+9uWgyGQdsL2ngpOhGRiuXn5wMqwIpI43Lo0CEyMzPx8/Ojffv23g5HRKTOaOiB+kNlbxGRKrJcFqumrT5lm1UPrcZyazgCEalfdAesiDRGx+9+7dChAwEBAV6ORkSk7mjogfpDBVgRkSpKWZlC/uH8U7bJO5TH4e8P11FEIiKVowKsiDQ2+fn5JCcnAxp+QEQaFw09UL/o7IuIVFFe6qmLr6Xt0irXTkSkrmgSLhFpbLZs2YJlWTRt2pQmTZp4OxwRkTqhoQfqH40BWwOW2yI3OR1XTgF+YYGEtInBdKimXROW5cZiO9hZYERg0gHTdHg7LJEyQuIrd+dYSEKIhyMREaka3QErIo2Jbdts3lwyWarufhWRxmTJkiUaeqCeUbWwmjI37mfr41+z7V/fsuOZBWz717dsffxrMjfu9+hx//a3v2EYRpmvNm3KTvTz+OOP07x5c5xOJz179mTx4sUejam2uKx1FLjvpcD1APnuxyhwPUCB+15c1jqPHvebb75h5MiRxMXFYRgGs2bNKtemMufUV8+7VF384HhCmoWAcZIGBoQmhtK0f1ydxiUi8kc0CZeINCb79u3j2LFjBAYG0rZtW2+HIyJSJ7Zu3cq+fftwOBwMHz5cQw/UE3oXqiFz436S/7uUnO2H8QtzEtgiEr8wJznbD5P836UeL8K2b9+evXv3ln4dv60c4NVXX+XBBx/k7rvvZuXKlXTr1o1x48Zx4MABj8ZUUyXF16dx25uBCAwSgQjc9mYK3E97tAh77NgxevTowZNPPlnh+sqcU18971I9hmkw+LGBv774/cqSb4OnD8QwT1ahFRGpe0VFRbhcLkAFWBFpHI5PvtWxY0ccDj1VJyINX25uLitXrgRKhh6IjIz0bkBSSgXYKrItm0Of/0hxTj5BidE4QpwYpokjxElQYjTFOfkc+t+PHp393OFwkJiYWPqVkJBQuu6ZZ57hiiuu4NZbb6Vv377MmjWLwMBAXnjhBY/FU1OW5abI/QG2nYVBa0wzFNP0wzRDMWiNbWdR5P4Qy3J75PgTJkzgmWee4ZprrqlwfWXOqS+ed6mZVme1YvQbowhNDC2zPKxVGGPeGk3LM1t6KTIRkYodH34gICAAPz+NQiUiDZO70EVucjqHN+1mz+49gIYfEJHGQ0MP1F/69F1Fuclp5O05irNJKIZR9u42wzBwNgklb/dRcpPTCWvvmceP9+zZQ1xcHAEBAfTr14//+7//o0OHDhQUFLBp0yb+8Y9/lLZ1OBwMHTqU1atXeySW2mCxHcvehUFcuVvjTdPEsuKw7J1YbMekc53GVplz6qvnXWqu1diWNBuRwL5pe9mTtZux759J4qDEcrlBRKQ+0ARcItKQuQuKSX5lGQc+XYc7t4gtx/ax2y+D7ucMIiwszNvhiYh43O+HHtDfpfWLCrBV5MopxCpyYQaFVrjeDPTHOpKLK6fAI8cfPHgw3bt3p2vXrhw4cIBHHnmEM844g02bNpGZmYnb7S5zRyxAXFwc27dv90g8tcLOwqYIg8CTNAjEJq1kYq46lpKS8ofntDJtpOEyTIPwlmGwEWJ7x+qXnIjUW5qAS0QaKqvIxYZb3yfr5wNg2bhti735abhsF35fH2BH3AI63D7a22GKiNSa4ux8Dn25kfRlO7CLXTjaRbHKbw9GWICGHqinVICtIr8wJ2aAH1Z+MY4QZ7n1VkExZoAffmEnKybWzIQJE8q8HjZsGG3atOGNN95g/PjxHjmmxxkRGAQABUBFhe2CkvVGRB0HJiIi0nBoAi4RaagOztlA1sb9YJe8TinMoMh2EWgGEBcQwb731xB/VnfCOsdXuL1VUEj2sjVkL12NOycX//hYIkeeRkjf7hiavEZE6pnsLYfYcOv7JTf+/Zr3Vi1bSFpBFl0vGULPP2vogfpIBdgqCmkTS3CraHK2HyYoOKDM3W62bVN45BhhHZsS0iamTuKJiYmhVatW7Nixg/j4eBwOB4cOHSrTJjU1lbi4+jsbu0kHTKMtbnszlhVcZhgCy7KwScVhdMWkQ53HVplz6qvnXUREGhfdASsiDdX+T9aVFiEA9uSnApDgjMbAwHCYHJyzgU6dzyq3rSsji32PvUBxShoYBtg2RSlp5K77mdCknjS7fRKGnybwEpH6wZVbyIZbP8B1rLA07+3NTyWtIAsTk7gFWWSu20tUv1beDVTK0eW8KjJMg4Tze+EfFkT+vqO4cwux3Rbu3ELy9x3FPzyIhHG9MB11c2qzsrLYt28fCQkJBAYG0rVrV+bPn1+63u12s2zZMgYMGFAn8VSHaToIcFyGYURgsxvLOoZlubCsY9jsxjAiCXBcimnW/QefypxTXz3vIiLSuKgAKyINVf7+zNKfXZabo8U5ACTnp/BN2g8sSfuJRSuXsn79enbv3k1WVhaWVTJp8qF/v01x6pGSje1fqxm/rjv2w0bSP/6qzvohIvJHDn/7C67sfLBK8lW+u5BNx/YB0Cm0OaHOYPa+u8qbIcpJ6A7Yaojs0YI2Nw7l0Oc/krfnKNaRXMwAP8I6NiVhXC8ie7Tw2LH/8pe/MH78eNq2bcu+ffuYNm0apmly3XXXAXDbbbcxefJkkpKSOP3003niiSfIz89n8uTJHoupNviZfQnkDorcH2DZu7BJwyAAh9GVAMel+Jl9PXbsrKwsNm3aVPp6165drFy5kpiYGDp06FCpc+qr511ERBoPTcIlIg2VX3AAxVklw6w4DJP2wQkcLszkmLsANxZZ7nyswiOsWbOmdBvbttm1axfpa7cR6fAnIsBJmL+TMP8AHMefyLNtMr9ZTJMLx2IGBHijayIiZRxdlQwGpXe//nJsLy7bDUCuq4DknEOkL82mfe65hISEeCQGd14+tsuN7fT3yP4bKhVgqymyRwvCuzYjNzkdV04BfmGBhLSJ8fidrwcOHODaa68lMzOTqKgo+vfvz/Lly2nWrBkAkyZNIjU1lccee4z09HQ6d+7MnDlzaNHCc0Xh2uJn9sWkFxbbSybcMiJKhifw8J2vy5cv59xzzy19PW3aNKZNm8bFF1/M7NmzK3VOffm8i4hI41DZO2CtwiLcx3JxhIZgOlVwEJH6r+lZ3dg/+wdw2xiGQefQRDqHJmLZFnnuIo6584kfPwijQxQZGRlkZmZSWFhITk4O+3KzOeC2SvflZ5gMb9aKJoEludLKL6Bw70GC2rf2Uu9ERH5jW1aZIVfc9m/5a29BWskPuQYZs2YREBBAZGQkUVFRpd+joqIICwur1uTROWt/4uhn31KwYw8ARmw0jO6H7XKDv4qxf0QF2BowHSZh7et2jM8vvvjiD9tMnTqVqVOn1kE0tc80HZh0rtNjnnPOOdi2fco2lTmnvnzeRUSkYSvKyOPIzoO4TPukBdiiQ6mkz/6KnO/XgdsCh0nYoL7EXHw2Ac2a1nHEIiKVl3hZfw598RPu/OLSx3IBTMMk1BlE09aJ9J90AeavY7natk1GRgb+/v402ZvFsbw80gvyyCkuwmVbuH7/t8Gp/1QQEakz4d2bk75sZ+mQKf0jOpDjyifHnc8xVwHHrHzc8UGYpklxcTFpaWmkpaWV2YfD4SAiIqJMYTYyMpKIiAgcjopvgDv65QLS3v60ZKzsX7kysgA49NzrtLzzRo2X/QfqxRiwL7zwAq1btyYwMJCBAweyevXqU7bPzMzk5ptvJiEhAafTSceOHfnqK43NIyKNl/KoiFQkb99Rfrr7YxafM5Nt/13ErpcWs+m22RxZtatMu8J9B9lz77/IWflr8RXAbZGzch177nuCwr0HvBB93VIeFfFdQc0i6fPClThjwwAwHCaGo6RIENGtGX2evaK0+ApgGAZhYWHExsbSKTKG/rHNCPj1qbvmIWE0DfrtsV0j0IkzMaEOe+O7lEdFPK/Zeb0w/H4r5ZmGSYR/CC0CY+gc2oKk8A5cf+8UJk2axCWXXMLo0aNJSkqiffv2NGnSBIfDgdvt5ujRo+zcuZMffviB+fPnM3v2bF577TU++OAD5s6dy+rVq9m2bRtpaWnk7jtYUnyF38bKPkHuT1vIWrSyrk6Bz/L6HbAffPABd955Jy+99BIDBw5k5syZjB07lq1bt1Y4g3xRURFjxowhLi6O2bNn07x5c/bs2UNkZGTdBy8iUg8oj4pIRfL2HWXtpDdx5RZS5CoCwMCgcHs6P97+Ad2nX0jcyJKnTlJeegersKh04plSloVVWMShl96h9WP/qOsu1BnlURHfF945gdM++StHVuwke/MhDD+TJoPaEt612Sm3c7ZJZPO69RwpzMfPMOkbc0Kx1TCIHHU6ZqDTw9H7PuVRkboR0CSE7o+O5+d7Swqi9vEL56YBlk3zS/oRN6ozhmGUDjlwItu2ycnJITMzs3RIluPfi4qKyMrKIisrq8w2Oas2YO/dSbifP+EBgYQHBNA0KJQgv1/nFTAMMr5dTOToIR7vvy/zegH2qaee4s9//nPpJFIvvfQSX375Ja+99hr33HNPufavvfYaR48eZcWKFfj/OsZE69at6zJkEZF6RXlURCqy49mFuHILwW1TYBUD4DT9MTDAhi0zviZmSHuKDh2mYOceitxuCt0uCi03btsmNjAY0zDAsijctZeCPfsJbNUwxzZXHhVpGAyHSczQDsQM7VDpbcKvvYhfli8GoFeTpgT7+Zc8YmvbBHfrSMxl4zwVboOiPCpSd2LP6Ej/N69j34drSV+yDavYTXjXZrS4JImYoe1POb6rYRiEh4cTHh5Oy5Yty6zLy8sjIyOjXGE2IyOTwuIi8oqLSMnPBcBhGJzbrmvJhrZN0YHDHutvQ+HVAmxRURE//PBDmXEzTdNk9OjRrFxZ8e3Ln3/+OYMHD+bmm29mzpw5xMbGcuWVV3L33XefdKyKwsJCCgsLS18fn4QC+MOxPytSnW3qO/Wp7h2Pz+VyUVxcfNJ2x9edqo2vaUh9crlcuN0ls04WFxfj51f5tFob/fdWHs3Ozi7tQ2X70ZDe9+PUJ9/QGPtUdDSPtNU7wWGAw6DAcmE7DCwTfinYR6FVTNExF1seeRbbyOXw7s0nDpsIQJ/YBDpENil9nbf3IA4PjQVb3fdIedT3qU++oSH3aeWmnwm9YDTxqVl0dPtjHcsjIK4JEcMHE9q/F24D3D7Qb+VR5VH1qX6r7T45W0XR/u9jaP/3MWWWu1yuau/T39+fuLi4cnetJ6fkcchcS05hAQdyszmUewyHaYJ/yd++bj8Hhp+fz79fns6jXi3Apqen43a7adq07If5pk2bsmXLlgq32bVrFwsXLuSqq67iq6++YseOHUyePJni4mKmTZtW4TYzZszg4YcfLn09YMAA/v3vf5cpnFRWVdv7AvXJO9xuN5ZlsWbNmjIfJE5m3rx5dRBV3WoIfXK73WzatAmA+fPnn/QDY0VOvBhUXd7Ko8fNnTv3D2dU/72G8L7/nvrkGxpdn+78bVLL7IMHKdyYQyFQ8ueqP+DP/iM7Shr07QRQOi4YwKGePSHht0dxtx89BF8dqtX4f6+q75HyaMOhPvmGhtanw4cPs2HDBkzTZNCgQWwKC/ttZUYKzE3xXnDVpDxaOQ3t3zKoT77CJ/vUMR46ngdA2saNpB48SGJiInu6ltwBu+vCMwDY1kDGcPZUHvX6EARVZVkWcXFx/Pe//8XhcNCvXz8OHDjAE088cdJEPXXqVO68887S15mZmezbt49Dhw4RFxdHQEDAKW/RPvHYplkv5i2rNeqTd9i2zZEjRwAYOnQoAQEBJ21bXFzMvHnzGDNmTOnjOb6uIfXJ5XKxb/sOtu7ZzRl9kwhvXvlJGo5fta9rtZFHs7OzSUxM5MwzzyQ8PLxSx21I7/tx6pNvaIx9yt9/lDXXvlH6urXlojDHxm27cZr+OB3+BJh+dJw0nMSzupMybSYBxW6yiwqYv28XfqbBaZkO/MzNQMkkNG1nTsN0nvz3lSf7czLKo75PffINDbFPubm5PPzww3Tt2pV+/fqRlJTk7ZBqRHlUeVR9qt98uU9WcTF77/8/cg+ns3HXJuIsmyGpxURuTWPXhWfQ9n/LaHvfLQS08O0JCz2dR71agI2JicHhcHD4cNmxIg4fPkx8fHyF2yQkJODv71/mLrMuXbqQkpJCUVFRhYUsp9OJ01l24PSxY8fywQcfsG/fvkoVX6GkaFbZtr5CffIe27Zp3rw5ISEhf9yYkscBfC1R/xFf71PWmo1sfXIOu39YB2Pi+eGat4ltadLu9vMI79/jD7evjb57M49C9d5DX3/fK6I++YbG1Ce/1nGEtmhCbnIa2ODEpGdQ2XG+DIdB30uG42wSStAFZ5H+4RccyMzAtCxaBIfjtGywSu6GbXL2CJyhlft95Yn+nKp9TSmP1g/qk29oSH3asGEDhYWFREVFMWDAgCo9xVSfKY9WTkP6t3yc+uQbfLJP/v60njqFxX9/CMvlJtIZSJwzCLer5HNiy5snEtKm5R/sxHd4Ko96tQAbEBBAv379WLBgAePHjwdKroQtWLCAKVOmVLjN6aefzrvvvlvmLsdt27aRkJBwyrsIf2/Lli20bNkSp9NJUVHRH7Z3uVysWbOG/v37V2mMx/pMffKuoKCgKv2blfolY8kafrxnLsVlBk00yNhps+72OfT+VzGRp/f1eBzezKMiUn8ZhkHbm85g4z9mn6QBNJ/QD2eTUACix5+J61gu+/79HzAMEsOjwDTBsog6dyRNLhxbh9HXLeVRkcYnJSWFzZtL7vAfMmRIgym+eovyqEjD5x8bzbELRhDZIpru/sFEJLTAr00LtudlENy98x/vQLw/BMGdd97JtddeS1JSEgMGDGDmzJnk5uaWzp44ceJEmjdvzowZMwD461//yvPPP89tt93GLbfcwvbt23nssce49dZbq3xs0zQJCgoiKCjoD9sWFxdTWFhIeHi4712tOAn1SaR6LMtiy4xvsCwH2L+/29rAcptsmf4lg77yfAEWvJtHRaT+ih3Wga7TxrH1iW9x5xVh+JnYbhsMaHFJPzrcOqq0rWGa2GeeTkjqHgKS99Gl9wACoiOJGDoA/7gmpzhKw6A8KtJ4WJbF0qVLAWjevDnNmjXzckQNg/KoSMOWmZlJaloaQa2aM+iqqwgODi6ZfKqBjPtaF7xegL3ssstIS0vjwQcfJCUlhd69e/PNN9+UDuC9d+/eMuN5JiYm8u2333LHHXfQs2dPmjdvzm233cbdd9/trS6ISCOTtXwd+RmnSp8GeUf9yFyxnsjT+ng8HuVRETmZ+LO7EzuiE6nfbaHgQCZ+4UHEjeiEM7ZkohnbclO8bQOu/TvYvGUHuKHHhPNpNmKElyOvW8qjIo3Hhg0byMjIwOl00qlTJ2+H02Aoj4o0bNu2bQNK/u9WddI7KeH1AizAlClTTvpowqJFi8otGzx4MN9//72HoxIRqVju5j2lP9snjEBw4s8AxzbvrpMCLCiPitQ3dvFRcGVCQByGI9SrsTgC/Uk4u/y41K4DOzk26wmsrHRsw2Trz3soKComzk7F6t8XMzTCC9F6j/KoSP3iiTyamZnJunXrgJL/w8cLClI7lEdF6pfayqOWZZXmS124qr56UYAVEfEljrDfrvgV2a7ffrZc+PPb0Bd+YboyKNLY2DnrsA88Azm//kFp+GFHnY3R4nYMZwvvBncCd0Yq2a88BEWFAKRk55JfVEyAw0HTnIPkvP4I4ZMfx3Doo6KI1C1P5VHbtlm6dCmWZZGYmEj79u1VgBWRBqm28+j+/fvJy8sjMDCQVq1a1XK0jYf5x01EROREMecMwTCsU7YxTIsmZw+po4hEpD6ws5Zib7kaclafsNAFR7/C3jQBu2Cf94L7nYJlX0BxIdgluWx3Zi4ArSKDMW0b96HdFG9e680QRaQR8mQe3bp1K4cOHcLPz48hQ/QZTUQaJk/k0eMXq9q3b19mKBGpGp05EZEqKj6USlhEzinbhEXk4EpJraOIRMTbbNuFvesewPr160RucGVj753uhcgqVvTjErBK4nRbNnt+LcC2jvr18TTDpPCn5d4KT0QaIU/m0fz8/NJH3ZOSkggLC6tRrCIi9ZEn8mhBQQG7d+8GNPxATVW7ALtq1arajENExGe40o8SGnGM0Igc4MSBX23AJiwim9DwYxSnHfFShCJS57KWgCud4zmhqNjmYJoLyzqeI9yQtQi7qH5cmLEL80u+2zZL96RS6HIT5O9HfGjgrw0s7PxcL0YoIo3O7/JoscsmJd11QoPq59EVK1ZQVFRETEwM3bt3r514RUTqGw/k0Z07d2JZFk2aNKFJkya1G28jU+2BvQYPHkz79u255ppruOqqq2jbtm1txiUiUm85wkIxDIiIysYZYhNYVEAh0CQ6n8jAIzj8rF/b6e4KkUYjPxlwAG7SM93MW5VLTp5FfBM/RiYFExpsAjYU7oWAOC8HC2ZUU6z0Q6w+kM6ezFxMw2Boq1hMw/i1gYkjJsG7QYpI43KSPNq9nZPTegb92qjqeXTv3r3s3LkTwzAYNmyYHp8VkYbLA3l069atgO5+rQ3V/u0za9YsOnTowCOPPEKHDh04/fTTeemllzh69GhtxiciUu8Ed+uII7zkMV0/PxtnYMkkNqHhub8VXyPDCe7a3msxikgdc4QCFpuTC/ls0TFy8kpyQcoRFx8vzGH3weIT2nlf4MCx/JSSwZa0bACGtoolISzotwaWhbP/KC9FJyKN0kny6M87C9m+t+h37SqnuLiYZcuWAdCjRw9iYmJqM2IRkfqllvPo0aNHSU9PxzRN2rfX37Y1Ve07YK+88kquvPJK0tPTef/993n33XeZPHkyt99+O2eddRZXX301559/PgEBAbUZr4iI1xl+DmKvuICU/7xz0jaxV5yP4XDUYVQi4k2usGEsWZvP9n0FALSK96dfl0CWrs8jLdPN3FW5dO/ckkF92lf/w1ctSg5P5MdCfzBMBjSP+m3s118FDhmHXzM93SQidaeiPBoeYrJxZyFLN+QTHe6gSdM2EFTxXVi2bVO89QcKVnyFa992DNPBOiOKLP8YIhLbkJSUVJfdERGpc7WdR9fsP0KBEU7HEecQGBhYl11pkGr8/EVMTAxTpkxhxYoVbN++nfvuu48tW7Zw2WWXER8fz4033lh61VFEpKGIGDGYpjdcjhlU9heRGRxE0xuvJOKMQV6KTETqWmZmJnO+XM72zN4YhsHAbkGMHRxCTKSDC84IpWd7JwC/pA/g888/Jysry6vx7tmzh2XfryLw9PPoN+ZcujRvWrrOjIgh+PwbCDp7ohcjFJHG5mR5dFCPQBKb+uNy28xdlUtRzF8xjg+VcgLbtsn7+i2OvfU4rp0/Q2E+aUeOsHHlUgoWf8qAJk78/OrD5S8REc+o7TxqFeSxMyUN9/4dNFv+IYU/qq5XU7X6WygoKIjg4GACAwOxbRvDMJgzZw6vvvoqffv25c0336Rr1661eUgREa+JHD2E4NP6EvHwo6TmZJJw09U0GdQXM8Df26GJSB3ZtWsXixcvpri4mODmFzKqc1sSjK8BAwwT03QzqGc4zXvdwOJfoklPT+eTTz5hyJAhdOjQoc7jTUlJYf78+di2Tedu3Rl2xhnYxYW40w9hOPwxYxIwND6iiNShE/NoSIuLGNnptzxqmCYjk4L5dHE+OQFjWLghkLMT7HLFg+LNayhc9r+SF7aFZdus2JuObVm0iQoleumHWIOGY0ZoAhkRaXg8kUf3Z+WVTtLaLMxJ7kfP4d+6i/JoDdS4AJuTk8Ps2bN55513WLx4MaZpcvbZZ/Pggw8ybtw4TNPk008/5W9/+xvXXXcdq1atqo24RUTqBTMggMDWibAxk9D+PVV8FWkkLMti1apVbNy4EYCEhARGjRpFcPCfsAvvhCNfYbsyMZwtoMl5tPKL4OLOuXz33XccPHiQ7777jgMHDjBkyJA6uysrIyODb7/9FrfbTcuWLRk6dCgAhr8Tv4TWdRKDiMhxJ8+j15bJo4HOFpzZ/jQ+//I79u/fz9q1a+nfv3+ZfRUs/xIME+yS8Q43p2WTkV9IgMNB/+ZNwLYpWD2P4DGX13k/RUQ8xZN5dMfRHADaRoWWTNKqPFpj1f7EP2fOHN555x2++OILCgoK6N+/PzNnzuTyyy+nSZOyFfEJEyaQkZHBzTffXOOARURERLwpNzeX+fPnc/jwYQB69epF//79S2fWNpyJ0Owv/P7hrpCQEM4991zWrVvHunXr2LZtG6mpqYwePZro6GiPxnzs2DG++uorCgsLadq0KaNHj9ZM4CLiNb/Po7179yYpKemkeTQGGDbMzcKFC1m/fj2xsbG0bt26dH+u/dtLiwZ5xS42HMoAIKl5NEH+DrAtXHu31VX3REQ8zpN5NLfIxf7sfAA6NPl1ngDl0RqrdgH2wgsvJDExkTvuuIOJEyfSqVPFg/ge16tXL6666qrqHk5ERETE6/bv38/ChQspKCggICCA4cOHl/nw+kcMw6Bfv340a9aMBQsWkJmZyaeffsrgwYM9NkxTYWEhX3/9Nbm5uURGRjJ27FiNhSgiXvP7PDpixAhatWr1h9u1b9+etLQ0Nm7cyHfffceFF15IZGRkyUrjtwtKvxzOwmVZxIYE0qFJ2G870OSoItJAeDqPbkrNwrZt4kIDiQgM+G0HyqM1Uu1P3wsXLmT48OGVbj9gwAAGDBhQ3cOJiIiIeI1t26xfv561a9cC0KRJE8aMGUN4eHi19peQkMCECRNYtGgRe/fuZdmyZRw4cIAzzjiDgICAP95BJblcLr799lsyMjIICQnhnHM0i62IeMfv82hMTAyjR4+uUh4dOHAg6enpHDp0iLlz5zJ+/HgCAgLw79iH4k2rKCgqZtuRksdme8VHnrClQUCH3rXXGRERL6jrPNqzaeQJWyqP1lS1nz2rSvFVRERExFcVFBTwzTfflH7Y7dy5MxdccEG1i6/HBQYGctZZZzFo0CBM0yQ5OZmPP/6Y1NTU2ggby7JYsGABKSkpBAQEcPbZZxMaGlor+xYRqYrayqOmaTJ69GhCQkLIzMxk0aJFWPmZONvEgmWxKbXk7tfoYCfNw4NLNjJMjMAgAvoOr+VeiYjUHeVR31ftAuz9999P7969T7q+T58+PPzww9XdvYiIiIjXpaam8sknn7Bv3z4cDgfDhw9n2LBhtfoIf8+ePUs/QOfk5PD555/z448/Ytt2jfa7bNky9uzZg8Ph4KyzzvL4OLMiIhU5WR51VPNR1qCgIMaMGYOBza7vXuOH/xuFufU/mE2OsiU9G4BepXdtGRjOQMKuux8zKKR2OiQiUseURxuGav/1MHv2bC688MKTrj/nnHP44IMPmDZtWnUPISIiIuI1v/zyCytXrsSyLMLDwznzzDM9VsSMjY3loosuYunSpezcuZNVq1Zx8OBBhg8fTlBQUJX3t3btWrZs2YJhGIwaNYr4+HgPRC0icmqeyqNxcXGcFraNJYfWshab2JAoUnOOYUbmEWOE0jreHzOuHf7dh+DsOxwzOOyPdyoiUg/VZR5NO6Y86knVLsDu3buXdu3anXR9mzZt2LNnT3V3LyIiIuIVxcXFLF26lB07dgAln2lqe2zWigQEBDBq1CiaN2/OihUr2LdvHx9//DEjR46kWbNmld7PL7/8wrp16wAYOnRolSYJExGpDZ7Oo3bWXjoVfk9qQiBbDuWzcFMWAIafRf8uBqHxGRgt/XAMGVcrxxMRqWvKow1PtQuwoaGhpyywJicna5IHERER8SmZmZnMnTuXzMxMDMNg4MCB9OzZs05j6Ny5M02bNmX+/PlkZGTwxRdf0LdvX/r27Ytpnnr0qF27drF8+XIAkpKS6Ny5c12ELCJSqi7yqLVrHhgOTusQztFcF6nZxQCEB/nRNjYQbAt77xLs4jwM/+BaPbaIiKcpjzZMNZqE6z//+Q8HDhwot27fvn3897//ZcSIETUKTkSkPrPdx7BT3sQ69F8ArM2XY6e8ie3O9XJkIlIdO3bs4JNPPiEzM5Pg4GDGjRtX58XX46KiorjwwgtLC6jr1q3jiy++IDe3JL/YtgWAteVjrK1zsPPSOHjwIAsXLgSga9eu9O3b1yuxi0jjVWd5tDATDAOHaTC6WyRBASV/1vZqGYxpGiVtbAuKcmr/2CIiHqQ82nBV+w7YRx55hAEDBtCtWzcmTZpEt27dAPj555957bXXsG2bRx55pNYCFRGpT+ziI9hbrsI+lgzFvxZc85Ox982AtA+g8ywMf014I+IL3G4333//Pb/88gsAzZo1Y9SoUdUae7U2+fn5MWzYMJo3b86SJUtISUlh9uzZnNGtKU2T34CAK7DWvYxlF3BknptvDiXiThxB23YdOP30070au4g0LnWdR43guNILUSFOB+N6R5OaXUz7pic8gWn6gzPSI8cXEaltyqMNX7ULsJ06dWLp0qXccsstPP3002XWDRs2jGeffZYuXbrUOEARkfrI3n0/FOwFTpyl3C75KtiNvftBjA7Peyk6EamsY8eOMX/+fFJTUwHo06cPSUlJGIbh5ch+065dO2JjY5k/fz5pezbx9YuP0aVZGO6+brAtcvLdfPPTUQqL0kkIcTDyz3+pV/GLSMP2+zzat29f+vXr59E8ZLQ7C9b/t/R1RLAfEcEn/GlrODDajcXwc3osBhGR2qI82jhUuwAL0LNnTxYvXkx6ejq7du0CoG3btsTExNRKcCIi9ZFduB8yv6Ns8fVEbsicj114CMOZUJehiUgV7Nu3j4ULF1JYWIjT6WTEiBG0bNnS22FVKDw8nPHjx7Pixc/ZaFv8vD+H/a7V9E0oYsWWDPKK3ESH+HFm3B7M7N0QdfKJUkVEaou38qgREovZ6zqsDa9WsNIBAaGYva7zeBwiIjWlPNp41KgAe1xMTIyKriLSeBz7kePF19/f/1rmVe6PoAKsSL1j2zY//PAD69atAyA2NpbRo0cTFhbm5chOzbCKGRS2k2bdI1i4JZfs7Gy+SEnHgYtQp4Oze0YR4O+PtWsujn5/9Xa4ItKA1Yc8avS6DtMZjvXjG1CQ8duKhH44Bv0NIzS+zmIREakq5dHGp8YF2P3797N+/XqysrKwLKvc+okTJ9b0ECIi9Yvx2/yFBYX2CT9bBAec2M5Rh0GJSGUUFBSwcOFC9u/fD5RMVjV48GAcDh/4/1qcC7ZFyyZOxicFk7w3CvIg0N/k7F5RBDt/7UNBlnfjFJEGrb7kUcMwMLpMwOg0HtJ+wS7Ox4hohRGmi98iUr8pjzZO1S7AFhQUcO211/Lxxx9jWRaGYWDbJYWIE8epUAFWRBqcsP6AA3CfvI3hB6H96ioiEamEw4cPs3jxYnJzc/Hz82Po0KF06NDB22FVXkAY+AWCq4AQp4OkpCQ6RSwgIQzCgo5/YLcxQvWhWUQ84/d5dNiwYbRv396rMRmmHzTthUa+FhFfoDzaeFW7AHvvvffyySefMH36dAYPHszw4cN58803SUhIYObMmRw8eJC33nqrNmMVEakXDP8Y7Cbj4MjnQPk7/8GEJuMx/KPrOjQROYk9e/aQkpKCYRhERkYyZswYoqKivB1WlRgOf4z252BvnQM2mKZJm7gg/Cn6rZENRvuzvRekiDRYDSGPioh4k/Jo42b+cZOKzZ49m+uuu467776bbt26AdC8eXNGjx7NF198QWRkJC+88EKtBSoiUp8YrR6E0L7HX/36/dc70MKSMFre542wROR3ioqKWLBgAVu2bMGyLNq2bcuFF17osx92zZ5/gqCokw5xYva5HiMkrm6DEpEGraHlURGRuqY8KlCDO2BTU1MZMGAAAEFBQQDk5uaWrr/44ov5f//v//Hvf/+7hiGKiNQ/hiMEOr+JEfwFBM6ELCB8EEa7SyByBIZRK3McikgNHD16lHnz5nH06FFM02TQoEH06dPH22HViBHcBMe5L1O86jnIOWFFcCxmrz9hdLzAa7GJSMPTEPOoiEhdUh6V46pdIWjatClHjhwBIDg4mKioKLZu3cq4ceMAyM7OpqCgoHaiFBGphwzDDyN6FGbcITi8EbPt4xhRGnZApD7Yvn07S5cuxeVyERwcTP/+/enevbu3w6oVRkgcfkMfhK++wjH6SRyBQRDdEcP0gYnERMRnNOQ8KiJSF5RH5UTVLsAOHDiQZcuWcffddwMwbtw4nnjiCRISErAsi6effppBgwbVWqAiIvWRbdvYuYdLfs7YhR0VVWYiQhGpW263mxUrVrB582YAWrRowdChQ1m4cKGXI/MMo2kPDH9/b4chIg1IY8ujIiK1TXlUKlLtAuytt97KRx99RGFhIU6nk0ceeYSVK1dyzTXXANCuXTueffbZWgtURKS+sQ6swr34/7B+/gmCR+Geewfu+JaYA2/DbDbA2+GJNDo5OTnMmzeP9PR0APr160ffvn1xuVxejkxExDcoj4qI1IzyqJxMtQuwQ4YMYciQIaWvExMT2bx5Mxs3bsThcNC5c2f8/DQGoog0TNb+FVjz74b84rIrsvZgzfsbjP4/zOYDvROcSCO0d+9eFi5cSFFREYGBgYwcOZIWLVp4OywREZ+hPCoiUjPKo3Iq1aqQ5uXlcfXVV3PxxRdz1VVXlS43TZNevXrVWnAiIvWRbVtY3z8J2BWtBRus75/EuOgDDUcg4mGWZbF27Vo2bNgAQFxcHKNHjyY0NNS7gYmI+AjlURGRmlEelcqoVgE2ODiY+fPnc/bZZ9d2PCIi9d/hn+BYyika2JBzANJ+hrgedRaWSGOTn5/PggULOHjwIADdu3dn0KBBmKbp5chERHyD8qiISM0oj0pl1WgIgpUrV/LnP/+5NuMREan3jk+6VZl2BirAinhCSkoK8+fPJy8vDz8/P8444wzatWvn7bBERHzGiXnU39+fYcOGKY+KiFSB8qhURbULsM8//zxjx47l/vvv56abbtK4FiLSeARFlf4Y6G+c8LMJWL+1C4xCRGrfTz/9xKpVq7Btm6ioKMaMGUNkZKS3wxIR8RnKoyIiNaM8KlVV7QJsr169cLlczJgxgxkzZuDn54fT6SzTxjAMsrKyahykiEh9YsT3LSmuFmSUGeO1zHCvQTEYTXvXeWwiDVlRURGLFi1i9+7dALRv356hQ4fi7+/v3cBERHyE8qiISM0oj0p1VbsAe/HFF2tyGRFplAzTDzPpZqxlj560jZk0GcN01GFUIg3bkSNHmDdvHtnZ2ZimyWmnnUbXrl29HZaIiM9QHhURqRnlUamJahdg33jjjVoMQ0TEt5jtzwbbgu+fAY78tsIZjtn/Fsx2Y70Wm0hDs3XrVpYtW4bb7SY0NJQxY8YQGxvr7bBERHyG8qiISM0oj0pNVbsAKyLS2JkdzsXRcgTGvgdhfx7msGk42g7FcOjxE5Ha4HK5WL58OVu3bgUgMTGRESNGEBgY6OXIRER8w+/zaMuWLRk+fLjyqIhIJSmPSm2pdgH2rbfeqlS7iRMnVvcQIiL1nuEIwIxqB/s3YrYYrOKrSC3Jzs5m3rx5HDlyBMMwSEpKonfv3hr+SESkkpRHRURqRnlUalO1C7B/+tOfTrruxH+MKsCKiIhIVezevZtFixZRVFREYGAgo0aNonnz5t4OS0TEZyiPiojUjPKo1LZqF2CTk5PLLXO73ezevZsXX3yRvXv38uabb9YoOBEREWk8LMti9erV/PTTTwA0bdqU0aNHExIS4uXIRER8w+/zaHx8PKNGjVIeFRGpJOVR8ZRqF2BbtWpV4fK2bdsycuRIzj33XJ5//nleeOGFagcnIiIijUNeXh7z588nJSUFgJ49ezJgwABM0/RyZCIivkF5VESkZpRHxZM8NgnXeeedxwMPPKACrIiIiJzSwYMHWbBgAfn5+fj7+zN8+HDatGnj7bBERHyG8qiISM0oj4qneawAu3PnTgoLCz21exEREfFxtm3z448/smbNGmzbJjo6mjFjxhAREeHt0EREfILyqIhIzSiPSl2p9n3US5YsqfDr888/56677uLZZ5/l7LPPrtS+XnjhBVq3bk1gYCADBw5k9erVldru/fffxzAMxo8fX91uiIg0CMqj4msKCwuZO3cuq1evxrZtOnbsyPjx4/VhV7xGeVR8jfKo1DfKo+JrlEelLlX7Dtjhw4djGEa55bZt43A4uOSSS3juuef+cD8ffPABd955Jy+99BIDBw5k5syZjB07lq1btxIXF3fS7Xbv3s1dd93F0KFDq9sFEZEGQXlUfE16ejrz5s0jJycHh8PB6aefTufOnb0dljRiyqPia5RHpb5RHhVfozwqda3aBdjvvvuu3DLDMIiKiqJVq1aEh4dXaj9PPfUUf/7zn7nuuusAeOmll/jyyy957bXXuOeeeyrcxu12c9VVV/Hwww+zdOlSMjMzT3mMwsLCMsMhZGdnA1BcXExxcXGl4jzerrLtfYH65BvUp/rN5XLhdruBkv74+VU+rdZW/5VHvUd9qrrNmzfz/fff43a7CQ0NZfTo0cTExHj0HOp9qv+q2x/lUd+nPlWd8mjtaGh9Uh5VHlWfKk95tHY0tD55Oo8atm3bVY6qlhQVFREcHMzs2bPLPG5w7bXXkpmZyZw5cyrcbtq0afz00098+umn/OlPfyIzM5PPPvvspMd56KGHePjhh8stf/fddwkODq5pN0SkEXO73cyfPx+A0aNH43A4Kr1tXl4eV155JVlZWZW+aPV7yqPiK9xuN5s2beLgwYMAxMXF0b17d/z9/b0cmfgy5VFpTJRHxROUR6UxUR4VT6hsHq32HbDJycn8/PPPjBs3rsL1//vf/+jRowetW7c+6T7S09Nxu900bdq0zPKmTZuyZcuWCrdZtmwZr776Khs2bKh0rFOnTuXOO+8sfZ2dnU1iYiJnnnlmpX/JFBcXM2/ePMaMGdNg/nOqT75BfarfXC4XBw8eZNOmTYwePZqgoKBKb3v8qn1NKI96l/pUOVlZWcyfP58mTZoQExNDv3796NWrV4VDGXmC3qf6r7r9UR71fepT5fw+jyYlJdGzZ0/l0RpoaH1SHlUeVZ9OTXm09jW0Pnk6j1a7AHvXXXeRnZ190gLsCy+8QGRkJO+//351D1FOTk4O11xzDS+//DIxMTGV3s7pdOJ0Osst9/f3r/I/kupsU9+pT75BfaqfDMMoveu1qv3xRt+VRz1DfTq5Xbt2sXjxYoqLiwkNDWXUqFE0a9asFiKsOr1P9Z/yaOU0tPcd1KdT+X0eHT16NAkJCbUQYdXpfar/lEcrp6G976A+nYryqGc1tD55Ko9WuwC7cuVKbr/99pOuHzVqFDNnzjzlPmJiYnA4HBw+fLjM8sOHDxMfH1+u/c6dO9m9e3eZoq9lWQD4+fmxdetW2rVrV/lOiIj4OOVRqa8sy2LVqlVs3LgRgISEBEaNGqVHBKXeUR6V+kp5VHyF8qjUV8qjUp9UuwCbkZFBWFjYSdeHhoZy5MiRU+4jICCAfv36sWDBgtKxYizLYsGCBUyZMqVc+86dO5f+xznu/vvvJycnh2eeeYbExMSqd0RExIcpj0p9lJuby/z580v/EOvVqxf9+/fHNE0vRyZSnvKo1EfKo+JLlEelPlIelfqm2gXYli1bsnz5cv76179WuH7p0qW0aNHiD/dz5513cu2115KUlMSAAQOYOXMmubm5pbMnTpw4kebNmzNjxgwCAwPp3r17me0jIyMByi0XEWkslEelPjlw4AALFiygoKCAgIAAhg8ffsrx4EXqA+VRqU+UR8UXKY9KfaI8KvVRtQuwV1xxBY888ggDBgxgypQppVcR3G43zz//PB988AH33XffH+7nsssuIy0tjQcffJCUlBR69+7NN998UzqA9969e3WFQkTkFJRHpT6wbZv169ezdu1aAJo0acKYMWOqPaOySF1SHpX6QHlUfJnyqNQHyqNSn1W7ADt16lSWLVvG7bffzvTp0+nUqRMAW7duJS0tjeHDh1eqAAswZcqUCh9NAP5/e/ceHVV9v3v8mUySSQIEgiEJQWIApYjcKhxoUA5FA3gpClVkSQVEVOTSKrH8RBQDRQVdSPEoSrnjOtBQELQVBMIltdyKcrEqCCI3L00QJQSIhEnyPX94GI1JIDPDntkzeb/WYq3Onr0nnw9DH1xPhh3l5eVd9NqFCxd6MzYAhCVyFMF07tw5bdq0SV988YWkH/5pYdeuXRUZ6fN/ZgABR44imMhRhANyFMFEjsLufP6T6HK5tG7dOi1atEgrVqzQ559/Lknq3Lmz7rrrLg0ePJjvcAEAEOaOHz+u9evX68yZM3I6nerWrZtatmwZ7LEAIGSQowDgH3IUocCvbwVERERo6NChnvu6AACA2mPv3r3aunWrysvLFR8fr549e+qKK64I9lgAEDLIUQDwDzmKUOFzAfvdd9/pyy+/VLt27ap8/qOPPtKVV16phIQEn4cDAAD243a79a9//UsHDx6UJDVr1kzdu3dXdHR0kCcDgNBAjgKAf8hRhBqfC9gxY8Zo//792r59e5XPDx8+XNdee63mzZvn83AAAMBeCgsLtW7dOhUWFsrhcKhLly7VfjMWAFAZOQoA/iFHEYp8LmA3btyoESNGVPt8nz59NGvWLF9fHgAA2Mznn3+uf/7znyotLVVcXJwyMzOVkpIS7LEAIGSQowDgH3IUocrnAvabb75RYmJitc9fccUVOn78uK8vDwAAbKKsrEzbt2/XJ598IklKTU3VzTffrNjY2CBPBgChgRwFAP+Qowh1PhewjRs31u7du6t9fufOnWrUqJGvLw8AAGzgzJkzWr9+veebqr/85S/VqVMnORyOIE8GAKGBHAUA/5CjCAc+F7B9+/bVzJkzdeutt+qOO+6o8Nzbb7+tBQsWXPQWBQAAwN6++OILbdy4USUlJXK5XOrRo4fS0tKCPRYAhAxyFAD8Q44iXPhcwE6cOFHr169Xv3791L59e7Vp00aS9PHHH+vDDz/Utddeq0mTJl22QQEAQGAYY7Rz50795z//kSQlJiaqZ8+eqlevXpAnA4DQQI4CgH/IUYQbnwvY+vXra/v27XrxxRe1YsUKLV++XJLUokULTZgwQWPHjlWdOnUu26AAAODyMea8Ss1WGXNSEY5UOR3/Sw5HhM6dO6edO3cqNTVVTqdTrVu3VkZGhpxOZ7BHBgBbIUcBwD/kKGoTnwtYSapTp44mTZrEJ10BAAgh58uW6lzZCzIq9BxzqImKvs3ShnVF+vbbb5WWlqZf//rXuuaaa4I3KADYFDkKAP4hR1Hb+FXAAgCA0FJS9n91ruyZSsc/+eSIPvj3cDnKBqtOnTq64447lJycHIQJAcDeyFEA8A85itrIrwL23LlzevPNN7Vr1y6dOnVK5eXlFZ53OByaN2+eXwMCAIDLw5jvda7shQrH3G6jLe+d05HDbkkOpV21RY1T/qiGDRsGZ0gAsDFyFAD8Q46itvK5gD169Kh69OihI0eOqEGDBjp16pQaNmyowsJClZWVKTExUXXr1r2cswIAAD+4yzdKOut5fPK7MuVt/F6nCssVEeFQpy4utWxZqM3r+QcyAFAVchQA/EOOoraK8PXCsWPH6tSpU9q+fbsOHDggY4yWLl2qM2fO6IUXXlBsbKzWrl17OWcFAAB+MPpGP/2r/1/vndOpwnLF1YnQLbfHqfV10cEbDgBCADkKAP4hR1Fb+VzAbty4USNHjlTnzp0VEfHDyxhj5HK5NHbsWN1888167LHHLtecAADATxGOJEk/3i7of3ePUdpVkbqjX5ySkvmpsgBwKeQoAPiHHEVt5XMBW1xcrPT0dElSfHy8HA6HTp065Xk+IyNDmzdv9ntAAABweUQ6bpZUz/O4QYJTN/WMU0zMhf8ciFCEo3VQZgOAUECOAoB/yFHUVj4XsGlpafryyy8lSZGRkWrSpIm2b9/ueX7v3r2KiYnxf0IAAHBZOBwuxTqfqu5ZSQ7FOMcGciQACCnkKAD4hxxFbeVzAXvTTTfp7bff9jy+//779ec//1kPPfSQhg0bppkzZ6pPnz6XZUgAAHB5RDvvUaxzuhxKqnA8Qs1VJ/INRUZ0DNJkABAayFEA8A85itrI5x8rN27cOL3//vsqKSmRy+XS+PHj9fXXX2v58uVyOp0aOHCgpk+ffjlnBQAAl0G0s6+iIn6jMvO+jE7KoSZyOtrJ4XDIXeYO9ngAYHvkKAD4hxxFbeNzAZuWlqa0tDTP45iYGM2dO1dz5869LIMBAADrOByRinRkBHsMAAhZ5CgA+IccRW3i8y0IAAAAAAAAAAAXRwELAAAAAAAAABahgAUAAAAAAAAAi1DAAgAAAAAAAIBFKGABAAAAAAAAwCIUsAAAAAAAAABgEQpYAAAAAAAAALAIBSwAAAAAAAAAWIQCFgAAAAAAAAAsQgELAAAAAAAAABahgAUAAAAAAAAAi1DAAgAAAAAAAIBFKGABAAAAAAAAwCIUsADgB2PKVVZ+VJJUWr5LxpQHeSIAAAAAAGAnFLAA4CN3+Tqddt+k8+WLJEnFpcN02v1rucvXB3kyAAAAAABgFxSwAOADd/laFZeOkNHXFY4bfaXi0uFyl+cGaTIAAAAAAGAnFLAA4CVjyvR96Z8uPPr5s5Kk70snczsCAAAAAABAAQsA3iozO2T0X1UuXy8wMvpSZeaDQI4FAAAAAABsiAIWALxUbgou63kAAAAAACB8UcACgJciHI1qeF6SxZMAAAAAAAC7o4AFAC85Hb+SQz+Uq5GRDv1ucH1lZmYqMtLhOcehxnI6OgVrRAAAAAAAYBMUsADgJYfDqRjnBM/jyEiHnE5nhXNiI5+Ww+H8+aUAAAAAAKCWoYAFAB9EO29XrPP/eD4Je4FDSYqLfFVREbcGaTIAAAAAAGAnkcEeAABCVbTzN4qKuEXnzFZJJxUXOVsxUV355CsAAAAAAPCwxSdgZ86cqfT0dMXExKhLly7asWNHtefOmTNH3bp1U0JCghISEpSZmXnR8wHASg5HpCIjMiRJkRG/Clr5So4CgH/IUQDwDzkKANULegG7dOlSZWVlKTs7W7t27VL79u3Vu3dvHT9+vMrz8/LydO+992rTpk3atm2bmjZtql69eumrr74K8OQAYA/kKAD4hxwFAP+QowBwcUEvYKdPn66HHnpIQ4cOVevWrTVr1izFxcVp/vz5VZ6/ePFijRw5Uh06dFCrVq00d+5clZeXa8OGDQGeHADsgRwFAP+QowDgH3IUAC4uqPeAPX/+vHbu3Kknn3zScywiIkKZmZnatm1bjV6juLhYbrdbDRs2rPackpISlZSUeB4XFRVJktxut9xud42+zoXzanp+KGCn0MBO9ufrPpdjf3I0uNgpNLCT/ZGj5Cg72Rs72R85So6yk72xk/1ZnaNBLWBPnDihsrIyJScnVzienJysTz/9tEav8cQTTyg1NVWZmZnVnjNlyhRNmjSp0vF169YpLi7Oq5lzc3O9Oj8UsFNoYCf783af4uJiv78mOWoP7BQa2Mn+yNGaCbf3XWKnUMFO9keO1ky4ve8SO4UKdrI/q3I0qAWsv6ZOnaqcnBzl5eUpJiam2vOefPJJZWVleR4XFRV57jETHx9fo6/ldruVm5urnj17Kioqyu/Z7YCdQgM72Z+v+1z4rn0wkaP+YafQwE72R46So+xkb+xkf+QoOcpO9sZO9md1jga1gE1MTJTT6VRBQUGF4wUFBUpJSbnotdOmTdPUqVO1fv16tWvX7qLnulwuuVyuSsejoqK8/kPiyzV2x06hgZ3sz9t9Lsfu5Kg9sFNoYCf7I0drJtzed4mdQgU72R85WjPh9r5L7BQq2Mn+rMrRoP4QrujoaHXs2LHCjbYv3Hg7IyOj2utefPFFTZ48WWvWrFGnTp0CMSoA2BI5CgD+IUcBwD/kKABcWtBvQZCVlaUhQ4aoU6dO6ty5s2bMmKGzZ89q6NChkqTBgwerSZMmmjJliiTphRde0DPPPKMlS5YoPT1d+fn5kqS6deuqbt26QdsDAIKFHAUA/5CjAOAfchQALi7oBeyAAQP0zTff6JlnnlF+fr46dOigNWvWeG7gfezYMUVE/PhB3ddff13nz5/X3XffXeF1srOzNXHixECODgC2QI4CgH/IUQDwDzkKABcX9AJWkkaPHq3Ro0dX+VxeXl6Fx0eOHLF+IAAIMeQoAPiHHAUA/5CjAFC9oN4DFgAAAAAAAADCGQUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARWxRwM6cOVPp6emKiYlRly5dtGPHjouev2zZMrVq1UoxMTFq27atVq9eHaBJAcCeyFEA8A85CgD+IUcBoHpBL2CXLl2qrKwsZWdna9euXWrfvr169+6t48ePV3n+1q1bde+992rYsGHavXu3+vbtq759++rjjz8O8OQAYA/kKAD4hxwFAP+QowBwcZHBHmD69Ol66KGHNHToUEnSrFmztGrVKs2fP1/jxo2rdP7LL7+sW265RWPHjpUkTZ48Wbm5uXr11Vc1a9asKr9GSUmJSkpKPI+LiookSW63W263u0ZzXjivpueHAnYKDexkf77uc7n2J0eDh51CAzvZHzlKjrKTvbGT/ZGj5Cg72Rs72Z/VOeowxhivp7pMzp8/r7i4OC1fvlx9+/b1HB8yZIgKCwv19ttvV7omLS1NWVlZeuyxxzzHsrOz9dZbb+nDDz+s8utMnDhRkyZNqnR8yZIliouL83sPAPBFcXGxBg4cqFOnTik+Pt6n1yBHAdRm5CgA+IccBQD/1DRHg/oJ2BMnTqisrEzJyckVjicnJ+vTTz+t8pr8/Pwqz8/Pz6/26zz55JPKysryPC4qKlLTpk3Vq1evGv8l43a7lZubq549eyoqKqpG19gdO4UGdrI/X/e58F17f5CjwcVOoYGd7I8cJUfZyd7Yyf7IUXKUneyNnezP6hwN+i0IAsHlcsnlclU6HhUV5fUfEl+usTt2Cg3sZH/e7hNKu5OjF8dOoYGd7I8crZlwe98ldgoV7GR/5GjNhNv7LrFTqGAn+7MqR4P6Q7gSExPldDpVUFBQ4XhBQYFSUlKqvCYlJcWr8wEgnJGjAOAfchQA/EOOAsClBbWAjY6OVseOHbVhwwbPsfLycm3YsEEZGRlVXpORkVHhfEnKzc2t9nwACGfkKAD4hxwFAP+QowBwaUG/BUFWVpaGDBmiTp06qXPnzpoxY4bOnj3r+emJgwcPVpMmTTRlyhRJ0qOPPqru3bvrpZde0u23366cnBx98MEHmj17djDXAICgIUcBwD/kKAD4hxwFgIsLegE7YMAAffPNN3rmmWeUn5+vDh06aM2aNZ4bch87dkwRET9+ULdr165asmSJnn76aY0fP17XXHON3nrrLbVp0yZYKwBAUJGjAOAfchQA/EOOAsDFBb2AlaTRo0dr9OjRVT6Xl5dX6Vj//v3Vv39/i6cCgNBBjgKAf8hRAPAPOQoA1QvqPWABAAAAAAAAIJxRwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwSGSwBwgGY4wkqaioqMbXuN1uFRcXq6ioSFFRUVaNFlDsFBrYyf583edCBl3IpFBCjv6AnUIDO9kfOVoz4fa+S+wUKtjJ/sjRmgm3911ip1DBTvZndY7WygL29OnTkqSmTZsGeRIA+CGT6tevH+wxvEKOArATchQA/EOOAoB/LpWjDhOK3+ryU3l5ub7++mvVq1dPDoejRtcUFRWpadOm+uKLLxQfH2/xhIHBTqGBnezP132MMTp9+rRSU1MVERFad4QhR3/ATqGBneyPHCVH2cne2Mn+yFFylJ3sjZ3sz+ocrZWfgI2IiNCVV17p07Xx8fFh8Qfrp9gpNLCT/fmyT6h90uACcrQidgoN7GR/5GjNhNv7LrFTqGAn+yNHaybc3neJnUIFO9mfVTkaWt/iAgAAAAAAAIAQQgELAAAAAAAAABahgK0hl8ul7OxsuVyuYI9y2bBTaGAn+wu3fawSjr9P7BQa2Mn+wm0fq4Tj7xM7hQZ2sr9w28cq4fj7xE6hgZ3sz+p9auUP4QIAAAAAAACAQOATsAAAAAAAAABgEQpYAAAAAAAAALAIBSwAAAAAAAAAWIQCFgAAAAAAAAAsQgELAAAAAAAAABahgP2JmTNnKj09XTExMerSpYt27Nhx0fOXLVumVq1aKSYmRm3bttXq1asDNGnNebPTnDlz1K1bNyUkJCghIUGZmZmX/D0IBm/fpwtycnLkcDjUt29fawf0gbc7FRYWatSoUWrcuLFcLpdatmxpqz9/3u4zY8YM/eIXv1BsbKyaNm2qMWPG6Ny5cwGa9tLee+899enTR6mpqXI4HHrrrbcueU1eXp6uv/56uVwuXX311Vq4cKHlc9oBOUqOBgs5So6GC3KUHA0WcpQcDRfkKDkaLOQoOXpRBsYYY3Jyckx0dLSZP3+++eSTT8xDDz1kGjRoYAoKCqo8f8uWLcbpdJoXX3zR7N271zz99NMmKirKfPTRRwGevHre7jRw4EAzc+ZMs3v3brNv3z5z//33m/r165svv/wywJNXz9udLjh8+LBp0qSJ6datm7nzzjsDM2wNebtTSUmJ6dSpk7ntttvM5s2bzeHDh01eXp7Zs2dPgCevmrf7LF682LhcLrN48WJz+PBhs3btWtO4cWMzZsyYAE9evdWrV5unnnrKrFixwkgyK1euvOj5hw4dMnFxcSYrK8vs3bvXvPLKK8bpdJo1a9YEZuAgIUfJ0WAhR8nRcEGOkqPBQo6So+GCHCVHg4UcJUcvhQL2/+vcubMZNWqU53FZWZlJTU01U6ZMqfL8e+65x9x+++0VjnXp0sUMHz7c0jm94e1OP1daWmrq1atnFi1aZNWIXvNlp9LSUtO1a1czd+5cM2TIENsFtbc7vf7666Z58+bm/PnzgRrRK97uM2rUKHPTTTdVOJaVlWVuuOEGS+f0VU2C+n/+53/MddddV+HYgAEDTO/evS2cLPjI0crI0cAgR8nRcEGOVkaOBgY5So6GC3K0MnI0MMhRcvRSuAWBpPPnz2vnzp3KzMz0HIuIiFBmZqa2bdtW5TXbtm2rcL4k9e7du9rzA82XnX6uuLhYbrdbDRs2tGpMr/i605/+9CclJSVp2LBhgRjTK77s9Pe//10ZGRkaNWqUkpOT1aZNGz3//PMqKysL1NjV8mWfrl27aufOnZ5/znDo0CGtXr1at912W0BmtoLd88EK5GjVyFHrkaPkaLggR6tGjlqPHCVHwwU5WjVy1HrkKDlaE5GXY6hQd+LECZWVlSk5ObnC8eTkZH366adVXpOfn1/l+fn5+ZbN6Q1fdvq5J554QqmpqZX+wAWLLztt3rxZ8+bN0549ewIwofd82enQoUPauHGjfve732n16tU6ePCgRo4cKbfbrezs7ECMXS1f9hk4cKBOnDihG2+8UcYYlZaW6pFHHtH48eMDMbIlqsuHoqIiff/994qNjQ3SZNYhR6tGjlqPHCVHwwU5WjVy1HrkKDkaLsjRqpGj1iNHydGa4BOwqNLUqVOVk5OjlStXKiYmJtjj+OT06dMaNGiQ5syZo8TExGCPc9mUl5crKSlJs2fPVseOHTVgwAA99dRTmjVrVrBH80leXp6ef/55vfbaa9q1a5dWrFihVatWafLkycEeDfALOWpf5CgQGshR+yJHgdBAjtoXOVr78AlYSYmJiXI6nSooKKhwvKCgQCkpKVVek5KS4tX5gebLThdMmzZNU6dO1fr169WuXTsrx/SKtzt9/vnnOnLkiPr06eM5Vl5eLkmKjIzU/v371aJFC2uHvgRf3qfGjRsrKipKTqfTc+zaa69Vfn6+zp8/r+joaEtnvhhf9pkwYYIGDRqkBx98UJLUtm1bnT17Vg8//LCeeuopRUSE3veJqsuH+Pj4sPy0gUSO/hw5GjjkKDkaLsjRisjRwCFHydFwQY5WRI4GDjlKjtZE6P0OWCA6OlodO3bUhg0bPMfKy8u1YcMGZWRkVHlNRkZGhfMlKTc3t9rzA82XnSTpxRdf1OTJk7VmzRp16tQpEKPWmLc7tWrVSh999JH27Nnj+XXHHXeoR48e2rNnj5o2bRrI8avky/t0ww036ODBg56/dCTpwIEDaty4cVBDWvJtn+Li4kphfOEvoR/ujR167J4PViBHf0SOBhY5So6GC3L0R+RoYJGj5Gi4IEd/RI4GFjlKjtaITz+6Kwzl5OQYl8tlFi5caPbu3Wsefvhh06BBA5Ofn2+MMWbQoEFm3LhxnvO3bNliIiMjzbRp08y+fftMdna2iYqKMh999FGwVqjE252mTp1qoqOjzfLly81///tfz6/Tp08Ha4VKvN3p5+z40xK93enYsWOmXr16ZvTo0Wb//v3mnXfeMUlJSebZZ58N1goVeLtPdna2qVevnvnrX/9qDh06ZNatW2datGhh7rnnnmCtUMnp06fN7t27ze7du40kM336dLN7925z9OhRY4wx48aNM4MGDfKcf+jQIRMXF2fGjh1r9u3bZ2bOnGmcTqdZs2ZNsFYICHKUHA0WcpQcDRfkKDkaLOQoORouyFFyNFjIUXL0Uihgf+KVV14xaWlpJjo62nTu3Nls377d81z37t3NkCFDKpz/t7/9zbRs2dJER0eb6667zqxatSrAE1+aNztdddVVRlKlX9nZ2YEf/CK8fZ9+yo5BbYz3O23dutV06dLFuFwu07x5c/Pcc8+Z0tLSAE9dPW/2cbvdZuLEiaZFixYmJibGNG3a1IwcOdKcPHky8INXY9OmTVX+f+PCHkOGDDHdu3evdE2HDh1MdHS0ad68uVmwYEHA5w4GcpQcDRZylBwNF+QoORos5Cg5Gi7IUXI0WMhRcvRiHMaE6GeBAQAAAAAAAMDmuAcsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAAAAAACARShgAQAAAAAAAMAiFLAAAAAAAAAAYBEKWAAAAAAAAACwCAUsAAAAAAAAAFiEAhYAAAAAAAAALEIBCwAAAAAAAAAWoYAFAAAAAAAAAItQwAIAAKBG0tPT9Zvf/CbYYwAAAAAhhQIWAAAAtlVcXKyJEycqLy8v2KMAAAAAPqGABQAAgG0VFxdr0qRJFLAAAAAIWRSwAAAAtdjZs2eDPQIAAAAQ1ihgAQAAaomJEyfK4XBo7969GjhwoBISEnTjjTeqtLRUkydPVosWLeRyuZSenq7x48erpKSkytdZt26dOnTooJiYGLVu3VorVqyo8uv83MKFC+VwOHTkyBHPsQ8++EC9e/dWYmKiYmNj1axZMz3wwAOSpCNHjqhRo0aSpEmTJsnhcMjhcGjixImSpPvvv19169bVV199pb59+6pu3bpq1KiR/vjHP6qsrKzC1y4vL9eMGTN03XXXKSYmRsnJyRo+fLhOnjxZ4byLzXNBTk6OOnbsqHr16ik+Pl5t27bVyy+/fOk3AAAAALVSZLAHAAAAQGD1799f11xzjZ5//nkZY/Tggw9q0aJFuvvuu/X444/r3//+t6ZMmaJ9+/Zp5cqVFa797LPPNGDAAD3yyCMaMmSIFixYoP79+2vNmjXq2bOnV3McP35cvXr1UqNGjTRu3Dg1aNBAR44c8RS6jRo10uuvv64RI0aoX79++u1vfytJateunec1ysrK1Lt3b3Xp0kXTpk3T+vXr9dJLL6lFixYaMWKE57zhw4dr4cKFGjp0qP7whz/o8OHDevXVV7V7925t2bJFUVFRl5xHknJzc3Xvvffq5ptv1gsvvCBJ2rdvn7Zs2aJHH33UuzcCAAAAtQIFLAAAQC3Tvn17LVmyRJL04YcfatSoUXrwwQc1Z84cSdLIkSOVlJSkadOmadOmTerRo4fn2gMHDujNN9/0lKHDhg1Tq1at9MQTT3hdwG7dulUnT57UunXr1KlTJ8/xZ599VpJUp04d3X333RoxYoTatWun++67r9JrnDt3TgMGDNCECRMkSY888oiuv/56zZs3z1PAbt68WXPnztXixYs1cOBAz7U9evTQLbfcomXLlmngwIGXnEeSVq1apfj4eK1du1ZOp9OrfQEAAFA7cQsCAACAWuaRRx7x/O/Vq1dLkrKysiqc8/jjj0v6oXD8qdTUVPXr18/zOD4+XoMHD9bu3buVn5/v1RwNGjSQJL3zzjtyu91eXftTP91Hkrp166ZDhw55Hi9btkz169dXz549deLECc+vjh07qm7dutq0aVON52nQoIHOnj2r3Nxcn+cFAABA7UIBCwAAUMs0a9bM87+PHj2qiIgIXX311RXOSUlJUYMGDXT06NEKx6+++upK93dt2bKlJFW4t2tNdO/eXXfddZcmTZqkxMRE3XnnnVqwYEG1956tSkxMjOc+sRckJCRUuLfrZ599plOnTikpKUmNGjWq8OvMmTM6fvx4jecZOXKkWrZsqVtvvVVXXnmlHnjgAa1Zs8arvQEAAFC7cAsCAACAWiY2NrbSsap+aJavqnutn/9gLIfDoeXLl2v79u36xz/+obVr1+qBBx7QSy+9pO3bt6tu3bqX/Fo1uQ1AeXm5kpKStHjx4iqfv1Dg1mSepKQk7dmzR2vXrtW7776rd999VwsWLNDgwYO1aNGiS84CAACA2odPwAIAANRiV111lcrLy/XZZ59VOF5QUKDCwkJdddVVFY4fPHhQxpgKxw4cOCBJSk9Pl/TDJ1AlqbCwsMJ5P/807QW/+tWv9Nxzz+mDDz7Q4sWL9cknnygnJ0fS5SmGW7RooW+//VY33HCDMjMzK/1q3759jeeRpOjoaPXp00evvfaaPv/8cw0fPlxvvPGGDh486PesAAAACD8UsAAAALXYbbfdJkmaMWNGhePTp0+XJN1+++0Vjn/99ddauXKl53FRUZHeeOMNdejQQSkpKZJ+KDwl6b333vOcd/bs2UqfED158mSlMrdDhw6S5Pln/3FxcZIql7neuOeee1RWVqbJkydXeq60tNTz2jWZ59tvv63wfEREhNq1a1fhHAAAAOCnuAUBAABALda+fXsNGTJEs2fPVmFhobp3764dO3Zo0aJF6tu3r3r06FHh/JYtW2rYsGF6//33lZycrPnz56ugoEALFizwnNOrVy+lpaVp2LBhGjt2rJxOp+bPn69GjRrp2LFjnvMWLVqk1157Tf369VOLFi10+vRpzZkzR/Hx8Z5iODY2Vq1bt9bSpUvVsmVLNWzYUG3atFGbNm1qvGP37t01fPhwTZkyRXv27FGvXr0UFRWlzz77TMuWLdPLL7+su+++u0bzPPjgg/ruu+9000036corr9TRo0f1yiuvqEOHDrr22mv9eSsAAAAQpihgAQAAarm5c+eqefPmWrhwoVauXKmUlBQ9+eSTys7OrnTuNddco1deeUVjx47V/v371axZMy1dulS9e/f2nBMVFaWVK1dq5MiRmjBhglJSUvTYY48pISFBQ4cO9Zx3oezNyclRQUGB6tevr86dO2vx4sUVflDY3Llz9fvf/15jxozR+fPnlZ2d7VUBK0mzZs1Sx44d9Ze//EXjx49XZGSk0tPTdd999+mGG26o8Tz33XefZs+erddee02FhYVKSUnRgAEDNHHiREVE8I/LAAAAUJnD/PzfWQEAAAAAAAAALgu+TQ8AAAAAAAAAFqGABQAAAAAAAACLUMACAAAAAAAAgEUoYAEAAAAAAADAIhSwAAAAAAAAAGARClgAAAAAAAAAsAgFLAAAAAAAAABYhAIWAAAAAAAAACxCAQsAAAAAAAAAFqGABQAAAAAAAACLUMACAAAAAAAAgEUoYAEAAAAAAADAIv8P9wptfGMZPU4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_changes(r_pthr, param_name='consec_preds', height=3.5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we may see on the plot above, the edge value of *consecutive_predictions* = 1 results in acceptance of all the prediction made. And main influence of this parameter is observed in the middle range of predictors. The accuracy of predictions doesn't sigificantly change, whereas the common trend is left shift resulting in lesser proportion of accepted labels." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Teaser " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another model exploiting the consecutive labels count is TEASER. But the way to prove acceptance of prediction is a bit more elaborate: instead of simple thresholding the evaluation mechanism is OneClass SVM which is trained for every prediction point on correct predictions. The features for their fitting are class probabilities with addition of most close proba differences for every prediction. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot_ind.core.models.early_tc import teaser as TEASER\n", + "\n", + "cons_preds = [1, 3, 5, 7]\n", + "r_teaser = eval_param_influence(TEASER.TEASER,\n", + " 'consecutive_predictions', cons_preds,\n", + " prediction_mode='all', interval_percentage=INTEVAL_PERCENTAGE)" + ] + }, + { + "cell_type": "code", + "execution_count": 200, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABiUAAAHgCAYAAADKeW7zAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAC6+ElEQVR4nOzdd3hb5dnH8e/RsOTtOPF2duLsHbLJ3iFACxQoBcpoSym7fVtCWW2ZZRQKFDpYLVBmIZCEELKXQwYJ2dOxs+zYieNtyxrn/cNExCSQOLEsy/59rsuXrTN07udY0i3pPs/zGKZpmoiIiIiIiIiIiIiIiASYJdgBiIiIiIiIiIiIiIhI86CihIiIiIiIiIiIiIiINAgVJUREREREREREREREpEGoKCEiIiIiIiIiIiIiIg1CRQkREREREREREREREWkQKkqIiIiIiIiIiIiIiEiDUFFCREREREREREREREQahIoSIiIiIiIiIiIiIiLSIFSUEBERERERERERERGRBqGihEgT89prr2EYBtnZ2cEORUREgkj5QEREQPlARERqKB9IY6KihIg0iB07dnDnnXcybNgwnE6nEqGISDP14YcfMmnSJFJTU3E4HKSnp3PppZeyefPmYIcmIiIN6MEHH8QwjJN+nE5nsEMTEZEG1K5du1PmA8Mw6Ny5c7DDkwCxBTsAEWkeMjMz+etf/0r37t3p1q0bGzZsCHZIIiISBJs2baJFixbcfvvttGrViry8PF555RUGDRpEZmYmffr0CXaIIiLSgF588UWioqL8t61WaxCjERGRhvbMM89QVlZWa1lOTg733nsvEydODFJUEmgqSog0Eh6PB5/PR1hYWLBDCYgLL7yQoqIioqOjefLJJ1WUEBH5Dk09H9x///0nLbvxxhtJT0/nxRdf5KWXXgpCVCIijU9TzwfHXXrppbRq1SrYYYiINFpNPR9cfPHFJy176KGHALjqqqsaOBppKBq+SRqlgwcPcsMNN/iHdmjfvj2//OUvqa6u9m+TlZXFZZddRnx8PBEREQwZMoTZs2fXup/FixdjGAbvvvsuDz/8MOnp6TidTsaNG8fu3btrbbtr1y4uueQSkpOTcTqdpKenc8UVV1BcXHzGcf/0pz8lKiqKrKwsJk2aRGRkJKmpqfzxj3/ENE3/dtnZ2RiGwZNPPskzzzxDx44dcTgcbN26FYDt27dz6aWXEh8fj9PpZODAgXz88ccnHW/Lli2MHTuW8PBw0tPTeeihh/D5fCdtt3btWiZNmkSrVq0IDw+nffv2XH/99WfcrvoQHx9PdHR0gx5TREKf8kHTywenkpiYSEREBEVFRcEORUQaKeWDppsPTNOkpKSk1vkQEfkuygdNNx+c6K233qJ9+/YMGzYs2KFIgKinhDQ6hw4dYtCgQRQVFfHzn/+crl27cvDgQd5//30qKioICwvj8OHDDBs2jIqKCm677TZatmzJ66+/zoUXXsj777/PD37wg1r3+dhjj2GxWPjNb35DcXExf/7zn7nqqqv44osvAKiurmbSpEm4XC5uvfVWkpOTOXjwILNmzaKoqIjY2Ngzjt/r9TJ58mSGDBnCn//8Z+bOncsDDzyAx+Phj3/8Y61tX331Vaqqqvj5z3+Ow+EgPj6eLVu2MHz4cNLS0rj77ruJjIzk3Xff5eKLL+aDDz7wty0vL48xY8bg8Xj82/3jH/8gPDy81jHy8/OZOHEiCQkJ3H333cTFxZGdnc3//ve/07alrKyMqqqq025nt9vrdI5ERM6E8kHTzgdFRUW43W7y8vJ45plnKCkpYdy4cWe0r4g0L8oHTTsfdOjQgbKyMiIjI7n44ot56qmnSEpKOqN9RaR5UT5o2vnguPXr17Nt2zZ+//vf12k/CTGmSCNzzTXXmBaLxVyzZs1J63w+n2mapnnHHXeYgLls2TL/utLSUrN9+/Zmu3btTK/Xa5qmaS5atMgEzG7dupkul8u/7bPPPmsC5qZNm0zTNM3169ebgPnee++dU+zXXnutCZi33nprrZinTZtmhoWFmQUFBaZpmubevXtNwIyJiTHz8/Nr3ce4cePMXr16mVVVVbXuY9iwYWbnzp39y46fgy+++MK/LD8/34yNjTUBc+/evaZpmuaHH35oAqc8n2fantP9jBo1qk73+8QTT9SKUUTkVJQPmnY+6NKli3+/qKgo89577/X/v0RETqR80DTzwTPPPGPecsst5ptvvmm+//775u23327abDazc+fOZnFxcZ1jE5GmT/mgaeaDb/v1r39tAubWrVvrvK+EDvWUkEbF5/Px0UcfMX36dAYOHHjSesMwAJgzZw6DBg1ixIgR/nVRUVH8/Oc/Z8aMGWzdupWePXv611133XW1xt47//zzgZoufT179vRXbT/77DOmTp1KRETEObXjlltuqRXzLbfcwuzZs5k/fz5XXHGFf90ll1xCQkKC/3ZhYSELFy7kj3/8I6WlpZSWlvrXTZo0iQceeICDBw+SlpbGnDlzGDJkCIMGDfJvk5CQwFVXXcXf/vY3/7K4uDgAZs2aRZ8+fbDb7Wfcjt/+9rf85Cc/Oe12LVq0OOP7FBE5E8oHTT8fvPrqq5SUlJCVlcWrr75KZWUlXq8Xi0Wji4rIN5QPmm4+uP3222vdvuSSSxg0aJA/3rvvvvuM4xKRpk/5oOnmgxP5fD7efvtt+vXrR7du3eq0r4QWFSWkUSkoKKCkpKRWgjiVnJwcBg8efNLy4y9YOTk5te6jTZs2tbY7/qJ47NgxANq3b89dd93F008/zZtvvsn555/PhRdeyE9+8pM6dzOzWCx06NCh1rKMjAygZmzAE7Vv377W7d27d2OaJvfddx/33XffKe8/Pz+ftLS07zwHXbp0qXV71KhRXHLJJfzhD3/gL3/5C6NHj+biiy/mxz/+MQ6H43vb0r17d7p37/6924iIBILyQdPPB0OHDvX/fcUVV/j/Z08++WS9HkdEQpvyQdPPByf68Y9/zK9//Wvmz5+vooSI1KJ80DzywZIlSzh48CB33nlnvd+3NC4qSkizYLVaT7ncPGEyoaeeeoqf/vSnzJw5k3nz5nHbbbfx6KOPsmrVKtLT0wMS17fH8zs+6dBvfvMbJk2adMp9OnXqVKdjGIbB+++/z6pVq/jkk0/47LPPuP7663nqqadYtWoVUVFR37lvcXExlZWVpz1GWFgY8fHxdYpLRCQYlA8aZz5o0aIFY8eO5c0331RRQkQahPJB48wHAK1bt6awsPCs9hURqSvlg8aVD958800sFgtXXnnlGe8joUlFCWlUEhISiImJYfPmzd+7Xdu2bdmxY8dJy7dv3+5ffzZ69epFr169uPfee1m5ciXDhw/npZde4qGHHjrj+/D5fGRlZfmr3QA7d+4EoF27dt+77/GKud1uZ/z48d+7bdu2bdm1a9dJy091XgCGDBnCkCFDePjhh3nrrbe46qqrePvtt7nxxhu/8xi33347r7/++vfGATXV9cWLF592OxGRM6V80PzyQWVlJcXFxWe1r4g0XcoHzSsfmKZJdnY2/fr1q/O+ItK0KR80/Xzgcrn44IMPGD16NKmpqWe0j4QuFSWkUbFYLFx88cW88cYbrF279qRxAk3TxDAMpk6dyjPPPENmZqZ/+Ify8nL+8Y9/0K5duzp3ISspKSEiIgKb7ZunRK9evbBYLLhcrjq34/nnn+evf/2rP+bnn38eu93OuHHjvne/xMRERo8ezd///nduvfVWUlJSaq0vKCjwjyl4/BysXr3aP05gQUEBb775Zq19jh07RlxcnH98RYC+ffsCnLZtmlNCRIJF+aDp5oP8/HwSExNrLcvOzmbBggWnHB9YRJo35YOmmw9OjP24F198kYKCAiZPnnza/UWkeVE+aLr54Lg5c+ZQVFTEVVdddcb7SOhSUUIanUceeYR58+YxatQofv7zn9OtWzdyc3N57733WL58OXFxcdx9993897//ZcqUKdx2223Ex8fz+uuvs3fvXj744IM6T5K5cOFCbrnlFi677DIyMjLweDz85z//wWq1cskll9TpvpxOJ3PnzuXaa69l8ODBfPrpp8yePZt77rnnpDfdp/LCCy8wYsQIevXqxc9+9jM6dOjA4cOHyczM5MCBA3z11VdATQL4z3/+w+TJk7n99tuJjIzkH//4B23btmXjxo3++3v99df529/+xg9+8AM6duxIaWkp//znP4mJiWHq1KnfG0t9jhFYXFzMc889B8CKFSuAmmQcFxdHXFxcrcmeRERA+aCp5oNevXoxbtw4+vbtS4sWLdi1axcvv/wybrebxx57rF6OISJNi/JB08wHbdu25fLLL6dXr144nU6WL1/O22+/Td++ffnFL35RL8cQkaZF+aBp5oPj3nzzTRwOR53Pq4QoU6QRysnJMa+55hozISHBdDgcZocOHcxf/epXpsvl8m+zZ88e89JLLzXj4uJMp9NpDho0yJw1a1at+1m0aJEJmO+9916t5Xv37jUB89VXXzVN0zSzsrLM66+/3uzYsaPpdDrN+Ph4c8yYMeb8+fPrFPe1115rRkZGmnv27DEnTpxoRkREmElJSeYDDzxger3ek47/xBNPnPJ+9uzZY15zzTVmcnKyabfbzbS0NPOCCy4w33///Vrbbdy40Rw1apTpdDrNtLQ0809/+pP58ssvm4C5d+9e0zRN88svvzSvvPJKs02bNqbD4TATExPNCy64wFy7dm2d2naujrf5VD9t27Zt0FhEJHQoHzS9fPDAAw+YAwcONFu0aGHabDYzNTXVvOKKK8yNGzc2aBwiElqUD5pePrjxxhvN7t27m9HR0abdbjc7depk/u53vzNLSkoaNA4RCS3KB00vH5imaRYXF5tOp9P84Q9/2ODHluAwTPOEmVtE5Jz89Kc/5f3336esrCzYoYiISBApH4iICCgfiIhIDeUDkdrq1mdJRERERERERERERETkLGlOCZEzUFxcTGVl5fduk5yc3EDRiIhIsCgfiIgIKB+IiEgN5QORs6OihMgZuP3223n99de/dxuNhCYi0vQpH4iICCgfiIhIDeUDkbOjOSVEzsDWrVs5dOjQ924zfvz4BopGRESCRflARERA+UBERGooH4icHRUlRERERERERERERESkQWiiaxERERERERERERERaRDNYk4Jn8/HoUOHiI6Oxuv14nK5gh2SNEPh4eFYLKoDSugzTZPS0lJSU1ND7jF9Yj4wDCPY4YiIhDTlAxERAeUDERGpUZd80CyKEocOHaJt27Y899xznHfeeVit1mCHJM1QZWUlN954I9u3bw92KCL1Yv/+/aSnpwc7jDo5dOgQrVu3DnYYIiJNivKBiIiA8oGIiNQ4k3zQLIoS0dHRPPfcc4wYMYLk5GQiIiLqVAH3er3NspChdtcfn8/HwYMHeeedd2jTpk2ju3rE7XYzb948Jk6ciN1uD3Y4DUbtPrt2l5SU0Lp1a6KjowMQXWAdj3n//v3ExMTUaV89XtTu5kDtVrvrQvlAj5fmQO1Wu5sD5QPlg7pQu9Xu5kDtDnw+aBZFCY/Hw3nnnUdycjKJiYl12tc0Tf+X1M2pK5/aXf/tTkxMZP/+/TgcDsLDw+v1vs+V2+0mIiKCmJiYZvdiq3affbtD8bXheMwxMTFn9aFDjxe1u6lTu9Xus6F80Dyo3Wp3c6B2Kx/UhR4vandzoHar3WfjTPJB47pcO0Cqq6uxWq1EREQEOxRpxhwOB4ZhUF1dHexQRERERERERERERIKiWRQljgvFqr00HXr8iYiIiIiIiIiISHPXrIoSIiIiIiIiIiIiIiISPCpKNANpaWn86U9/8t82DIM33ngjiBGJiIiIiIiIiIiISHOkokQztG/fPi655JJgh3HGHnjgAeLj44mPj+fBBx+stW7RokX06NEDt9sdnOBERERERERERERE5IzZgh2ABE5VVRVOp/Ok5a1btw5CNGfniy++4PHHH+e9997DNE1+9KMfMXXqVAYNGoTb7ebmm2/m73//+znNCC8iIiIiIiIiIiIiDUM9JRoJr9fLPffcQ1paGk6nky5duvDaa6/513s8Hi6//HL/+vbt2/PQQw/Vuo9LL72UCRMmcPfdd5OYmEjHjh1PeawTh2/asWMHhmHw73//m8GDB/uPvXDhwlr7zJs3j4EDB+J0OklOTua6666jpKTEv/7xxx+nbdu2OBwOWrZsyeTJk/3rXnvtNTIyMnA6ncTFxTFs2LBa+36fLVu20KVLF6ZPn86FF15IRkYGW7ZsAWp6UAwdOpSRI0ee0X2JiIiIiIiIiIiISHCpp0Qj8fvf/553332X5557jm7dujF//nx+8YtfkJSUxJQpU/B6vaSmpvL222+TkJDA4sWLufPOO0lNTeX666/338/KlSuJjo7m008/rdPxH3zwQR599FG6d+/Ob3/7W6655hp2796N1Wpl69atXHzxxcyYMYPXXnuNvLw8brvtNq6//nref/99li1bxu9//3v+9re/MXr0aI4cOcKiRYsAyMnJ4cYbb+T+++/n8ssvp7i4mEWLFmGa5hnF1a9fP7Kzs9m1axemaZKdnU3fvn3ZunUrb731Fhs2bKhTO0VEREREREREREQkeFSUaAQqKyt59tlnmTVrFuPGjQOgW7duLF++nBdffJEpU6bgcDj4y1/+4t+na9euZGZm8u6779YqSoSHh/PWW2+dctim73Pbbbdx+eWXA/Dwww8zYMAAtm3bRr9+/fjDH/7AxRdfzH333QdAz549eeaZZ5gyZQoVFRXs3buX8PBwfvSjHxEXF0dGRgbDhg0D4MCBA3i9Xq644goyMjIAGDRo0BnH1a9fP+69914mTpwIwH333Ue/fv0YNmwYDz30EB999BEPP/wwNpuNv/zlL7V6aIiIiIiIiIiIiIhI46KiRCOwdetWqqqqmD59eq3lbrebbt26+W8/9thj/Oc//+HQoUO4XK6T1gN06dKlzgUJgP79+/v/Pj7nRG5uLv369WPLli3s3LmTjz76yL+NaZr4fD527NjBhRdeyEMPPUT79u0ZPXo0kyZN4qqrriI6OprBgwczdOhQ+vfvz/nnn8+ECRO4+uqrSUhIOOPY/u///o//+7//899+/vnniYqKYvTo0XTr1o3MzEyys7O55ppryMnJITw8vM7tFxEREREREREREZHAa/A5JZYuXcr06dNJTU3FMIxaX3R/l8WLF9O/f38cDgedOnWqNddCU3B8foUPPviANWvW+H82bNjAhx9+CMC//vUvHnzwQa655hpmz57NmjVruOyyy6iurq51XxEREWcVw4kTRRuGAeAfYqmiooKrrrqqVmxr165l8+bNdOvWjbi4OLZs2cJrr71GcnIyDz/8MD179uTIkSPYbDaWL1/Ohx9+SNeuXXnppZfo0qUL27dvP6s4c3Nzeeyxx3jppZdYunQp7du3p2fPnlxwwQW43W42bdp0VvcrIg1P+UBERED5QEREaigfiIg0Hw1elCgvL6dPnz688MILZ7T93r17mTZtGmPGjGHDhg3ccccd3HjjjXz22WcBjrTh9OvXj7CwMLKzs+nRo0etn+OTVS9fvpx+/frxu9/9jmHDhtGjRw+ys7MbJL5evXqxY8eOk2Lr0aOHv1eG3W7noosu4sUXX2TLli0cPHiQOXPmAGCxWJgwYQJ/+ctf2Lp1K3a7nbfffvusYrn55pv55S9/SYcOHfB6vbjdbv86r9eL1+s99waLSINQPhAREVA+EBGRGsoHIiLNR4MP3zRlyhSmTJlyxtu/9NJLtG/fnqeeegr4Zq6Fv/zlL0yaNClQYTaouLg4brrpJu655x68Xi9jx47l2LFjLF68mNjYWG655RY6d+7MBx98wP/+9z86d+7Myy+/zKZNm0hLSwt4fPfccw+jRo3i2muv5aabbiI6OpqNGzfy2Wef8frrr/P222+zZ88exo4dS8uWLfnoo48wTZMePXqwaNEi5s2bx9SpU0lOTmb58uUcO3aMnj171jmOjz76iD179vD+++8DMHz4cPbu3cv7779PTk4OVquV3r1713fzRSRAlA9ERASUD0REpIbygYhI89Ho55TIzMxk/PjxtZZNmjSJO+644zv3cblcuFwu/+2Kigr/38eHJDob57Lv6fzlL38hISGBp556ijvvvJPo6Gh69OjB73//e0zT5K677mL9+vVce+21GIbBRRddxLXXXsv8+fNPiuu74jzddt+136BBg/jss8/4/e9/z/jx4zFNk9atW/PDH/4Q0zSJj4/n6aef5s9//jMul4u2bdvyz3/+k/79+7NhwwaWL1/O3//+d8rLy0lNTeXBBx/kkksuwTRN5syZwwUXXMD27dv9E2GfSnl5OXfeeSdvvfUWFosF0zRp3749jz76KDfddBNhYWG89NJLRERE1Nv/qb7/38fvz+Px1Orh0Rgcj6exxRVoavfZtTtY56s+8sHx4fLcbned26HHi9rdHKjdavfZ7N/QlA+CQ+1Wu5sDtVv5oC70eFG7mwO1W+0+m/3PRKMvSuTl5ZGUlFRrWVJSEiUlJVRWVp5yUuNHH32UP/zhD/7bw4YN47nnnjun4X0aYligu+++m7vvvvuUx/6+IY+Ox3Z8/bdjzcnJqbX8+APE6/XSsWPHWrehpufGt5cNHz6cxYsXn/LYY8eOZeXKladc16tXLxYtWvSdMe/evZs2bdrQunXr7z3HTqeTXbt2ndS+W2+9lVtvvfWk+z1Xgfh/+3w+TNNkzZo1td4ENSaff/55sEMICrW7bk4s9Dak+sgHx82bN++s5+DR46V5UbubF7W7bpQP9HhpTtTu5kXtrhvlAz1emhO1u3lRu+umLvmg0RclzsaMGTO46667/LcLCgooLi7GarVitVrrfH9er/es9gt1DdHuuXPn8uCDD57yzUKwBKrdFosFwzA477zziImJqff7Pxdut5vPP/+cCRMm1Jr0vKlTu8+u3cevJgoF384HJSUltG7dmokTJ9b5eajHi9rdHKjdanddKB/o8dIcqN1qd3OgfKB8UBdqt9rdHKjdgc8Hjb4okZyczOHDh2stO3z4MDExMd/5RbbD4cDhcPhvl5eXU1xcDIBhGHU6/olD+NR131DWUO3+9NNPA3bfZyOQ7T5+fzabrdG+oNnt9kYbWyCp3XXfLxjqIx8cdy7/cz1emhe1u3lRu+u+XzAoHwSX2t28qN3Ni/KB8kFdqN3Ni9rdvDREPrDU+d4b2NChQ1mwYEGtZZ9//jlDhw4NUkQiIhIMygciIgLKByIiUkP5QEQkdDV4UaKsrIwNGzawYcMGAPbu3cuGDRvYt28fUNOV7pprrvFvf9NNN5GVlcVvf/tbtm/fzt/+9jfeffdd7rzzzoYOXURE6pHygYiIgPKBiIjUUD4QEWk+GrwosXbtWvr160e/fv0AuOuuu+jXrx/3338/ALm5uf6EA9C+fXtmz57N559/Tp8+fXjqqaf417/+xaRJkxo6dBERqUfKByIiAsoHIiJSQ/lARKT5aPA5JUaPHl1r3P5ve+211065z/r16wMYlYiINDTlAxERAeUDERGpoXwgItJ8NPo5JUREREREREREREREpGlQUUJERERERERERERERBqEihIiIiIiIiIiIiIiItIgVJQQEREREREREREREZEGoaJEHXi9PrZszmfFsn1s2ZyP1+sL+DHnzp3L2LFjSUxMxDAM3njjjZO2eeyxx0hLS8PhcNC7d2+WLFkS8Ljqi8/rpSrnIBVbd1GVcxCf19sgx927dy8XX3wxcXFxOJ1OMjIyWLZsmX/9448/HrLnVERERERERERERKSxsgU7gFCxetVB/v36RnbtLMTl8uBw2OicEc81P+3N4CHpATtuWVkZvXr14rrrruOaa645af3LL7/M/fffz5NPPsmIESN44oknmD59Otu2bSMtLS1gcdWHip1ZFC9cSXVuPqbbg2G3EZaSSOzYYURkdAjYcQsKChgxYgRDhw5l5syZJCUlsW3bNlq2bAnAq6++GrLnVERERERERERERKQxU0+JM7Bm9SEeeWg5mzYeJi7OQfv2ccTFOdi08TCP/Gk5X6w6ELBjX3rppTz77LNcffXVp1z/7LPPcuWVV3LbbbfRv39/3njjDZxOJy+88ELAYqoPFTuzOPrubFw5B7FGRmBPaok1MgJXzkGOvjubip1ZATv2gw8+SEpKCu+//z6jRo2ia9eu/OAHP6B79+4APPfccyF5TkVEREREREREREQaOxUlTsPnM/nP65s4dqySTp3iiY5xYLVZiI5x0KlTPMeOVfKf1zY2yFBO31ZVVcXWrVuZMGGCf5nVauX8889n9erVDR7PmfJ5vRQvXIm3vAJ7SgKWcAeGxYIl3IE9JQFveQXFizIDNpTT3Llz6du3L1OmTCE+Pp5u3brx9NNPA+ByuULynIqIiIiIiIiIiIiEAhUlTmP7tgJ27yokKTkKw2LUWmdYDJKTo9i5s5Dt2440eGx5eXl4vV5SUlJqLU9MTCQ/P7/B4zlT1QfyqM7Nx9YiFsP41jk1DGwtYqk+dJjqA3kBOf6BAwf4z3/+Q8eOHZk1axY33ngj99xzD88//3zInlMRERERERERERGRUKA5JU6j6JgLl8tLRMSpT1V4hJ3Dh8spOlbVwJGFLl95Rc0cEg77KdcbYXbMIg++8orAHN/no2fPnjz//PMADBs2jM2bN/Ovf/2LCy64ICDHFBERERERERERERH1lDituBYOHA4rFRWeU66vrHDjcNiIa+Fs4MggOTkZq9VKbm5ureX5+fkkJiY2eDxnyhIZgWG3Ybrcp1xvVrsx7DYskREBOX5CQgIZGRm1lnXr1o1Dhw6F7DkVERERERERERERCQUqSpxG124JdOocz+G8MkyfWWud6TPJyysjIyOert1aNXhsTqeT7t27M3/+fP8yr9fL8uXLGTRoUIPHc6bC0pMJS0nEc6wY0/zWOTVNPMeKCUtNIiw9OSDHHzhwIHv27Km1bOfOnaSlpeFwOELynIqIiIiIiIiIiIiEAhUlTsNiMbj62l60aBHO7t2FlJa48Hh8lJa42L27kBYtwrn6p72xWgNzKouLi8nMzCQzMxOArKwsMjMz2bVrFwC33347//3vf3n++edZv349V199NZWVldx8880Biac+WKxWYscOwxoZgTu3AF+lC9Prw1fpwp1bgDUqktgxQ7FYrQE5/m9+8xs2bNjAjBkz2LJlC3//+9958803+cUvfgHArbfeGnLnVERERERERERERCQUaE6JM3DeoFTuuXcE/359I7t2FnL4cDkOh43evZO4+qe9GTwkPWDHXrFiBdOmTfPffuCBB3jggQe45JJLeP/997nhhhvIz8/nkUce4ciRI3Tt2pWZM2eSnh64mOpDREYH+NE0iheupDo3H7PIg2G34WiXTuyYoTXrA2TkyJG88cYb3H///Tz99NOkpaXx8MMPc9NNN2GaJtdddx0FBQUhd05FREREREREREREGjsVJc7QoCFpDByUyvZtRyg6VkVcCyddu7UKWA+J46ZOnXrSEEffNmPGDGbMmBHQOAIhIqMDzo5tqT6Qh6+8AktkBGHpyQHrIXGiK664giuuuOI718+YMYN77rkn4HGIiIiIiIiIiIiINCcqStSB1WqhR09NdlyfLFYrzrZpwQ5DRERERERERERERBqA5pQQEREREREREREREZEGoaKEiIiIiIiIiIiIiIg0CBUlRERERERERERERESkQWhOCREREREREZFz5PX62LQxn8oKN506x5OQGBnskEREREQaJRUlRERERERERM6SaZq8/upXPPbICg4dLAXAajW48KIMHntiPCmp0UGOUERERKRx0fBNIiIiIiIiImfpicdXctuv5voLEgBer8nHM3cydtS/yT9cHsToRERERBofFSVEREREREREzsLBAyU8/Mdlp1zn9Zrk5Zbx5J9XNnBUIiIiIo2bihIiIiIiIiIiZ+HNNzZhsRj+2z5fIV7fQUyzGqgpTPzn9Y14PL5ghSgiImfp6JEKHv7TMrp0fJ52ac8C8JenMiksrAxyZCKhT0UJERERETljx45VsWF9Hrt2HsU0zWCHIyISVDnZxRjGN0UJr3mwpijBN19YlZe7KTpWFYzwRETkLB3YX8KIoa/y5OMryT1Uhvvr4vLTT37ByGGvkZdbFuQIRUKbihJ14PX62LupkE1L89i7qRCvN/BXu9xzzz307NmTyMhI4uPjmTBhAhs3bqy1zWOPPUZaWhoOh4PevXuzZMmSgMdVX3xeL57cHNx7t+DJzcHn9Qb8mGlpaRiGcdLPNddc49/m8ccfD9lzKiIiEgiH88r4+Q2z6NT2r4wc9hoD+vyT8/r9kw/e2xbs0EREgiY+PrzWbQMHAKb5TRHCajWIig5r0LhEROTc3PTz2eTlluH11r4Ix+czOXiwhFtv/jRIkYk0DbZgBxAqtmXmM/fVXezfUYLb5cXusNK6SwyTr8+g+9DEgB132bJl/OIXv2DYsGG43W7uvvtuJk+ezPbt24mJieHll1/m/vvv58knn2TEiBE88cQTTJ8+nW3btpGWlhawuOqDO2cHrrUL8B05hOlxY9jsWFql4hg4DnvbLgE77po1a/CeUPz48ssvufjii7niiisAePXVV0P2nIqIiARC/uFyxoz8N7mHSmt9MNu1s5DrrplJQUE5N908MIgRiogEx2WXd+fZv3zhv20YTjABaooSVqvBhRd3wenUR28RkVCxa1chSxfn+G+bpgufz4fPV3NxstdjMu+zPeTkFNG2bVyQohQJbeopcQa2f1HAv/+wgT0bjhHdIoyUDtFEtwhjz4Zj/PvB9WzNzA/YsZctW8att97KgAEDGDJkCG+99Ra5ubmsXFkzWdqzzz7LlVdeyW233Ub//v154403cDqdvPDCCwGLqT64c3ZQueAdvLnZGM5ILC2SMJyReHOzqVzwDu6cHQE7dmpqKq1bt/b/fPzxx7Ru3ZrJkycD8Nxzz4XkORUREQmUxx9dcVJBAuD46E33/G4hRwoqghCZiEhw9e6TxMU/7HLCvBLf9JSwWAzsdiu/vXtY8AIUEZE6+3Jtbq3bPvMILvdmtm7d6l9mmrDhy7yGDk2kyVBR4jR8PpPPXt1DaWE16RnRRETbsVoNIqLtpGdEU1pYzdxXdzXIUE4Ax44dA6BVq1ZUVVWxdetWJkyY4F9vtVo5//zzWb16dYPEczZ8Xi+utQswK8qwtEzBcIRjWCwYjnAsLVMwK8pwrV3YIEM5VVVV8b///Y+rrroKi8WCy+UKyXMqIiISKC6Xhzf/s9FfkDBNDz5fAT7fMf82Pp/J229tDlaIIiJB9Y+Xp3PlVT2xWAwsFicWC5i4SE2LZuasy+nRM3A960VEpP6FhVlr3TbNmvkjYmJivnc7ETlzKkqcRs6WIg7sLKZlanitCcwADMOgZWo4+7cXs29rUcBj8Xq93HLLLfTv35+BAweSl5eH1+slJSWl1naJiYnk5weu98a58uUfwHfkEJboFqc8p5boFviOHMSXfyDgsbz11luUlpbyi1/8AiBkz6mIiEigHCmooKLC47/tM4/i8e3Fax7yL7NaDbL2FgUhOhGR4HM6bbz4j2ls2fFLHn18Ghde1IVf/qoXG7f+gqHDWwc7PBERqaORo9pgt3/zlalJOQCxsbH+ZQ6HVa/xIudARYnTKCuqxu3y4Yg49RigYRFW3C4vZceqAx7Ltddey86dO3nvvfcCfqxAMqvKMD1usDtOvYHdgelxY1aVBTyWV199lZEjR9KuXbuAH0tERCQURcc4OPEaAovRAgDTLMc0q7/+G+JivyOvi4g0E2npMfzq1uFMmNSJrt1aUlFRHuyQRETkLLRsFcFPrumNxWJgmlWYpgcwiIqKAsCwwPU39iUuzhncQEVCmIoSpxEVF4bdYcF1whWCJ6quqJn0OqpFWEDjuPbaa5k/fz4LFiygQ4cOACQnJ2O1WsnNrT3WXX5+PomJjbeLsOGMwrDZwe069QZuF4bNjuGMCmgcO3fuZOXKldx4443+ZaF6TkVERAIlJsbB+IkdsFprKhOGEYZh1ORon1kzhJPH4+OSy7oFLUYRkcbCMAz/8B4lJSVBjkZERM7W40+OZ/yE9phmGRYLWIxIrNaa4ZqmTuvMnx4ZG+QIRUKbihKn0bZHHOkZsRw9VIlpfntyR5Ojhypp3TWWNt3jAnJ8n8/Htddey6effsr8+fPp2rWrf53T6aR79+7Mnz/fv8zr9bJ8+XIGDRoUkHjqgyUxHUurVHylx055Tn2lx7C0SsOSmB7QOP7+978THx/PZZdd5l/mcDhC8pyKiIgE0u9mDK8ZYvHrd47f9JYoxGIxuPiHXTRmuojI144XJYqLi4MciYiInC2n08Z7H17GY08OZeB5qfTrV3OB8PsfXspb7/xQ80mInCMVJU7DYjGYdF1HouPDOLCzlIpSNx6vj4pSNwd2lhLTMozJ13XGag3Mqbz22mv53//+x+uvv05sbCz79+9n//79lJfXdAW+/fbb+e9//8vzzz/P+vXrufrqq6msrOTmm28OSDz1wWK14hg4DiMiCt/RXExXJabPh+mqxHc0FyMiCsfAsVisgXuB93q9/Pe//+VHP/oRdru91rpbb7015M6piIhIIA0anMY7H1xKXItwAMLsLb+eyLWUi3/Ykb//64IgRygi0ngcH3NcPSVEREKbYRi0bmPjJ9f05m9/r7mgdeiw1ifNjyoidXfqiRKklq6DE7jmgb7MfXUX+3eU4M6rGbKpY794Jl/Xme5DA3dl4BtvvAHA1KlTay3/61//yq233soNN9xAfn4+jzzyCEeOHKFr167MnDmT9PTA9jI4V/a2XWDc5bjWLsB35BBmWRGGzY41pT2OgWNr1gfQxx9/TG5uLjfddNNJ66677joKCgpC7pyKiIgE0oSJHdiZdQtzZu1i+/Yj7N4dRodOTi65pBfh4fbT34GISDNxvCihnhIiIqHN5/Nx5MgRABITE9m1a1eQIxJpOlSUOEPdhibSZXAC+7YWUXasmqgWYbTpHhewHhLHfXt4o1OZMWMGM2bMCGgcgWBv2wVreid8+Qcwq8ownFE1QzsFsIfEcT/4wQ++99zOmDGDe+65J+BxiIiIhJKwMCsX/7BmKMkvv4xg7dq17N27ly5dAnsxgYhIKNHwTSIiTUNhYSFer5ewsDB/wVlE6oeKEnVgtVpo3ys+2GE0KRarFUtK22CHISIiInXUvn171q5dy4EDB3C73ScNhygi0lwd/+KqtLQUn8+HxaJRk0VEQlF+fj5Q00tCROqX3h2JiIiISJ21aNGC2NhYfD4f+/btC3Y4IiKNRmRkJFarFZ/PR1lZWbDDERGRs6SihEjgqCghIiIiImelffv2AOzduzfIkYiINB6GYfiHcNJk1yIioUtFCZHAUVFCRERERM7K8aLEvn378Hq9QY5GRKTx0GTXIiKhrbq6mqKiIgASEhKCG4xIE6SihIiIiIiclYSEBCIjI/F4PBw4cCDY4YiINBqa7FpEJLQdOXIEgOjoaMLDw4McjUjTo6KEiIiIiJw1DeEkInKy4z0lNHyTiEhoOj50k3pJiASGihIiIiIictbatWsHQE5ODj6fL7jBiIg0EuopISIS2jSfhEhgqSghIiIiImctOTkZp9OJy+UiLy8v2OGIiDQKx3tKlJaWqmArIhKCVJQQCSwVJURERETkrFksFn9vCQ3hJCJSIzIyEqvVis/no6ysLNjhiIhIHZSXl1NRUYFhGLRq1SrY4Yg0SSpKiIiIiMg5ObEoYZpmcIMREWkEDMPwD+GkeSVERELL8V4S8fHx2Gy2IEcj0jSpKFEHXq+PQ5uK2LM0n0ObivB6A98N989//jMZGRlERUURFRVF3759ef/992tt89hjj5GWlobD4aB3794sWbIk4HHVF5/Pi69wN7689TW/fd6AH9Pj8XDHHXeQlpaG0+mkdevW/N///V+tbtWPP/54yJ5TERGRhpaWlobdbqeiooKCgoJghyMi0igcH8JJ80qIiISW4+9nNXSTSOCo3HeGcjKP8sUrWeTvKMXj8mJzWEnsEs3gGzrQbmjgunK1bt2ahx9+mG7dumGaJv/85z+58sorad++PQMGDODll1/m/vvv58knn2TEiBE88cQTTJ8+nW3btpGWlhawuOqDL38Tvp2zoGQfeKvBGgYxbSDjAiyJvQJ23Pvuu4/XX3+dl156ib59+5KZmcmvfvUr4uLiuOeee3j11VdD9pyKiIgEg9VqpW3btuzevZu9e/fqA5yICJrsWkQkVB3vKZGQkBDkSESaLvWUOAM5q44y98HNHNxwjPAWYbTsEEV4izAObjjG3Ac2k515JGDHvvLKK7nsssvo2bMnvXr14q9//SsREREsW7YMgGeffZYrr7yS2267jf79+/PGG2/gdDp54YUXAhZTffDlb8K3/mU4thPCoiA6reb3sZ341r+ML39TwI69atUqJk6cyOWXX06XLl346U9/yogRI1izZg0Azz33XEieUxERkWDSvBIiIrUd7ymh4ZtEREKHaZrqKSHSAFSUOA2fz2TNq9lUFLpIyIjGGW3DYjVwRttIyIimotDFF69kNchQTh6Ph3/9619UVlYycuRIqqqq2Lp1KxMmTPBvY7VaOf/881m9enXA4zlbPp+3podEdTFEt8GwR2IYFgx7JES3gepifLtmBWwopyFDhrB8+XI2baopfKxatYq1a9cyefJkXC5XSJ5TERGRYGvdujVWq5WSkhIKCwuDHY6ISNB5q8L4cv4hPvzbl/zv2S0cO1wZ7JBEROQ0ioqKcLvd2Gw2WrRoEexwRJosDd90Goe3FFOws4yY1AgMw6i1zjAMYlIjyN9eyuGtJaT2igtIDKtXr2b06NFUV1cTHh7Om2++Sf/+/cnOzsbr9ZKSklJr+8TERHbt2hWQWOpF0d6aIZvCW53ynJrhraB4X8128Z3q/fAPPfQQJSUl9OnTB4vFgs/n4+677+amm24K3XMqIiISZHa7ndatW5Odnc3evXuJj48PdkgiIkFhmiav37+eNx9fwyFfNlbDQvanX/DP363lqt/34ap7+5z0OUhERBqHE4du0mu1SOCoKHEalceq8bi8hEVYT7k+LMJKaZ6XymPVAYuhd+/erFmzhmPHjvH222/z85//nI4dO4buh/3q0po5JGzhp15vdYK3sGa7AHj11Vf54IMP+Mc//kHv3r1Zt24dM2bMIDU1lQsuuCAgxxQREWkO2rZty9JPN/DFh5/RLbWSdj3imHBNJ2JbOYMdmohIg3nn8U3899GNGGYYmBa8+PAYLmxmOP/5wwYiou388I4ewQ5TRERO4XhRQkM3iQSWihKnEd4iDJvDSnWFF2f0yaeruqJm0uvwFmEBi8HpdNKjR82b1hEjRvDll1/y5JNP8sorr2C1WsnNza21fX5+fuN+8QyLrpnU2lMJ9siT13urataHRQfk8Pfeey933HEHN954IwCDBg1i7969PPXUU9x4442heU5FRESCrKigimev3sKStdswrLDPEo/VdPLqvV9yx9+HMeHq+u/9KCLS2FRVeHj7sY1ATS9wG+F4zHI8VGKj5qKsNx/6igt+2ZUwx6kvfBMRkeDRfBIiDUNzSpxGUo9YEjKiKDlUgWmatdaZpknJoQoSu0aT1D2mwWLy+XxUV1fjdDrp3r078+fP96/zer0sX76cQYMGNVg8dRbXHmLaQOWRU55TKo9AbJua7QKgqqoKi6X2Q99ms2GaJg6HIzTPqYiIyCn4vD6qStz4Ajz3lWma3H/hfPZuLMVhxGF6Tco9+Zg+8FT7ePL65WxYmHv6OxIRCXHrFxyisszjv328EOGmwr+srKiaTUvyGjw2ERH5fh6Ph6NHjwI1wzeJSOCop8RpWCwG513Xjs//uI2CnaXEpEYQFlHTc6LkUAWR8Q4GX98BqzUw9Z1bbrmFCy64gPbt21NcXMy///1vVq9ezd133w3A7bffzs0338zAgQMZPnw4TzzxBJWVldx8880Biac+WCxWyLgA3/qXoXRfzRwSVmdND4nKIxAWi6XzBTXbBcD48eN56qmnaNu2LX379mX16tW8+OKLXHnllQDceuut3HLLLSF1TkVERE5UmFPOwse2sfbf2bgrvdgjrAz6aXvG3t2NuPSIej/eV4vz2LHmCADhtMLFMSo5QjRtgJr3U/99bCN9x6Z8392IiIS8ihJ3rdthRFHFEUrMbJy0wG7U9BSvKHWfancREQmiI0dqLp6NiIggKioq2OGINGkqSpyBtkNaMvnBnnzxShb5O0opzasZsimtXwsGX9+BdkNbBezYBQUF3HDDDRQUFBAVFUXXrl353//+x8UXXwzADTfcQH5+Po888ghHjhyha9euzJw5k/T09IDFVB8sib2g3w34ds6qmfTaW1gzZFN8Rk1BIrFXwI79r3/9i1//+tfceeedFBYWkpCQwDXXXMPjjz8OwHXXXUdBQUHInVMRCS2mzwXFS8FdAPZEiB2JYQncUIDSfBzeXsJzwxfgKnXj89T0SHRXeMn8xx42vLef21aMo1Wncxsi0eVyUVlZ6f/536urKbNk4/ZW46ESgGqzBA8ubIYDn9dkw8JcKsvchEfZz7mNIk2J8kHTkp4RW+t2FOlUGYVUmyUUmF+RSD9sRjhpnRuup72IhAZ3cRGH/vM2rrxSnCmxpP70SmyRgRnWWk5NQzeJNJygFCVeeOEFnnjiCfLy8ujTpw/PPffc9w6N88wzz/Diiy+yb98+WrVqxaWXXsqjjz6K09lwkya2HdqS1oPjOby1hMpj1YS3CCOpe0zAekgc984775x2mxkzZjBjxoyAxhEIlsRe0Ko7FO2tmdQ6LBri2gesh8RxcXFxvPzyy6dcd3w4qRkzZnDPPfcENA4RCc18UB/Mgvcw9/8ZvCXfLLTGQpt7MFpdHLS4pGl4+7ovcJW48XlrD5Ho85hUFlbzzs/W8KtFY2ut83g8VFVVUVJSQkFBATt27PAvO7H4UFlZSVVVFT5f7eGgtmfvptgs+NawjAZQe7vqKq+KEnJKygfKB01FxsCWtOsZx76txfh8JhbDRiuzF/nGBjxmOUeNjQzuO44OveODHapIo9Rc88HuPz7F/rmVmD4rYAJHyXrzWdpcEEWHe+4IcnTNx/FJrjV0k0jgNXhR4p133uGuu+7ipZdeYvDgwTzzzDNMmjSJHTt2nLIS+dZbb3H33XfzyiuvMGzYMHbu3MlPf/pTDMPg6aefbtDYrVYLqb3iGvSYTZ3FYoV4TXwp0hyFcj44F2bBB5jZ9568wluMufd3YNgwWl7Q8IFJk5C7qYh9XxT6b5umSQXFuKnCgxuPp5p9i7eS8HIxjlYWf6HB7a4ZRsTr9bJp0ybcbjdW6/dfJBAWFkZ4eDjh4eF072UhZ6UNw7BjJQwLduxEYTPC/dvHJTqJaqGrv+VkygffonwQ0gzD4M5/DOf/xs7F4/bh85pYDDsJZm+OWDZg2l30/FEJVVVVIfelqUigNdd8sOehv7BvTjXfTPtqAODzWcj+uBIj7Dna/+bWoMXXnBwvSqinhEjgNXhR4umnn+ZnP/sZ1113HQAvvfQSs2fP5pVXXvHPk3CilStXMnz4cH784x8D0K5dO6688kq++OKLBo1bRETqV3PMB6bPjXngie/fZv/jED8FwwhsrzFpmg59VVTr9lEOcIgdJ223abWT1gNqX6VrsVhwOp3ExMSQnp5OdHS0v+jgdDr9fx+/fWLRYsz5Lta+9g5u16kn1LZYDKbf1DXgPUwlNCkffMc2ygchq+ugBJ5ZMY1/P/AlX8w+gGmCzepk6uRpJAw7iCPOw9y5c5k2bRp2u3qPiRzXHPOBp6Kc/Z+WU1OQML611gBM9n1URJvbXFjDHA0fYDNSVVVFaWkpoJ4SIg2hQYsS1dXVrFu3rtZQQxaLhfHjx5OZmXnKfYYNG8Ybb7zB6tWrGTRoEFlZWcyZM4err776O4/jcrlwuVz+2xUVFf6/aw8rUDfnsm8oU7vr9/48Ho//itTG4ng8jS2uQFO7z67d9XG+gpUPSkpK/G2oazvq4/FilqzErK4Aaj5QlFf62JHjIqNNGFERX3/p5CrGOLYGI3rAWR+nPul5ElrttobX/BwX6YvE8nWdwEkUkUYL7EYYA/sOovuY1rWKDWFhYbjdbj7//HPGjRv3vV+U+Xy+WkM4OaMs/PqVoTz9sxUYFqPW0FGGxaBz/5ZcfEeXRns+Q/X/fa6UDxpPPsgv9PDx0hIuHh1Dq7ivP6IpHzQKZ9vuNt2jufe9UZQWuig+UkVsQjjRLcI4duwYs2bNIjc3l9mzZzN58uTT9kwLBv2/1e6z2f9cNNd8cOjtd/Fa7fD1y0BeVSFrj+3ivBZdSHLGAeDBSu77/yPpskvP+jj1qak+Tw4ePIjX6yU2NhbDME5qX1Nt9+mo3Wr32ex/JgyzAb9xPnToEGlpaaxcuZKhQ4f6l//2t79lyZIl31nN/utf/8pvfvMbTNPE4/Fw00038eKLL37ncR588EH+8Ic/+G8PGzaM5557jk6dOhEREVF/DRKpg8rKSrKyssjNza31Jkgk1FRUVPDjH/+Y4uJiYmLObpLGYOWD4956661GkQ927dpFVlYWTqeTQYMGER4efvqdROpo+/bt5OTkEBYWxrBhw3A4dJWd1A/lg/rz2WefATVfwE2YMCHI0UigFRcXs3btWjweD4mJifTp0weLRT3JJHQpH9Sf4/kAYMKECXptaEC7d+9mz549pKam0qtXr2CHIxKS6pIPgjLRdV0sXryYRx55hL/97W8MHjyY3bt3c/vtt/OnP/2J++6775T7zJgxg7vuust/u6CggOLiYqxW61ldheL1ehvl1SuBpnbXL4vFgmEYnHfeeWf9Ri1Qjl8ZO2HChGbVhVztPrt2H7+aqKHVRz4oKSmhdevWTJw4sc7Pw/p4vJilazF3/cJ/O66gnPDYmiKlZ/f7jDo/hnCnBaPLaxiRjeONsJ4nodfuT+/fzPLndtbMkQj4zDYc9OZRRRkHXvqS6357BRPv63HKfeur3dVVHtwuH+HRdiyWbw9F0PiE8v/7XCgfNJ58cCD2m7lgJicv9f+tfBB8gWr36NGjmTt3Ll6vl6ioKEaNGoVhNJ7XS/2/1e66UD44+8fL4ffeY8dL+/23Hbl5/r8PZS6ga3RrALre2ZnEC6af1THqW1N9nsydO5eIiAiGDRtG9+7dT1rfVNt9Omq32l0XdckHDVqUaNWqFVarlcOHD9dafvjwYZKTk0+5z3333cfVV1/NjTfeCECvXr0oLy/n5z//Ob///e9PWTV2OBy1rgIsLy+nuLgYoM5v9E7sSNKY3iQGmtpd/+0+fn82m63RvqDZ7fZGG1sgqd113+9cBSsfnNiGs23HuexrthiM6YyH6jzAxGa4sRo13RvLKmBe5jGmj++GM7Zfo3vt1fMkdEz7Ux8qDlez5rVsLDYDTCtt6MUOzxe0PN9C+g+N07bpXNttt9sh+qx3D5pQ/H/XB+WD4OeD47kAwG5xAQY42mAoHzQa9d3uNm3aMHnyZObNm8fevXuJiIhg+PDh9Xb/9UX/7+ZF+aDh80HKZZex58XH8LqtgIFxwhCYe4oPkmBEkRwdQfKFFzW6i0ab2vOksLAQq9VKamrq97arqbX7TKndzUtD5IMG7QcWFhbGgAEDWLBggX+Zz+djwYIFtbrnnaiiouKkRHL8hbi5znUgIhLqmms+MAwLRtsHqZm07psvmTqk2YlwWigs8fHp9rHNbtxKqV9Wm4UrXhnMb76axMjbM+h3RRsm3jmA38+8ngE/aceatWs4cuRIsMMUAZQPvp0Pvl4LGBhtH2h0BQmpX23atGH06NEAbNmyhfkzlzLzrvX8sfXH3NvyQ14Ys5AN7+7D5wuNx7XIuWiu+cBqt9Ph6tSvb307ZpMNJVmk/yS10RUkmorPX9rBz9q+z0TbSzx5/WLeuGc9Gz8sPP2OInLOGnz4prvuuotrr72WgQMHMmjQIJ555hnKy8u57rrrALjmmmtIS0vj0UcfBWD69Ok8/fTT9OvXz98d77777mP69Ol6URYRCWHNNR8YcaMh4x+Y+x4FNgMQE2lhQN8ezNo8mCMVrZg7dy5Tp07FZmv0oyxKI5bSK47pT/Sttcw1r5js7GwWLlzID3/4Qz3GpFFQPngUWPfNivBOGG1+jxFz6i/hpGnp1KkT1dXVfPjap/zv1jdJ9naipa8NAHuXF5C1pIB+Vx7kx/8ejMWqseWlaWuu+aD1z2/AsL3G3n/n1FoeE24nYnAUWe3b0SFIsTVlT16ymM9nZgNQaRbjM01Kj9n58y8z2bIkn9vePD+o8Yk0dQ3+SfTyyy+noKCA+++/n7y8PPr27cvcuXNJSkoCYN++fbUq3ffeey+GYXDvvfdy8OBBEhISmD59Og8//HBDhy4iIvWoOecDI/Z86DkCo+C/GKVfYnQYRPywy5jW5SizZs0iLy+PefPmMWnSpJD6QCWN38iRI8nPz6eoqIjMzEzOP18ftiT4lA9GYKT8AbxlYIvB6HGfekg0M53bZ5D9j7n43D4OmjsxsBJvpGF6a9avf3sf7Ya1YsSvOgc3UJEAa875IP36n5JyrZc1d96B+1gJ9pax/Pjue5g1axZZWVns2LGDLl26BDvMJmPJ63v8BQmAamrGwQ8jGgyY/fYeBv+gDYMvbRukCEWavqBcHnfLLbdwyy23nHLd4sWLa9222Ww88MADPPDAAw0QmYiINKTmnA8Mw8AIb48RUYbhbIdhGLRq1YopU6Ywe/ZsDhw4wIIFCxg/fvwpx8MVORtOp5MxY8Ywe/Zstm3bRnp6Ou3btw92WCLKB47UWreleVn/zn5iStJoZVZQQA4H2IbFtBFnJPm3WfrMDobf3EmPD2nymnM+sFqtRPXs5b+dnJzMwIEDWb16NStWrCA5OZnY2NggRth0vP3QV7VuV1MKQBhfT3ZuwFsPblBRQiSA9C1HHfi8Po5uLSQ3M5ejWwvxeX0Nevx77rkHwzC44YYbai1/7LHHSEtLw+Fw0Lt3b5YsWdKgcZ0Ln8+Lr2IHvpI1Nb993oAfs6ioiBtuuIHU1FScTif9+vVj6dKltbZ5/PHHQ/acikhoS0pKYvLkyVitVrKzs1m8eHHIjIkroSEtLY2+ffsCsHTpUsrKyoIbkIhIM5ez8ggWm0GK0Zl40gDYzxaqzcqaDUw4uqecymPVQYxSRIKhT58+pKSk4PF4WLhwIT5fw34P1VQd3PfN+1+v6abarLntL0oA+/aUNHhcIs2JihJn6PDafFb8ZiUr785k9YNrWHl3Jit+s5K8NfkNcvylS5fy+uuvk5GRUWv5yy+/zP3338/vfvc7MjMz6dGjB9OnT+fgwYMNEte58JWuh+z7IOdB2P9Yze/s+2qWB9BVV13FkiVLeOWVV1i3bh1jx45l2rRp7N27F4BXX301ZM+piDQNqampTJgwAYvFwu7du1m2bJkKE1KvBg4cSEJCAi6Xi0WLFunxJSISRIbV8M93Hk3Lb5Z/6+O6YVUvCZHmxjAMxowZg8PhoKCggLVr1wY7pCbhxE5nRewCfNiMSGxG+Cm3EZH6p6LEGShYV8D6J9ZzdHMhjtgwottG44gN4+jmQtY/8WXACxPFxcVcc801/O1vfzupq96zzz7LlVdeyW233Ub//v154403cDqdvPDCCwGN6Vz5StfDwb9C5TawxoKjdc3vym1w8K8BK0yUl5fz2Wef8fDDDzN58mR69OjBU089RZs2bXjmmWcAeO6550LynIpI09KmTRvGjh2LYRhs376dzMzMYIckTYjFYmHcuHHY7XZyc3PZsGFDsEMSEWm2MiYk43Ob+Ewvh9gJQAJtsRsOAAwLpPWNIzw2LJhhikiQREVF+ecB27BhA7m5uUGOKPR17NoCgErzCJVmPmAQT+05OzJ6xQchMpHmQ0WJ0zB9Jrve3o3rWDUxHaOxR4VhWC3Yo8KI6RiN61g1u/67M6BDOV1//fWMHz+eiy66qNbyqqoqtm7dyoQJE/zLrFYr559/PqtXrw5YPOfK5/NCwXvgLQJHe7BGgmGt+e1oX7P8yPsBGcrJ7Xbj9XoJDw+vtdzpdJKZmYnL5QrJcyoiTVOHDh0YNWoUAJs3b2bNmjVBjkiakpiYGIYPHw7A2rVrOXz4cJAjEhFpnnpcmEqLthEcse7DTRV2nCTSzr/e9MGY33YLXoAiEnQdOnTwT3S9cOFCXC5XkCMKbdc+1h+f6eaYWVMIjjZaE2Z8PXSTWdN57donBgYvQJFmQEWJ0zi2o4ji3SVEJIdjGN/qPmtYiEgOp2h3Mcd2FAXk+P/617/YtGkTf/3rX09al5eXh9frJSUlpdbyxMRE8vMbZlips1K1G1x7wZ50cn84w6hZXpVVs109i4uLo2/fvjz00ENkZ2fj8Xh48cUX2bBhAwUFBaF7TkXOgGmaZC0r4H+3rgPgo9u/ZO/KIxq2pZHLyMhgxIgRAKxfv15XtEu9ysjIoFOnTpimycKFC6mu1njlIiINzWqzcMV7/SiJOgAGpBidsRhWLLaaz0rjf9+dfle0CXKUIhJsw4YNIzY2lvLycpYtWxbscEJan8lpnHeZFx/V2IxIYk4oBBsG3PD7fnQbmRS8AEWaAVuwA2jsqotd+Kq92MJPfaqs4VZ8+V6qi+u/Sr1nzx5+97vfMWfOHCIiIur9/oPGUwK+arA7T73e4gR3Qc12AfDmm29y7bXX0r59e6xWK927d2f69Ols3LgxIMcTaQw8Li//uTKTzR8dxB5tMHSKg3Vv5rDqpWz6XNaaq94YgtWuOnVj1b17dzweD6tWrWL16tXYbDZ69uwZ7LCkiRgxYgSHDx+mtLSUFStWBDscEZFmKatwKxP/0IOybVZidnbCVe4ltXcsQ2/qROsBGkJERMButzN27FhmzpxJVlYWO3bs8PeekLrJzs6m3VgLV7brzZGFKeTu8IFhkNE7np8+NZCO57UKdogiTZ6KEqcRFuvAEmbFU+nBHnXyGJ7eSi8Wh5WwWEe9HzszM5PCwkL/0AoAXq+XtWvX8vrrr1NaWorVaj1pPMH8/HwSExPrPZ56Y4sBSxj4qmqGbPo2X1XNeltMQA7fvXt31qxZQ0lJCceOHaNt27ZMmzaNNm3akJycHJrnVOQ0Pv7NBrZ8XDNZu89j1vq98f39xLWO4MIn+wYrPDkDvXv3xu12s27dOlauXIndbteHEKkXYWFhjB07lo8//pg9e/YEOxwRkWZn//79ZGdn44i08+NHL6FFixbBDklEGqmEhAQGDhzI6tWrWbFiBcnJySfNPSrfz+VysXz5cgAm/nAEgx4bFOSIRJonXRZ7Gi26xBHbKYaKvEpMs/a8EabpoyKvkrhOsbToElfvx77gggtYs2YNq1at8v/06NGDiy66iFWrVhEeHk737t2ZP3++fx+v18vy5csZNKgRv6g6O9XMHeE+DN8eNsY0a5Y7O9RsF0AxMTG0bduWgoICli5dyvTp03E4HKF5TkW+R/lRF6v+sYfjL2GmadYassk0YcXfdlFZrGFbGrsBAwbQu3dvAJYsWaIvkKXeJCUlMWDAAAC2bdtGSUlgeiuKiEhtXq+XlStXAtCzZ08VJETktPr06UNKSgoej4eFCxfi8wVujtOmaOXKlVRUVBAXF+d//ysiDU9FidMwLAadr+iEo0UYJXtKcZdV4/N6cZdVU7KnFEeLMDpfmYHFWv+nMi4ujoEDB9b6iYiIID4+noEDaybcuf322/nvf//L888/z/r167n66quprKzk5ptvrvd46ovFYoWEy8AaVzO3hLccTG/Nb9desMVBq0trtguA//3vf3zwwQds376djz76iPPPP58OHTpwyy23AHDrrbeG3DkV+T67F+XjddcUIUzTZKN3AfPmzatVmPBU+dizpCBYIUodDBkyhO7duwOwaNEicnJyghyRNBX9+vUjKSkJj8fDokWL9AFXRKQBbNq0ieLiYsLDw+nfv3+wwxGREGAYBmPGjMHhcFBQUMC6deuCHVLIyMnJYdeuXRiGwejRo7FaA/O9k4icnoZvOgMJAxLo93/92PXWLor2FOPLrxmyqWWveDpfmUHyecEb1ueGG24gPz+fRx55hCNHjtC1a1dmzpxJenp60GI6E5bofvjSboOC92oKEe6CmiGbIrrXFCSi+wXs2EVFRTzwwAMcPnyY2NhYpk6dytNPP43D4cA0Ta677joKCgpC7pyKfBdv9TdfLHpw1/rbQtgpt5PGbfjw4bjdbnbt2sXnn3/O5MmT9Rol5+z4B9zFixdTUFDA2rVr1UtQRCSAysvL+fLLL4Gaiw7Cwk4eLlhE5FSioqI4//zzmT9/PuvXryc9PZ2UlJRgh9WouVwu/wThvXv31hDdIkGmosQZShqYSGL/BI7tKKK62EVYrIMWXeIC0kPi+6xevfqkZTNmzGDGjBkNGkd9sET3wxfZG6p210xqbYsBZ6eA9ZA47vrrr+f666//3m1mzJjBPffcE9A4RBpKWv8zGwYgrZ+GCwgVhmEwatQoPB4Pe/fuZd68eUydOpXk5ORghyYhLioqih49euDz+diwYQPp6emkpqYGOywRkSbBVeRi9/t7KNpVjD3Cxr7EHNwWNykpKXTu3DnY4YlIiOnQoQNdunRhx44dLFy4kEsvvRSHo/7nO20qMjMz/cM2HR99RESCR0WJOrBYLbTsHh/sMJoUi8UKEZqoVSSQkrrG0HF0AnuXH+GEjhJ+FqtB53FJtOoY1fDByVmzWCyMGzeOzz77jP379/Ppp59ywQUXkJCQEOzQJMQlJycTHR3N7t27WbRoEZdccglOpzPYYYmIhLTd7+9h+a9X4K32YrEaFHqP8VX5BuI6x3HhBxcGOzwRCVHDhg0jNzeXkpISli1bxvjx44MdUqO0b98+du7cqWGbRBoRzSkhItIMXPHKYKISHBhWo9Zyw2oQkxrOj/51XpAik3NhsViYMGECqampuN1u5syZQ2FhYbDDkiZg6NChxMbGUl5eztKlS4MdjohISDu49BBLbl2K1+UFEzxuL7srdwMQmRPJ+v/7qtZcXyIiZ8putzNu3DgsFgtZWVns2LEj2CE1Oi6Xy/9+tlevXhq2SaSRUFFCRKQZiG8XyV1fTmLUHRmERdkBiEwIY9zvunHnuonEpUcEOUI5WzabjUmTJpGYmIjL5WL27NkUFxcHOywJcTabzf8BNzs7m61btwY7JBGRkLXh6Q0YxjcXhuR6D1HuK8Nu2GlrbUfuslyOrD8SxAhFJJQlJCT4hyNasWIFJSUlQY6ocdGwTSKNk4oSIiLNRHSSkyl/6sXUh3oB8NtNU5jyUC+iWmnc0VBnt9uZMmUKLVu2pLKyklmzZlFaWhrssCTEtWrVyj/RdWZmJseOHQtyRCIiocdV5CJv1WFMX01PiGqzmmx3NgDt7O2xG3YMm0H2nJwgRikioa5Pnz6kpKTg8XhYsGABPp8v2CE1CseHbQIYNWoUNptGsRdpLFSUEBERaQIcDgdTp04lLi6O8vJyZs+eTUVFRbDDkhDXq1cv0tPT8Xq9LFiwAK/XG+yQRERCiqfCU+t2vicfj+khyhJFijUFAAMDd/kpJv4SETlDhmEwZswYHA4HBQUFrFu3LtghBV11dbV/2KbevXuTlJQU5IhE5EQqSoiIiDQR4eHhTJs2jZiYGEpKSpg9ezZVVVXBDktC2PHJAJ1OJ4WFhaxYvIIdb+3kiwdX8+Wf13N0i+YwERH5PuEJ4dij7f7bBd4CAJJtyf4hnXxeH3EZccEIT0SakKioKM4//3wA1q9fT25ubpAjCq7jwzbFxsZq2CaRRkhFCRERkSYkMjKSadOmERkZybFjx5gzZw7V1dXBDktCWEREBKNHj+bIxiO8esVrzLz9Y7a+vI0Nz37FR+NmMu8nn1Ndpit8RUROxWK30PUnXTCsBi7TRYmvZt6nVtYE/zZWh5VOl3QMVogi0oR06NCBLl26ALBw4UJcLleQIwqOffv2+Sf9Hj16tIZtEmmEVJQQEWkmqqo8vPXGRv72whoALr/0fd58YxMul+c0e0qoiY6OZtq0aYSHh3PkyBE+/fRT3G59aSxnz34gjLK3y/FWe9lRvZ3K6kpMb8346AcWHWTRzxcFOUIRkcar7519iOscyxGzZjLrWEssDsOBYTHAgPOfHk5YTFiQoxSRpmLYsGHExMRQXl7OsmXLgh1Og6uurva3W8M2iTReKkqIiDQDx45VMX70f7jj1nnkZNdcofflujx++bPZTB7/JiUlzfMKmqYsLi6OadOm4XA4OHz4MJ999hkejwpQcnbWP72BDmEdiTSicJtudlTvwDRrihKm1+TAwoMc+epIkKMUEWmcwmLCuODjaURMCMcWbvX3kkgemsSU9ybR8YfqJSEi9cdutzNu3DgsFgtZWVn+iZ6bi8zMTMrLyzVsk0gjp6KEiEgzcPstn7Jlcz4AX3+PiM9X88eG9Xn8+o55wQpNAig+Pp6pU6dit9s5dOgQ8+fPx+fzBTssCTGuYhe5y3MxfAbdwrphMSwc8xayz7PPv41hM9g7Kzt4QYqINHIem4eYUdEMenAQN63+BVfvvIqpH0whdURqsEMTkSYoISHB/4X88uXLKSkpCXJEDWP//v3+YZtGjRqlYZtEGjEVJerA5/VRujufY+v3Ubo7H5838F/s/PrXv8YwjFo/7du3r7XNY489RlpaGg6Hg969e7NkyZKAx1VffD4vHt92PN4v8Pi24/N5A37MuXPnMnbsWBITEzEMgzfeeOOkbR5//PHTntNQPu/SvBzYX8LMD3fg/XqolW/zek3ef3crh/PKGjgyaQgJCQlMnjwZm83Gvn37WLhwoQoTUiee8m962ERaIulor7miN9u9l13VO/GZPgzDwF2qIcJERL5LdnY2AMkpySR1TNRwTSIScH369CElJQWPx8OCBQua/GeA6upqli5dCkCvXr1ITk4OckQi8n1UlDhDxZsOsOOxT9n558/Y/ewCdv75M3Y89ilFmw4E/NidOnVi3759/p+VK1f617388svcf//9/O53vyMzM5MePXowffp0Dh48GPC4zpXH9yVV3nuo8txHpfcRqjz3UeW9B4/vy4Aet6ysjF69evHUU0+dcv2rr7562nMayuddmp8vVh30946Ab96Imif87fWarP5Cj9+mKiUlhYkTJ/q7cC9dutQ/9I7I6ThbObFH2f23U21pdLB3AOCQ5xCbqzdR7a4mtlNssEIUEWn0srKygJpJaEVEGoJhGIwZM4awsDAKCgpYt25dsEMKqFWrVlFeXk5MTAznnXdesMMRkdNQUeIMlGw+xN5/Lqd012Fs0Q6c6XHYoh2U7jrM3n8sC3hhwmq10rp1a/9PSkqKf92zzz7LlVdeyW233Ub//v154403cDqdvPDCCwGN6VzVFCT+gtfcBsRi0BqIxWtuo8r7l4AWJi699FKeffZZrr766lOuf+655057TkP1vEvzpC+fBSA9PZ3x48djGAY7d+5kxYoVwQ5JQoQ1zErGVRkYVsO/rLW9DT0cPbEaVo55j7HBs4HEiQlBjFJEpPEp2ZbLzqfmsfa3b7Ph3wuoyis+qde7iEggRUVFMXLkSADWr19Pbm5ukCMKjAMHDrB9+3YARo8erWGbREKAihKnYfpM8mZtxF1aSXjreKyRDgyLBWukg/DW8bhLK8n95KuADuWUk5NDYmIi6enpXHTRRezatQuAqqoqtm7dyoQJE/zbWq1Wzj//fFavXh2weM6Vz+el2vsOplmMQTssligsFhsWSxQG7TDNYqq97zbIUE7f5nK5TntOQ/W8S/M1ZGg6hvH921gsBoMGpzVMQBI07dq1Y8yYMQBs3bqVVatWBTkiCRX97uhDdLvoWoWJVtZW9A3vh8NwkHpxMnMWzWmyH3RFROrC5/Gy+f6ZrL3uNQ7+bz2b5n3BsXU5FL3xFfufWYLP07SHUBGRxqVDhw506dIFgEWLFuFyuYIcUf2qrq72D6fds2dPDdskEiJUlDiN8r0FVOw7hqNlFMa3vtUzDANHyygqsgsp33skIMcfOnQof/vb3/jkk0/461//yr59+xg1ahRFRUXk5eXh9Xpr9ZwASExMJD8/PyDx1Acfu/CZWRgkYrHUfghaLBYMEvGZe/Cxq8FjO5NzGqrnXZqvtPRoUp1R8F0dJkxoHRlNYlJkg8YlwdGpUyf/1VIbN27kyy8DO2SeNA2OFg4unHUB3a7tii38myvP2vdvx/+9+Rt6TuuJy+Vi9uzZ/skFRUSaq93PLyL/860AmF4fuZWFmKZJsqMFeXM2sfefS4McoYg0N8OGDSMmJoaysjKWLVsW7HDq1YnDNg0aNCjY4YjIGVJ/ptPwlLowqz1YnFGnXG9x2vEdLcdTWhWQ41966aW1bo8cOZL27dvz2muvcfHFFwfkmAFnFmNSjYHzOzZwYlIAZnGDhiXSVG1dmU9qSQTHrFWU4/qmOPH170jTRmKhk13rjpIxsFXQ4pSG07VrV9xuN5mZmaxduxabzUbv3r2DHZY0co4WDoY+MoTz7htIRV4FtkgbEYkRAHTwdGDx4sVkZWWxZMkSjh07xuDBg0+6oENEpKlzl1Ry8IMv/e+zqn1ujrpLAUhxxIMJ+99ZS9trhmKLdAQxUhFpTux2O+PGjWPmzJlkZWWxc+dOMjIygh3WOdOwTSKhSz0lTsMW7cAIs+Grcp9yva/KjSXMhi36u75gr1+tWrWibdu27N69m+TkZKxW60lDJeTn55OYmNgg8ZwVIxaDMOC7CjlVNeuNhp8w80zOacied2m28rLLsGGhl7cl7b3ROL9+6Y82rXT0xNDT2xIbFnL3lgY5UmlIvXr18k8At2rVKrZu3RrkiCRU2MJtxLSP8RckAGw2G+PHj2fAgAFATS+cefPm4Xaf+v2TiEhTdWxtDqb7m2Fo91cewcQk1hZBpK3mM6Ovyk3Rhv3BClFEmqmEhAQGDhwIwPLlyykpKQlyROemurqapUtrep5p2CaR0KOixGlEtk8gok0LXEfLTpos1jRNXEfLiGgXT2T7hrm6uLi4mP3795OSkoLT6aR79+7Mnz/fv97r9bJ8+fJG3WXNQmcsRgdM8vH5ao+n6vP5MMnHYnTEQucGj83hcJz2nIbqeZfmK7pFzVV4FgySzBhSfDXDNHXzJpJoRmDBqLWdNB/9+vWjb9++QM0Hk+NzFomcrQEDBjB27FisVis5OTl8/PHHlJWVBTssEZEG46v2+P/OqshjW3lN8SHNWfvzos/lQUSkofXp04eUlBQ8Hg8LFiw46TuZUPLFF19QVlamYZtEQpSKEqdhWAySL+iNPTqcyv2FeMtdmF4f3nIXlfsLsceEkzK9DxZrYE7lL37xCz799FN27NjB/PnzmTp1KhaLheuuuw6A22+/nf/+9788//zzrF+/nquvvprKykpuvvnmgMRTHywWK2HWyzGMWEyy8fnK8Pk8+HxlmGRjGHGEWX+ExWINyPGLi4vJzMwkMzMTgKysLDIzM/1fxt16662nPaeheN6l+eo7LoWouLCvb504lMo3f8clOuk9SleWNEeDBg2iZ8+eACxevJi9e/cGOSIJdZ06dWL69OmEh4dz9OhRPvzwQ825JCLNRlRGEqZpsrV0H1vL9gHQPjyJ9uFJtbfrpB7WItLwDMNgzJgxhIWFUVBQwLp164Id0lk5ePAg27ZtA2DUqFEatkkkBOlZewZieqbS/mcjyP1kIxU5hfiOlmMJsxGdkUTK9D7E9UoP2LEPHjzItddeS1FRES1atOC8885jxYoVpKamAnDDDTeQn5/PI488wpEjR+jatSszZ84kPT1wMdUHm6U/Tu6k2vsOPjMLkwIMwrAa3Qmz/gibpX/Ajr1ixQqmTZvmv/3AAw/wwAMPcMkll/Dee+9x3XXXUVBQ8L3nNFTPuzRPYQ4r1/6xHy/c9sV3bnPtH/tjs6tO3VwNHToUt9vNjh07WLBgAZMmTaJ169bBDktCWGJiIj/4wQ+YO3cuhYWFfPLJJ4wePZqOHTsGOzQRkYCKaNeSXa1KyDpyGIBuka3pGJnyzQZWgxb92hDRJj5IEYpIcxcVFcXIkSOZP38+69evJz09nZSUlNPv2EhUV1ezZMkSAHr06BFSsYvIN1SUOEOxvdKJ6ZFG+d4jeEqrsEU7iWzfKmA9JI6bNWvWabeZMWMGM2bMCGgcgWCz9MdCH3zsqpnU2oitGdopQD0kjps6depJQ3Edd3z5jBkzuOeee773fkL1vEvzNP2XXamu8vLqfWvhm2GOcURYueGRgUy9MfQnOZOzZxgGI0eOxOPxsGfPHubNm8eUKVP8BXCRsxEVFcVFF13EggUL2LdvHwsWLKCoqMg/74SISFPjdruZN28envOTsOdk0yMsnfSwlt9sYDUIi4ug6z1TgxekiAjQoUMHunTpwo4dO1i0aBGXXHIJDkdoDOd74rBNgwcPDnY4InKWdFlsHVisFqI7JdKiXxuiOyUGvCDRHFgsVmyWrtisg7FZuga8ICHSXBmGwaV39eSt/T9izJUdALj1uSG8ffAKLrqlW5Cjk8bgeFfutm3b4vV6mTt3robckXNmt9uZNGkSvXv3BmDdunUsWLAAj0djqYtI01JZWcknn3zCwYMHiUiM5WdvPMCQqydhjagZQtMaGUbrH53Hea9fT3hqXHCDFREBhg0bRkxMDGVlZSxbtizY4ZyRE4dtGjlypIZtEglhevaKiDQjkTFhdB2UwKZNeYy+ogPh4fZghySNiMViYfz48cydO5eDBw8yZ84cJk2aFOywJMQZhsGQIUOIi4tj+fLl7Nmzh9LSUiZOnEhERESwwxMROWfFxcV8+umnlJSU4HQ6mTJlCgkJCdC3G53vGI+v2oslzIphGKe/MxGRBmK32xk3bhwzZ84kKyuLnTt3kpHReHvQu93uWsM2qVe3SGjTpf4iIiLiZ7VamTRpEsnJyVRXV/Ppp59SVlYW7LCkCejatStTp07F4XCQn5/PRx99xNGjR4MdlojIOSkoKGDmzJmUlJQQExPDxRdfXFOQ+JphGFgdNhUkRKRRSkhI8A+tuXz5ckpKSoIc0Xc7PmxTdHQ0gwYNCnY4InKOVJQQERGRWmw2G5MnT6ZVq1a4XC7Wrl3bqD+gSOhITU3lBz/4AXFxcZSVlTFz5kxycnKCHZaIyFnZv38/n3zyCVVVVbRq1YqLLrqImJiYYIclIlInffv2JSUlBY/Hw8KFC/H5fMEO6SSHDh1i69atAIwaNQq7XT3+RUJdsypKfNfkxiINQY8/aQx8Hh9Fu4sByF2Zh8/b+N5wSuMQFhbG1KlTadGiBS6Xi9mzZ6vHhNSLmJgYLrroItLS0vB4PHz22Wd89dVXwQ5LRKROdu7cydy5c/F4PKSnpzN9+nTCw8ODHZaISJ0dn1suLCyM/Px81q1bF+yQanG73SxevBiA7t27a9gmkSaiWRQlwsLC8Hq9VFRUBDsUacZcLhemaRIWFhbsUKSZ2vNhFu8MfJftr9VMDLbg+gW8M+A99n6SHdzApNE6Pi52REQE5eXlzJ49m8rKymCHJU2Aw+FgypQpdO/eHajpjr9kyZJaV+ZVF1VQlVeMz+0NVpgiIqe0YcMGFi9ejGmadO7cmcmTJ+uqXREJaVFRUYwcORKA9evXk5ubG+SIvnHisE2DBw8OdjgiUk+axUTXdrudNWvW4HA4AIiMjDzjMT1N08Tn82GxWJrVOKBqd/222+v1kp+fj81m8z8ORRrSno+yWPzLJXjN2l/uVeRVsPBnixj7rzG0v6BdcIKTRi0iIoKBAwdSXV1NcXExs2fPZvr06Xotk3NmsVgYMWIEcXFxZGZmsmPHDkpKSugf1ZHcN9dSsvkgANZIB2kX96Xd9cOxRepxJyLBY5omK1euZMuWLQD06dOHQYMGNavPSyLSdHXo0IEuXbqwY8cOFi1axCWXXBL09/watkmk6WoWRQmAW2+9lU2bNpGbm1vnN42maTbLN5pqd/3fb6dOnbBYmkUHJWlEfB4fX9y/+nu3+eKB1bSb2hbD0vye83J64eHhTJgwgU8//ZTCwkLmzJnDtGnT1PNL6kXPnj2JjY1l/vz5bJu7mi/mvcmgFl2IsjgB8Ja72Pff1RR+sZf+f/+JChMiEhRer5clS5aQlZUFwLBhw+jZs2eQoxIRqV/Dhg0jNzeXkpISli1bxvjx44MWi9vtZsmSJYCGbRJpippNUcLn85Geno7T6azT0BMej4c1a9Zw3nnnYbM1m9Oldgeg3ZGRkc3qXErjkbsyj8r873/dKz9YTt4Xh0kZmtxAUUmoiYmJYdq0aXzyyScUFBQwd+5cpk6dqtc1qRetW7dm6piJPPfYb6jwulh+dAsDYjqR4Iit2cBnUpZVQM6/V9Hxl6OCG6yINDtut5tPP/2UgoICLBYLY8aMoWPHjsEOS0Sk3tntdsaOHcvHH39MVlYWO3fuJCMjIyixrF69mtLSUqKiojRsk0gT1Oy+SQgLC6vTlZ1utxuXy0VMTEyz6iamdjevdkvTdrqCRF23k+arRYsWTJ06lVmzZpGXl8e8efOYNGkSVqs12KFJE+BacYARcd1ZW7SLQncpq4t30iOqDe0ikmo28Jkc/PBLOvz8fAyreh2KSMMoLy9nzZo1tGvXjvDwcCZOnKirdUWkSUtMTGTAgAGsWbOGFStWkJycTExMTIPGcOjQIf9QeRq2SaRp0ic6EZEmLiI5/Iy2i0yJCHAk0hS0atWKKVOmYLPZOHDgAAsWLKg1ObHI2SrPOYrDFsag2AwirA5MTDaX5VDiqfBv4ympwl1SFcQoRaQ5KSoq4uOPP6a0tJTw8HCmT5+ugoSINAt9+/YlJSUFt9vNwoULG/T9vsfj8Q/b1K1bN9LS0hrs2CLScFSUEBFp4pKHJhOR/D0FBwOi2kSRODCx4YKSkJaUlMTkyZOxWq1kZ2ezePFiTNMMdlgS4mwRYRS4ill+bAsVXhcA8fZoIiwnzCFhgNXZ7Dr6ikgQHD58mJkzZ1JeXk5kZCQXXnghLVu2DHZYIiINwjAMxowZQ1hYGPn5+axbt67Bjq1hm0SaBxUlRESaOIvVwtCHB4NBzc+Jvr499KEhmuRa6iQ1NZUJEyZgsVjYvXs3y5YtU2FCzlpZWRlbnPmsKtxGmbeKMMNGn+j2DI3ris3y9fBgFoP4IR2whmuCdREJrOzsbGbNmoXL5SIhIYFBgwYRHR0d7LBERBpUVFQUI0eOBGD9+vXk5uYG/Ji5ubls3rwZgJEjR9Zp+HURCS0qSoiINAPtprVj3CtjiUqLrLU8uk0UE14fT5uJrYMUmYSyNm3aMHbsWAzDYPv27WRmZgY7JAkxXq+XDRs28O6773LYKCWidTztI5MZ07I3rcMTMIyvi6Vf/2p37bDgBSsizcK2bdv4/PPP8Xq9tGnThmnTpulLMRFptjp06OCf6HrRokW4XK6AHevbwzalp6cH7FgiEnzq/y4i0ky0m9KWtHGp7L9vHzmlOUx8awJthrZRDwk5Jx06dMDj8bB48WI2b96M3W7nvPPOC3ZYEgIOHDjAihUrKC4uBiAlJYWLXv8Dh55YQtG6fTWTWRtgenxYnXa63XcBcX1VQBWRwFm3bp1/iJKuXbsyYsQIvF5vkKMSEQmu4cOHk5eXR0lJCcuWLWPUqFEBOc7q1aspKSnRsE0izYSKEiIizYhhMYhpFwObILF/ogoSUi8yMjLweDwsX76c9evXY7fb6du3b7DDkkaqrKyMlStXkp2dDUB4eDhDhgyhc+fOACS/cBUlWw5RsGQn3io3kR1akTSxB7YIXaksIoHh8/lYvnw527dvB6B///4MHDgQQEUJEWn27HY7Y8eO5eOPPyYrK4uUlJR6P4aGbRJpflSUEBERkXPWvXt33G43X3zxBatXr8Zms9GzZ89ghyWNiNfr5auvvmLDhg14PB4Mw6Bnz54MGDDgpA+eMT1SiemRGqRIRaQ58Xg8LFiwgJycHAzDYPjw4XTv3j3YYYmINCqJiYkMGDCANWvWkJmZSWRk5Ol3OkMnDtvUtWtXDdsk0kyoKCEiIiL1ok+fPrjdbr788ktWrlyJ3W6nS5cuwQ5LGoF9+/axcuVKSkpKgJqhmoYPH058fHyQIxOR5qyqqorPPvuMw4cPY7VaGTduHO3atQt2WCIijVLfvn05cOAABw4cYOPGjfzwhz+sl/s9cdimIUOG1Mt9ikjjp6KEiIiI1JuBAwfi8XjYuHEjS5YswWaz0bFjx2CHJUFSUlJCZmYmOTk5AERERDBkyBA6deoU5MhEpLkrKytjzpw5FBUVERYWxuTJk0lOTg52WCIijZZhGIwZM4Z33nmH4uJivvzyS4YOHXpO95mXl6dhm0SaKRUlREREpF4NGTIEt9vNtm3bWLRoETabjbZt2wY7LGlAHo/HP1ST1+vFYrHQq1cv+vfvj91uD3Z4ItLMFRYWMmfOHCoqKoiMjGTq1Km0aNEi2GGJiDR6UVFRjBgxgnXr1rFhwwbatWt31nNMeDweFi9eDGjYJpHmSEUJERERqXcjRozA4/Gwa9cuPv/8c6ZMmUJaWlqww5IGkJOTw8qVKyktLQUgLS2N4cOHExcXF9zARESAQ4cOMW/ePKqrq4mPj2fKlCn1Oja6iEhT16FDB1JTa+b+WrRoEZdeeulZ9XBYs2YNJSUlREZGatgmkWZIRQkRERGpd4ZhMGrUKNxuN9nZ2Xz22WdMnTpVQ2OEMNPno2z1Vxz7fCnV+3OxOB1EDx1A3MTzsbdsQUlJCStWrGD//v0AREZGMnToUDp06BDkyEVEamRlZbFw4UJ8Ph/JyclMmjQJh8MR7LBEREJOt27dqKyspKysjGXLljFu3Lg67Z+Xl8emTZsADdsk0lypKCEiIiIBYbFYGDduHPPmzWP//v3MnTuXadOmkZCQEOzQpI5Mn4/c516jNPNLsBjgM/GWlFH4yXyOzF3MkYtHs+NoPj6fD4vFQu/evenXr5+GahKRRmPz5s2sXLkSgPbt2zN27FisVmuQoxIRCU02m40xY8bw6aefsmfPHlq3bk1GRsYZ7evxeFiyZAkAXbp0oXXr1oEMVUQaKUuwAxAREZGmy2q1MmHCBFJTU6murmbOnDkUFhYGOyypo2Nzl9QUJAB8pn/5gZIiZu/czJK/vIjX7SY9PZ1LL72UQYMGqSAhIo3G6tWr/QWJHj16MH78eBUkRETOUWJiIgMGDABgxYoVlJSUnNF+a9eupbi42N+rVkSaJ/WUEBERkYCy2WxMmjSJ2bNnk5+fz+zZs7nwwguJjY0NdmhyBkyfj2OzF9ZaVlLtYv2RXPIqywGIAIYltabX1KlBiFAkdHkLD1I15znKP54HPsACFVHZOKbeijXu7CYOlW/4fD6WLl3Kzp07ATjvvPPo169fkKMSETlZqOaDvn37cuDAAXJzc1m4cCEXXnghFst3X/98+PBhNm7cCMD555+vYZtEmrGg9JR44YUXaNeuHU6nk8GDB7N69erv3b6oqIhf/epXpKSk4HA4yMjIYM6cOQ0UrYiIBIryQfNht9uZMmUKLVu2pLKyklmzZvknQpaG4zmYhWvDUqq3rsZ0VZ7RPt7iUqoKjnK4ooyvjuYx78AePt2/m7zKciwYdG+RwJR2XUgsObP7EzmV5pgPvLnbKXn2Nlxbd4EXMAEvVG3eQckzt+I9vCfYIYY0t9vN3Llz2blzp3+eIxUkRBo/5QNCKh8YhsGYMWMICwsjPz+fdevWfee2Ho+HxYsXA5CRkUGbNm0aKEoRaYwavKfEO++8w1133cVLL73E4MGDeeaZZ5g0aRI7duwgMTHxpO2rq6uZMGECiYmJvP/++6SlpZGTk0NcXFxDhy4iIvVI+aD5cTgcTJ06lU8++YSioiJ/j4mIiIhgh9bkeXKzKf/gBbyH9n6z0O4gfORFOMdcivGtK9pM0+TIkSMcOHCAfTt2sSl7O17TrLVNakQU/VqlEGUPA4sFwzAaoinSBDXXfFD2nz9iugG+/dwxMKtNyt94kJhf/ycIkYW+yspK5s6dS0FBATabjfHjx+vLL5EQoHwQmvkgKiqKkSNHMn/+fNavX096ejopKSf37jg+bFNERATDhg0LQqQi0pg0eFHi6aef5mc/+xnXXXcdAC+99BKzZ8/mlVde4e677z5p+1deeYXCwkJWrlzpH5u4Xbt2DRmyiIgEgPJB8xQeHs60adP4+OOPKSkpYfbs2UyfPh2n0xns0Josb8FBSv5xL1RX117hdlG54F18leVEXnAdpaWl7N27l6+++oojR47g9XqBmgIFMVGEl1WSFBFFcngkieGRhNtOmDPC6yWiZ5cGbJU0Jc0xH3iy1uAtqubkL6COM/AcrcCzfzO21j0bMrSQV1JSwpw5cygpKcHpdDJ58uRTfpkpIo2P8sGphEY+6NChAxkZGezcuZNFixZx6aWXEhYWhmmaGIZRa9imkSNHatgmEWnYokR1dTXr1q1jxowZ/mUWi4Xx48eTmZl5yn0+/vhjhg4dyq9+9StmzpxJQkICP/7xj/nd7373nZOTuVwuXC6X//bxyXbcbjdut7tOMR/fvq77hTq1W+1uDppjuz0ej/+LRrfbjc1W9zRQH+dL+aDG8f+Hx+NptI/DQLQ7LCyMSZMm8cknn3DkyBE+/vhjpk2b1qg+nDSl14fyBe/h8frAYgW+ea64PF5ySyvJe+8Nig5VUe4Fr9dLXl4e8fHxOJ1OUlJSSE1NJSa5Da735ta6X+/xPywW7C1bENYzI2TPV1P6f9fFubZb+eDs2+/auQaP9ZvCntfyTdwLco6SEhNOSnQEYbtW40huHAW/UHieHDlyhLlz51JVVUVUVBRTpkwhNjb2nGIOhXYHgtqtdp/N/udC+aDGifngxOVVIZAPBg0axIEDBzhy8Ah/v+kftFgfT/UxF7Z4Owe67iOqXxTd+3UnJSUlJJ9jen1Qu5uDhswHDVqUOH7VXVJSUq3lSUlJbN++/ZT7ZGVlsXDhQq666irmzJnD7t27ufnmm3G73TzwwAOn3OfRRx/lD3/4w0nL582bd9ZDRHz++edntV+oU7ubF7W76fN6vWzduhWA+fPnf+eb9e9TUVFxznEoH9TYvn07OTk5lJWVUVBQUG/3GwiBeJ7YbDZ27NhBdXU1GzZsYMCAAWdVKAukJvH6EN4eBrQHat4k5uTkcOTIEYqLi2vWRwIbNmGxWIiNjaVTp05ER0cTExODx+Nh3759NdtdNvZ7D7N17tzvXR8KmsT/+yycbbuVD87l8ZIGA6/w3yqoXumfZyfr+MIyCJ+9hZarniA+Pp6WLVs2iuJtY32eHDlyhK+++gqPx0N0dDQDBgxgxYoV9Xb/jbXdgaZ2Ny/KB8HPB1lHPwNq5mrIHDjxm81KgUY2V8ap2u31etmRvQPTadLzZz1JS0tjx44dHMg+gOOQg9T2qSE358e36fWheVG766Yu+aBxffI/BZ/PR2JiIv/4xz+wWq0MGDCAgwcP8sQTT3xnkpkxYwZ33XWX/3ZJSQmtW7dm4sSJxMTE1On4brebzz//nAkTJvi7AzYHarfa3Rw0x3Z7PB4OHTrE1q1bGT9+POHh4XW+j+NXEzW0ppgP4uPjiYmJoU+fPpx33nn1cp/1LdDPk/HjxzNr1iyqq6uxWq1MnDixURQmmsrrg7f4KCXP3glAXmklK3IOY1Z7aAm0BOKcYaTERNJ2+Dg6/PhWgO9td9WubIoXZ+I6kIvF6SDqvL5EDx+ANTy0h99qKv/vujrXdisfnP3jxVt4gJLnZ3B8uA5b/hG25ReRGBVOanQ4h0orOVJeiWPspVjCY/zzvMTHx5OWlkZqaiopKSkN+nrZmJ8nu3btIi8vj27dupGamsr48ePrrYDTmNsdSGq32l0Xygf1mA8qa/JBz6QW9F9b+PVWJjF3PIU1pnEMRfd97f7sqs9JWJnI3sos1r67lmJHCdurtgMmHaI6ElkQxfhXxwUn8HOk1we1uzloyHzQoJ/6W7VqhdVq5fDhw7WWHz58mOTk5FPuk5KSgt1ur3U1b7du3cjLy6O6uvqUbzYdDgcOh+Ok5Xa7/awfSOeybyhTu5sXtbvpMwzD/3p6tu2uj3OlfFDDZrNhtVqx2WyN/jEYqOdJUlIS06dPZ/bs2eTn57NkyRImTpyI5VsTLwdLqL8+2GLiwOth/YEjbC2o6RkRG2anT3IcqTHhRNhtYFgI79iJ8IgIf3fb72q3vXtnort3bsgmNKhQ/3+fLeWDhs8H9qT2uNNa4N6XDxjYvB6sPi8p4Tb6J0bTPzEKo3USZRf8mIMHD3LgwAEKCwspLi6muLiYrVu3YrFYSEpKIi0tjbS0NBISEhrktbOxPU82btzIqlWrMAyDLl26MHr06ICch8bW7oaidjcvygeNJx9YfR5sXjdgEtY+FWfLtLO6/0D6druLdhZxeOlh0s3WHPUVUuwtYlv1NgCSbMm0cMVzaH4ulYeqiGkbHaywz5leH5oXtbvu+52pBv3EHxYWxoABA1iwYIF/mc/nY8GCBQwdOvSU+wwfPpzdu3fj8/n8y3bu3ElKSkqj6L4sIhJKilZ8Sd4HSwDYcc/fKV71VVDiUD6QEyUmJjJ58mRsNhv79u1j4cKFtf7PcvaOFJcytyKSrUdqhoXJaBXDhV3T6NQyuqYgAYBJWN+RwQtSmrXmnA8ir/0z1vjjPRbNWr9tCVHEXv8UrVu3ZsiQIVx66aVcffXVjBs3jq5duxIVFYXP5yM3N5e1a9cyc+ZMXn/9dT777DM2b97MsWPHgtKmhmSaJpmZmaxatQqA3r17M2bMmEZT1BaRulE++O58EHHNn4MSV10d2XQUqLkQrqu9Kzaj5r1mmOGgo72jf7ujX28nIs1bg79ju+uuu/jnP//J66+/zrZt2/jlL39JeXk51113HQDXXHNNrYmNfvnLX1JYWMjtt9/Ozp07mT17No888gi/+tWvGjp0EZGQ5XW5+er6J1j/23mU7KsGIH9dOevumMOmnz+JLwiTNykfyIlSUlL8PSSysrJYunQppmmefkc5JZ/Px7p16/joo4+oaNODCKeTcR1TGdq6FXZr7bd/zhHTsbZICFKkIs03H1jCo4m+8zUiL7oca0I0lggbtoQYIn/wY6JufxWLo/YQi+Hh4XTs2JGRI0fy4x//mCuuuILzzz+fDh064HA4/HPGrFy5kvfee4833niDRYsWsXPnTsrLy4PUysDwer0sXLiQTZs2ATBkyBCGDBmCYRhBjkxEzoXywZnlg8bKav/mPabT4qRrWDciLVF0C+uK3fjm6mlrWN3nNRSRpqfBB22+/PLLKSgo4P777ycvL4++ffsyd+5c/2RG+/btq3V1S+vWrfnss8+488476d27N2lpadx+++387ne/a+jQRURC1o67X+To1q8LD/6Lb2o+uBdsrGbX7/9Jlz/f3KAxKR/It6WnpzN+/Hg+//xzdu7cic1mY8SIEcEOK+QUFRWxaNEi/+TpnXr3Y8glF+Od8wre/bu+2dARTvioH+Ac9YMgRSpSoznnA4vVhmPwZUT60gnftInIvn1xnDfojPaNiYkhJiaGbt26YZomR48e5eDBgxw8eJDc3FwqKirYtWsXu3bVPO/j4uL8Qz2lpKSccviSUFBdXc28efM4dOgQFouF0aNH06lTp2CHJSL1QPng7PJBY5EyIgWL3YLPXdNzpaW1JS2tLWttY3VaSR6adKrdRaSZCcpMkrfccgu33HLLKdctXrz4pGVDhw71d8sVEZG6cRUUcviLUr67c5xB7opjdDhWjL1FbEOGpnwgJ2nXrh1jxoxh4cKFbN26FbvdzuDBg4MdVkgwTZPNmzezevVqvF4vDoeDESNG0LHj193lf/konrx9+AoOQJgTe4ceGPbQ/FJSmh7lg3NjGAatWrWiVatW9OnTB6/Xy+HDh/1FioKCAoqKiigqKmLLli0YhkFCQgKpqamkp6eTlJRUa0z2xqqiooJPP/2Uo0ePYrfbmThxImlpjW+cdRE5e8oHocsZ76TL1Rlse207nGokVgO6X9eNsOjQGVpLRAInKEUJERFpOEfnrMD01RQkThwN58S/fV4LRz9bSfIVUxo4OpGTderUCY/Hw9KlS/nqq6+w2+30798/2GE1amVlZSxevJhDhw4BNb1ORo0aRWRkZK3tbMltILlNMEIUkQZktVpJTU0lNTWV8847j+rqag4dOuQvUhQVFZGfn09+fj4bNmzAarWSkpLiL1K0bNmy0Q2FVFRUxJw5cygrKyM8PJwpU6bQqlWrYIclIiInGPzAICpyK8j5dB+GzcD0mP7f7S9sx8B7BgQ7RBFpJFSUEBFp4rxVLv/f1abnm799Hux8M7anr6q6QeMS+T5du3bF7XaTmZnJ2rVrsdls9O7dO9hhNUo7duxg5cqVuN1ubDYbQ4YMoXv37sEOS0QakbCwMNq1a0e7du0AKC8v9xcoDh48SEVFBQcOHODAgQOsXr0ah8NBWlqav0gRExMT1Pjz8/OZO3cuVVVVxMbGMmXKlKDHJCIiJ7M6rIx7ZSz5a/LZ9e5uKg5XEpEcTsYVnUnon9DoCt4iEjwqSoiINHExfTOAXafdLrpP58AHI1IHvXr1wuPxsGbNGlatWoXNZtOX7SeorKxk6dKl5OTkAJCUlMSYMWP0RZ2InFZkZCQZGRlkZGQAcOzYMX+B4tChQ7hcLrKyssjKygIgKiqKpKQkcnNzqaysxG63f9/d16t9+/Yxf/58PB4PCQkJTJkyBafT2WDHFxGRujEMg6RBSSQN0twRIvLdVJQQEWniYgf3xhHxIa6K7/oCwcQZVU10P33ZK41Pv379cLvdbNiwgeXLl2O32+ncWQW0vXv3smzZMqqqqrBYLAwcOJA+ffro6jMROSstWrSgRYsW9OzZE5/PR0FBgb9IcfjwYcrKyiguLmbTpk2YpklCQkKtSbMDVaTYsWMHS5cuxTRNWrduzYQJE7DZ9BFWREREJNTpHZ2ISBPn2p9LbItjFFS1Aq/5rbUmFouP2NhjVOfmE5aSGJQYRb7PoEGDcLvdbNmyhcWLF2Oz2Wjfvn2wwwqK6upqVqxYwa5dNb2f4uPjGTt2LPHx8UGOTESaCovFQlJSEklJSfTv3x+Px0NeXh7Z2dlkZ2cDUFhYSGFhIZs2bcJisZCYmOgvUiQmJmKxWOp0TG9FJaarGmtMFMbXE25/+eWXrF27FoCMjAxGjhxZ5/sVERERkcZJRQkRkSbOlXMAu91DYmoBh4/aMYprChOG4SMyupzo2FKsNh+unIMqSkijNWzYMDweDzt27GDBggVMmjSJ1q1bBzusBnXw4EEWL15MeXk5hmHQp08fBgwYgPXrL/BERALBZrORnp5OUlISR48eZezYsRQUFHDgwAEOHjxIaWkpeXl55OXlsW7dOux2OykpKf4ixfcVTcs3buPoh59RuW03AJbICGLHDWd7UjTb99Qs69evH+edd16DtFVEREREGoaKEiIiTZxhr3mpt9m8JCZUE1NRQhUppLc5Qrh58nYijZFhGIwcORKPx8OePXuYN28eU6ZMITU1NdihBZzH42H16tVs3rwZgJiYGMaMGUNSksbpFZGG53Q66dChAx06dACgpKSEQ4cOceDAAQ4dOkRVVRX79u1j3759AISHh/sLFGlpaURFRQFQvGQVeS++AZZvhp2rLi3jk3++wmGHlfjp4zh/9Gh69OjR8I0UERERkYDSN1AiIk1cRI8uGDYbpseDwTcf/C2GAebXvSbCwojornH6pXEzDIMxY8bg8XjIyclh7ty5XHDBBSQmNt0ePvn5+SxatIji4mIAunfvzpAhQzSmuog0GjExMcTExNC1a1dM06SwsNA/H8XxibF3797N7t01PR9iY2NJimuB98W3SLSHE0ZNb69qr5flefsoqKrAUmEw0OdQQUJERESkiTrrT7RffPEFgwcPrs9YREQkAP6/vTuPt6qu9wb+2WcGEUFRQAVxnoWUIKccQnHIwqvmkyZEVubQvUl5HTLRvIb1mNcyzets97FHH0krh3BKUhyyHEqvQ07oNQM1RRACzrCeP7geJSA5+xz2Afb7/Xrx8py1fmvt7/ecvc9H+J61V22vnumz7+55+1dTl7mm7/57pKZHU+WKgjLV1NRk1KhRmTJlSv785z/ntttuy0EHHZR11lmnu0vrUm1tbXn00Ufz2GOPpSiK9OzZM3vuuWc23HDD7i4NYJlKpVLWWWedrLPOOtlhhx3S2tqa119/vX1I8frrr+edd97Ja9Meypw/v5wkWbuxR/r3WCOvzZuTdxYuSH1NTXYbMDh9/vhcira2lNxHAgBgtVP2UGLnnXfOZpttlqOOOipHHnlk++W7AKx81j1iTFremZ237/tdUvqfqyVqa5K21vTeY2T6feaT3VsgdEBtbW1Gjx6d2267LTNmzMitt96aT33qU+nTp093l9Yl3nrrrUydOjVvvvlmkmSzzTbLrrvumsbGxm6uDKBjamtrM3DgwAwcODDDhw/PwoUL85e//CV/fP7PebGpKe/Mn5+3Fvwtby34W5KkZ21ddh+4Ufo0NqV1zty0vjsvdb17dXMXAAB0tbJ/7eT//J//k8033zxnn312Nt988+y666655JJL8tZbb3VlfQB0gVJdbdb/6vhs9G/fSI+tNk2S9Nlrl2w06eQMPPaolNwol1VMXV1d9ttvv/Tr1y/z58/PLbfcktmzZ3d3WZ1SFEX++Mc/5sYbb8ybb76ZpqamjBo1KnvvvbeBBLBaaGhoyEYbbZQR22yf/QZvnk9ttEVGrrdBhqzZJxussWb23mDj9Gl8/8rNmob6bqwWAIAVpeyhxBFHHJFbb701r732Wn7wgx+kKIocd9xxWX/99TNmzJhMnjw5Cxcu7MpaAeikpk0Gp/fIYUmSdT/7qTRtPKh7C4JOaGhoyAEHHJC+fftm3rx5ufXWW/Puu+92d1llmT17dm6++eY89NBDaWtry+DBg3PooYe6EhVYLa05YmjS2pYedfUZsmafjFxvg+w2YHDWqG9YtKCmlJ47bJ2aJgNZAIDVUaffoLNfv3454YQT8sADD+S5557LN7/5zTzzzDM5/PDDM2DAgHz5y1/OtGnTuqJWAIDFNDU15cADD0zv3r0zZ86c3Hrrrfnb3/7W3WV1yDPPPJPJkydnxowZqa+vz8c//vHst99+6dmzZ3eXBrBC9Nh6szRtsUmyrPtFFMk6B4+ubFEAAFRMl941rEePHunZs2eamppSFEVKpVJ+8YtfZI899shHP/rRPPXUU135cAAA6dmzZz75yU+mV69eeeedd3LrrbdmwYIF3V3Wh5o3b16mTJmSe++9Ny0tLRk4cGAOPfTQbLXVVt1dGsAKVSqVsuG/HpMeW2y8aENtTVJbm5SSUkN9Bn718+m59WbdWyQAACtM2Te6fs+cOXMyefLkXHvttfnNb36Tmpqa7L///jnjjDNy0EEHpaamJjfddFO+/vWvZ/z48fntb3/bFXUDALTr1atXPvnJT+aXv/xl3nrrrdx222058MAD09DQ0N2lLdWLL76Y++67LwsWLEhNTU1GjBiR7bffPqX3bkQPsJqr7bVGBk38Wub/6aXMefjxFAub0zBoYHrv9tHU9uzR3eUBALAClT2U+MUvfpFrr702t9xyS+bPn5+PfvSjueCCC/K//tf/yjrrrLPY2kMPPTRvv/12jj/++E4XDACwNL17986BBx6Ym2++OW+88UamTJmSAw44IHV1nf4djC6zYMGC3H///Xn++eeTLHobzL322it9+/bt5soAKq9UKqXHlpukx5bunwMAUE3K/lv6wQcfnEGDBuXEE0/M2LFjs+WWW/7D9UOHDs2RRx5Z7sMBAHyovn375oADDsgtt9ySGTNm5I477sjo0aNTW1vb3aXl1VdfzdSpUzNv3ryUSqV85CMfyY477piaZb2nOgAAAKyGyh5K/PrXv86ee+653OtHjBiRESNGlPtwAADLpV+/ftl///1z66235tVXX83dd9+dUaNGdds//re0tOShhx5qv7fWWmutlb322ivrrbdet9QDAAAA3ansv513ZCABAFBJ/fv3b79CYvr06Zk6dWqKoqh4HTNnzszkyZPbBxLbbbddDjnkEAMJAAAAqlbZQ4nTTz89w4YNW+b+j3zkIznrrLPKPT0AQKdssMEG2WeffVJTU5Pnn38+9913X8Ueu7W1NQ8//HB++ctfZvbs2enVq1cOPPDA7LLLLivVPS4AAACg0soeSkyePDn777//MvcfcMABuf7668s9PQBApw0ePDh77713SqVSnnnmmTzwwAMr/DHfeuut3HTTTXn88cdTFEW22GKLHHroodlggw1W+GMDAADAyq7sX9V75ZVXsummmy5z/8Ybb5yXX3653NMDAHSJTTbZJC0tLZk6dWqefPLJNDQ0ZPjw4V3+OEVR5A9/+EN+//vfp62tLU1NTfn4xz+eIUOGdPljAQAAwKqq7KFEr169/uHQ4aWXXkpTU1O5pwcA6DJbbLFFmpubc//99+fRRx9NXV3dP3wbyo6aPXt27rnnnsycOTNJMmTIkOy+++7p0aNHlz0GAAAArA46daPr//iP/8if//znJfb993//dy699NLstddenSoOAKCrbLvtthk5cmSS5OGHH86TTz7ZJed96qmnMnny5MycOTMNDQ3Zc889s++++xpIAAAAwFKUfaXE2WefnREjRmTbbbfN0UcfnW233TZJ8uSTT+bKK69MURQ5++yzu6xQADqnaJmd4i/Xpu21i5J8PG3/dUiKDf4pWe9/pVTbq7vLg4oYOnRompub8+ijj+aBBx5IfX19ttxyy7LONXfu3PzmN7/Jq6++miRZf/31s+eee6ZXL68nAAAAWJayhxJbbrll7rvvvnz1q1/Nv//7vy+27+Mf/3h++MMfZuutt+50gQB0XrHw9RTPHJFi7qtJy7uLNi747xSvnpe8eUOy1U9Tql+ne4uEChk+fHhaWlryxz/+Mb/5zW9SV1f3D++TtTTPP/98pk2bloULF6a2tjYjR47Mtttum1KptIKqBgAAgNVD2UOJJNlhhx3ym9/8Jm+++WZefPHFJItuJtmvX78uKQ6ArlG8dFqy4LUkbX+/J5n/3ymmfyulzS/ujtKgW3zsYx9Lc3Nznn766dxzzz2pq6vLRhtt9KHHzZ8/P9OmTWv//5511103e+21V/r06bOCKwYAAIDVQ6eGEu/p16+fQQTASqqY/0oy+75/sKI1mfXrFAteS6lx/YrVBd1tt912S0tLS5577rnceeed2X///bN+/34pXr0/bXP+mqQhRcvCpL4+SfLKK6/k3nvvzbx581JTU5Mdd9wxw4YNS01N2bfoAgAAgKrT6aHEq6++msceeyzvvPNO2tr+/jdwk7Fjx3b2IQDojLl/bP+w+MDmYrFFRTL3icRQgipSKpWyxx57pLm5OdOnT8+vrp6U/fr8MQPWaE1bGpO+/5LWGw/PgmFH57d/XSfPPPNMkqRPnz7Ze++9/UIGAAAAlKHsocT8+fMzbty4/OxnP0tbW1tKpVKKYtE/cX3w/ZQNJQC6Wen93+Kev6D4wMdt6dnwwXW1FSwKVg41NTX5xCc+kSlXfjuvPPer3F5XkwN26Js+vRe9OP7yxlt54MKT8+6g/VKz7nbZYYcdMnz48NTVdcnFpgAAAFB1yn6/gdNOOy033nhjzjnnnEydOjVFUeSaa67JHXfckf333z9Dhw7NH/7wh66sFYByrDkiyYcMHEr1yZo7VaQcWNnUpC2f6PHbDOzTkIUtbfnVH9/OG7MX5tlnn81tj/81c+a3Zo0Z0/LJAw/Ixz72MQMJAAAA6ISyhxKTJ0/O+PHjc/LJJ2fbbbdNkmywwQYZNWpUbrnllvTp0ycXXXRRlxUKQHlK9f2Sfp/Osn/k1yT9Dk2prm8ly4KVRvGXR1LXMiejt+uT9XrXZ0FLW3756JuZPn16kmSrgT1yyNDGDKh5vXsLBQAAgNVA2UOJ119/PSNGjEiS9OjRI0kyd+7c9v2HHHJIbrzxxk6WB0BXKA0+I1nzo+999j///Z+rJ3rvktLgU7qjLFg5LJiVJKmvq8l+2/fNOr0W3di6oaEh+2y3dnbfcq001NW0rwMAAADKV/ZQon///vnrX/+aJOnZs2f69u2bZ599tn3/7NmzM3/+/M5XCECnlWp7pLTlVSltekHSY8tFG9f6eEqbX5rSFpelVNPUrfVBdyqtMaD948b6mhw4tG8+vlWf7Lrrrhncr2mp6wAAAIDylP2myCNHjsy0adNy8sknJ0kOOuig/O///b8zcODAtLW15d///d/zsY99rMsKBaBzSqXalPp8PDXrvpLMeCI1G387pT5rd3dZ0P3675D0GpC8OzNJkcb6mmw+oCkvNTQkc5OklPQelPTbupsLBQAAgFVf2VdK/PM//3M22WSTLFiwIEly9tlnp0+fPjnqqKMybty4rLXWWvnhD3/YZYUCAKwIpVJNaj52UlIq5f23N2vfm5RqUrPzN1Iq/f0+AAAAoKPKvlJit912y2677db++aBBg/L000/niSeeSG1tbbbaaqvU1ZV9egCAiqnZ8GPJPuen7XcXJm+/8P6OvpumduRxKQ34SPcVBwAAAKuRsqYG8+bNy+c+97kccsghOfLII9u319TUZOjQoV1WHABdr2hr/Z//tnRzJbByqVn/oyl96ppk1ktpffeN5NEZqTvg4pTq67u7NAAAAFhtlPX2TT179sxdd92VefPmdXU9AKwgxZw/p/Wh76ft9xcnSVpv/GxaH/heindndHNlsPIolUop9d0kNQN27O5SAAAAYLVU9j0ldttttzz44INdWQsAK0gx66W03vyFFC/ckRSLrpRI68IUz92S1pvHp3jnle4tEAAAAICqUPZQ4kc/+lHuu+++nH766Xn11Ve7siYAuljrtHOS5nlJ0bb4jqI1WfhuWh84t3sKAwAAAKCqlD2UGDp0aF599dVMmjQpG220URobG9O7d+/F/qy11lpdWSsAZSjeei558+klBxLtC9qSmX9IMWt6ResCAAAAoPqUdaPrJDnkkENSKpW6shYAVoDi7ReXb92sl1LqM2TFFgMAAABAVSt7KHH11Vd3YRkArDB1Te0fNtWXPvBxTZIPXD1R11jBogAAAACoRmW/fRMAq4bSwOFJbcOijz9whdtiF7vV9Uip/0cqXBkAAAAA1absKyV+8pOfLNe6sWPHlvsQAHSBUsMaKW1zeIon/nOZa2q2OyKl+h4VrAoAAACAalT2UOLzn//8Mvd98DdxDSUAul/NR76UtgXvJE//Iin9z0VypdokLSltdUhKQz/fneUBAAAAUCXKHkq89NJLS2xrbW3N9OnTc/HFF+eVV17JNddc06niAOgapZra1O5yctq2OCSlF85MZiWlbQ5L7dYHpLTWRt1dHgAAAABVouyhxEYbLf0fsTbZZJPsvffeOfDAA/OjH/0oF110UdnFAdC1Sn2GpGbQrsmsJ1I79PMp9fCWTQAAAABUzgq70fUnP/nJXH/99Svq9AAAAAAAwCpmhQ0lXnjhhSxYsGBFnR4AAAAAAFjFlP32Tffee+9St8+aNSv33ntvfvjDH2bMmDHlnh4AAAAAAFjNlD2U2HPPPVMqlZbYXhRFamtrc9hhh+XCCy/sVHEAAAAAAMDqo+yhxD333LPEtlKplL59+2ajjTZK7969O1UYAAAAAACweil7KLHHHnt0ZR0AAAAAAMBqruwbXb/00ku5+eabl7n/5ptvzvTp08s9PQAAAAAAsJopeyjxjW98Iz/84Q+Xuf+iiy7KKaecssx9Q4YMSVNTU0aOHJmHH354uR7zuuuuS6lUcgNtgNWEPAAgkQcALCIPAKpD2UOJBx98MPvss88y93/iE5/Ifffdt8T266+/PhMmTMjEiRPz6KOPZujQoRk9enRef/31f/h406dPzze+8Y3svvvu5ZYMwEpEHgCQyAMAFpEHANWj7KHE22+/nTXXXHOZ+3v16pW//vWvS2w///zz86UvfSnjx4/PNttsk0suuSQ9e/bMlVdeucxztba25sgjj8xZZ52VTTbZpNySAViJyAMAEnkAwCLyAKB6lH2j68GDB+f+++/Pscceu9T99913XzbccMPFti1cuDCPPPJITj311PZtNTU1GTVqVB588MFlPta3v/3trLfeejn66KOXevXF31uwYEEWLFjQ/vns2bOTJM3NzWlubv7Q4z/ovfUdPW5Vp299V4Nq7LulpSWtra1JFvVdV9fxGOiKr5c8WOS970dLS8tK+zysxtdJom99V4fO9i0P5EE10Le+q4E8kAcd4XWi72qg7xWfB2UPJT772c/m7LPPzogRI3LCCSekpmbRRRetra350Y9+lOuvvz7f/OY3FzvmzTffTGtra/r377/Y9v79++eZZ55Z6uNMmzYtV1xxRR5//PHlrm3SpEk566yzlth+xx13pGfPnst9ng+68847yzpuVafv6qLv1V9ra2ueeuqpJMldd92V2traDp9j3rx5na5DHizyzDPP5OWXX867776bN954o8vOuyJU0+vkg/RdXfTdMfJAHlQTfVcXfXeMPJAH1UTf1UXfHdORPCh7KHHqqadm2rRp+drXvpZzzjknW265ZZLk2WefzRtvvJE999xziaFER82ZMydHHXVULrvssvTr169DtU2YMKH989mzZ2fQoEHZd99907t37w7V0NzcnDvvvDP77LNP6uvrO3Tsqkzf+q4G1dh3S0tLXnvttTz11FMZNWpUevTo0eFzvPfbRJW0uubB2muvnd69e2fo0KH56Ec/2iXn7GrV+DpJ9K3v6tDZvuWBPKgG+tZ3NZAH8qAjvE70XQ30veLzoOyhRGNjY+64445cc801ufHGG/PCCy8kSUaMGJFDDjkkY8eObb964j39+vVLbW1tZs6cudj2mTNnZsCAAUs8xgsvvJDp06fnoIMOat/W1ta2qPC6ujz77LPZdNNNl1pbY2PjEtvr6+vLfiJ15thVmb6ri75Xf6VSqf3qiHL77oqvlTxYpK6uLrW1tamrq1vpn4PV9Dr5IH1XF313/LjOkgeLyIOVn76ri747flxnyYNF5MHKT9/VRd8dP255lT2USBa9v9/48eMzfvz45Vrf0NCQnXbaKXfffXfGjBmTZFFo3H333TnhhBOWWL/VVlvliSeeWGzb6aefnjlz5uQHP/hBBg0a1JnyAegm8gCARB4AsIg8AKguZQ8l3nrrrbz66qvZYYcdlrr/iSeeyIYbbpi+ffsutn3ChAkZN25chg8fnhEjRuSCCy7I3Llz2wcbY8eOzQYbbJBJkyalqakp22233WLH9+nTJ0mW2A7AqkUeAJDIAwAWkQcA1aPsocSJJ56YZ599Ng899NBS9x9zzDHZeuutc8UVVyy2/fDDD88bb7yRM844IzNmzMiwYcMyZcqU9psZvfLKK0u87RMAqx95AEAiDwBYRB4AVI+yhxK//vWvc+yxxy5z/0EHHZRLLrlkqftOOOGEpV5+lyRTp079h4979dVXL2+JAKzk5AEAiTwAYBF5AFAdyh4xv/HGG+nXr98y96+zzjp5/fXXyz09AAAAAACwmil7KDFw4MA89thjy9z/yCOPZN111y339AAAAAAAwGqm7KHEmDFjcsUVV+SXv/zlEvt+8Ytf5KqrrsrBBx/cqeIAAAAAAIDVR9n3lDjzzDNz11135eCDD87QoUOz3XbbJUmefPLJ/OEPf8jWW2+ds846q8sKBQAAAAAAVm1lXymx1lpr5aGHHsrpp5+e5ubmTJ48OZMnT05zc3O+9a1v5be//W369OnThaUCAAAAAACrsrKvlEiSNdZYI2eddZYrIgAAAAAAgA9V9pUSAAAAAAAAHdGpKyXmz5+fn/3sZ3n00UfzzjvvpK2tbbH9pVIpV1xxRacKBAAAAAAAVg9lDyVefvnl7LXXXpk+fXr69OmTd955J2uvvXZmzZqV1tbW9OvXL7169erKWgEAAAAAgFVY2W/fdNJJJ+Wdd97JQw89lD/96U8piiLXX3993n333Xz3u99Njx49cvvtt3dlrQAAAAAAwCqs7KHEr3/96xx33HEZMWJEamoWnaYoijQ2Nuakk07KJz7xiXzta1/rqjoBAAAAAIBVXNlDiXnz5mXIkCFJkt69e6dUKuWdd95p37/zzjtn2rRpnS4QAAAAAABYPZQ9lBg8eHBeffXVJEldXV022GCDPPTQQ+37n3rqqTQ1NXW+QgAAAAAAYLVQ9o2u99577/ziF7/IxIkTkySf//znM2nSpLz99ttpa2vLf/7nf2bs2LFdVigAAAAAALBqK3soccopp+R3v/tdFixYkMbGxpx22ml57bXXMnny5NTW1uaII47I+eef35W1AgAAAAAAq7CyhxKDBw/O4MGD2z9vamrK5Zdfnssvv7xLCgMAAAAAAFYvZd9TAgAAAAAAoCMMJQAAAAAAgIowlAAAAAAAACrCUAIAAAAAAKgIQwkAAAAAAKAiDCUAAAAAAICKMJQAAAAAAAAqwlACAAAAAACoCEMJAAAAAACgIgwlAAAAAACAijCUAAAAAAAAKsJQAgAAAAAAqAhDCQAAAAAAoCIMJQAAAAAAgIowlAAAAAAAACrCUAIAAAAAAKgIQwkAAAAAAKAiDCUAAAAAAICKMJQAAAAAAAAqwlACAAAAAACoCEMJAAAAAACgIgwlAAAAAACAijCUAAAAAAAAKsJQAgAAAAAAqAhDCQAAAAAAoCIMJQAAAAAAgIowlAAAAAAAACrCUAIAAAAAAKgIQwmAKtNWzP2f/87q3kIAAAAAqDqGEgBVorXtqcxt/mIWtH4/SfJu896Z2/yltLY9282VAQAAAFAtDCUAqkBL2+N5t+WQtBTTPrC1SEsxNe+2/FNa257sttoAAAAAqB6GEgCruaIo8reWk5M0J2n9u72tSRZkXutplS8MAAAAgKpjKAGwmmst/pi2PJekbRkr2tJWPJnWtqcqWRYAAAAAVchQAmA111a8tHzrMn3FFgIAAABA1TOUAFjNlUprLufKXiu0DgAAAAAwlABYzdWVdkmyxqKP60o5cuxaGTVqVOrqSu1rSlkrdaWR3VQhAAAAANXCUAJgNVcq9UhT7fHtn9fVlVJbW7vYmsbar6ZUaqx0aQAAAABUGUMJgCrQUHNMGmtOSFKb93/01ySpS2PtiWmoGd99xQEAAABQNeq6uwAAVrxSqZSmuglpKI7K39puSZI01Z6UHvWfTE2pXzdXBwAAAEC1cKUEQBWpKa2bhtrPJUkaao80kOhmRdGStmJuiqKlu0sBoBvJAwASeQBUj24ZSlx00UUZMmRImpqaMnLkyDz88MPLXHvZZZdl9913T9++fdO3b9+MGjXqH64HYNVRrXnQVrySec0nZ27L57Kg9ZzMbTki81pOS1vx5+4uDaBbyAN5AJDIA3kAVIuKDyWuv/76TJgwIRMnTsyjjz6aoUOHZvTo0Xn99deXun7q1Kn57Gc/m3vuuScPPvhgBg0alH333Td//rMfzACrsmrNg9a2P2VO86fSXNyYpPl/tjanue2GvNt8UFqLF7uzPICKkwfyACCRB/IAqCYVH0qcf/75+dKXvpTx48dnm222ySWXXJKePXvmyiuvXOr6a6+9Nscdd1yGDRuWrbbaKpdffnna2tpy9913V7hyALpStebB31r/NcncJK1/t6c1Rebkby2ndkNVAN1HHsgDgEQeyAOgmlR0KLFw4cI88sgjGTVq1PsF1NRk1KhRefDBB5frHPPmzUtzc3PWXnvtFVUmACtYteZBa9tTaS3+mCX/wtG+Iq3F79JavFDJsgC6jTyQBwCJPJAHQLWpq+SDvfnmm2ltbU3//v0X296/f/8888wzy3WOk08+Oeuvv/5iQfX3FixYkAULFrR/Pnv27CRJc3Nzmpubl3XYUr23vqPHrer0re9qoO/y+u6Kr1e15sHC1j+lpbWx/fPWlra0thZpbalPS/P72xcUz6a+ZnDZj9OVvE70XQ30LQ86Qh54nVQDfeu7nOM7Qx4sIg9WXvrWdzWoZB5UdCjRWeeee26uu+66TJ06NU1NTctcN2nSpJx11llLbL/jjjvSs2fPsh77zjvvLOu4VZ2+q4u+q0u5fc+bN6+LK+m4VTcPSknObf/spZdeysy//CV1xQaZN2ujD6xrTnJbJx6n63mdVBd9Vxd5IA86wuukuui7usgDedARXifVRd/VpRJ5UNGhRL9+/VJbW5uZM2cutn3mzJkZMGDAPzz2vPPOy7nnnpu77rorO+ywwz9ce+qpp2bChAntn8+ePbv9hke9e/fuUM3Nzc258847s88++6S+vr5Dx67K9K3vaqDv8vp+77eJOqNa86Ao5mRO895JFv121m5LXdUza9ZPTam07L9MVZLXib6rgb7lQUfIA6+TaqBvfXeEPJAH1UDf+q4GlcyDig4lGhoastNOO+Xuu+/OmDFjkqT9JkQnnHDCMo/73ve+l3POOSe33357hg8f/qGP09jYmMbGxiW219fXl/1E6syxqzJ9Vxd9V5dy++6Kr1X15sHaWaPmyCxouyRJsdQVjbXHpqF2zTLPv+J4nVQXfVcXeSAPOsLrpLrou7rIA3nQEV4n1UXf1aUSeVDxt2+aMGFCxo0bl+HDh2fEiBG54IILMnfu3IwfPz5JMnbs2GywwQaZNGlSkuS73/1uzjjjjPz0pz/NkCFDMmPGjCRJr1690qtXr0qXD0AXqdY8aKydkLa8nea265LUfmBPa+prjkpjzbL/0gWwOpIH8gAgkQfyAKgmFR9KHH744XnjjTdyxhlnZMaMGRk2bFimTJnSfjOjV155JTU1Ne3rf/zjH2fhwoU59NBDFzvPxIkTc+aZZ1aydAC6ULXmQalUm55130lr8YU0t96YtryRmvRPfe0/pba0SXeXB1Bx8kAeACTyQB4A1aRbbnR9wgknLPPyu6lTpy72+fTp01d8QQB0i2rOg9rSZqmt+9fuLgNgpSAP5AFAIg/kAVAtaj58CQAAAAAAQOcZSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARXTLUOKiiy7KkCFD0tTUlJEjR+bhhx/+h+tvuOGGbLXVVmlqasr222+f2267rUKVArAiyQMAEnkAwCLyAKA6VHwocf3112fChAmZOHFiHn300QwdOjSjR4/O66+/vtT1DzzwQD772c/m6KOPzmOPPZYxY8ZkzJgxefLJJytcOQBdSR4AkMgDABaRBwDVo+JDifPPPz9f+tKXMn78+GyzzTa55JJL0rNnz1x55ZVLXf+DH/wg++23X0466aRsvfXWOfvss7PjjjvmRz/6UYUrB6AryQMAEnkAwCLyAKB61FXywRYuXJhHHnkkp556avu2mpqajBo1Kg8++OBSj3nwwQczYcKExbaNHj06P//5z5f5OAsWLMiCBQvaP589e3aSpLm5Oc3NzR2q+b31HT1uVadvfVcDfZfXd1d8veTBqkPf+q4G+pYHHeH5ou9qoG99l3N8Z8iDVYe+9V0N9L3i86CiQ4k333wzra2t6d+//2Lb+/fvn2eeeWapx8yYMWOp62fMmLHMx5k0aVLOOuusJbbfcccd6dmzZxmVJ3feeWdZx63q9F1d9F1dyu173rx5nX5sebDq0Xd10Xd1kQfyoCP0XV30XV3kgTzoCH1XF31Xl0rkQUWHEpVy6qmnLjYtnz17dgYNGpR99903vXv37tC5mpubc+edd2afffZJfX19V5e60tK3vquBvsvr+73fJloVyIPO07e+q4G+5UFHeL7ouxroW98dIQ88X6qBvvVdDSqZBxUdSvTr1y+1tbWZOXPmYttnzpyZAQMGLPWYAQMGdGh9kjQ2NqaxsXGJ7fX19WU/kTpz7KpM39VF39Wl3L674mslD1Y9+q4u+q4u8kAedIS+q4u+q4s8kAcdoe/qou/qUok8qOiNrhsaGrLTTjvl7rvvbt/W1taWu+++OzvvvPNSj9l5550XW58suoRkWesBWPnJAwASeQDAIvIAoLpU/O2bJkyYkHHjxmX48OEZMWJELrjggsydOzfjx49PkowdOzYbbLBBJk2alCT5l3/5l+yxxx75/ve/nwMPPDDXXXddfv/73+fSSy+tdOkAdCF5AEAiDwBYRB4AVI+KDyUOP/zwvPHGGznjjDMyY8aMDBs2LFOmTGm/OdErr7ySmpr3L+DYZZdd8tOf/jSnn356TjvttGy++eb5+c9/nu22267SpQPQheQBAIk8AGAReQBQPbrlRtcnnHBCTjjhhKXumzp16hLbDjvssBx22GEruCoAKk0eAJDIAwAWkQcA1aGi95QAAAAAAACql6EEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFREXXcXUAlFUSRJZs+e3eFjm5ubM2/evMyePTv19fVdXdpKS9/6rgb6Lq/v936WvvezdVUiDzpO3/quBvqWBx3h+aLvaqBvfXeEPPB8qQb61nc1qGQeVMVQYs6cOUmSQYMGdXMlAKuPOXPmZK211uruMjpEHgB0PXkAQCIPAFhkefKgVKyKo+wOamtry2uvvZY111wzpVKpQ8fOnj07gwYNyn//93+nd+/eK6jClY++9V0N9F1e30VRZM6cOVl//fVTU7NqvQugPOg4feu7GuhbHnSE54u+q4G+9d0R8sDzpRroW9/VoJJ5UBVXStTU1GTDDTfs1Dl69+5dVU/C9+i7uui7unSm71XtN6DeIw/Kp+/qou/qIg/K4/lSXfRdXfTdcfLA86Va6Lu66LvjljcPVq0RNgAAAAAAsMoylAAAAAAAACrCUOJDNDY2ZuLEiWlsbOzuUipK3/quBvqurr47q1q/bvrWdzXQd3X13VnV+nXTt76rgb6rq+/Oqtavm771XQ30veL7roobXQMAAAAAAN3PlRIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSSS666KIMGTIkTU1NGTlyZB5++OF/uP6GG27IVlttlaampmy//fa57bbbKlRp1+pI35dddll233339O3bN3379s2oUaM+9Ou0suro9/s91113XUqlUsaMGbNiC1xBOtr3rFmzcvzxx2fgwIFpbGzMFltssUo+1zva9wUXXJAtt9wyPXr0yKBBg3LiiSdm/vz5Faq2a9x777056KCDsv7666dUKuXnP//5hx4zderU7LjjjmlsbMxmm22Wq6++eoXXuTKSB/JgecgDebCqkAflkwfyYHnIA3mwqpAH5ZMH8mB5yAN5sKpYqfKgqHLXXXdd0dDQUFx55ZXFf/3XfxVf+tKXij59+hQzZ85c6vr777+/qK2tLb73ve8VTz31VHH66acX9fX1xRNPPFHhyjuno30fccQRxUUXXVQ89thjxdNPP118/vOfL9Zaa63i1VdfrXDlndPRvt/z0ksvFRtssEGx++67F5/+9KcrU2wX6mjfCxYsKIYPH14ccMABxbRp04qXXnqpmDp1avH4449XuPLO6Wjf1157bdHY2Fhce+21xUsvvVTcfvvtxcCBA4sTTzyxwpV3zm233VZ885vfLG688cYiSXHTTTf9w/Uvvvhi0bNnz2LChAnFU089VVx44YVFbW1tMWXKlMoUvJKQB/JAHixJHsgDeSAP5MHSyQN5sCqRB+WRB/JAHixJHsiDrsqDqh9KjBgxojj++OPbP29tbS3WX3/9YtKkSUtd/5nPfKY48MADF9s2cuTI4phjjlmhdXa1jvb991paWoo111yzuOaaa1ZUiStEOX23tLQUu+yyS3H55ZcX48aNWyVDpqN9//jHPy422WSTYuHChZUqcYXoaN/HH398sffeey+2bcKECcWuu+66QutckZYnZP71X/+12HbbbRfbdvjhhxejR49egZWtfOTBIvJAHnyQPHifPKge8mAReSAPPkgevE8eVA95sIg8kAcfJA/eJw86p6rfvmnhwoV55JFHMmrUqPZtNTU1GTVqVB588MGlHvPggw8utj5JRo8evcz1K6Ny+v578+bNS3Nzc9Zee+0VVWaXK7fvb3/721lvvfVy9NFHV6LMLldO37/85S+z88475/jjj0///v2z3Xbb5Tvf+U5aW1srVXanldP3LrvskkceeaT9kr0XX3wxt912Ww444ICK1NxdVoefa50lD+SBPJAHHyQP3req/VzrLHkgD+SBPPggefC+Ve3nWmfJA3kgD+TBB8mD93XVz7W6Tp9hFfbmm2+mtbU1/fv3X2x7//7988wzzyz1mBkzZix1/YwZM1ZYnV2tnL7/3sknn5z1119/iSfmyqycvqdNm5Yrrrgijz/+eAUqXDHK6fvFF1/Mr3/96xx55JG57bbb8vzzz+e4445Lc3NzJk6cWImyO62cvo844oi8+eab2W233VIURVpaWvKVr3wlp512WiVK7jbL+rk2e/bs/O1vf0uPHj26qbLKkQfyIJEHSyMP5IE8WEQefDh5sOqQB/Lgw8gDeSAPFpEHS5IH8qCr8qCqr5SgPOeee26uu+663HTTTWlqauruclaYOXPm5Kijjspll12Wfv36dXc5FdXW1pb11lsvl156aXbaaaccfvjh+eY3v5lLLrmku0tboaZOnZrvfOc7ufjii/Poo4/mxhtvzK233pqzzz67u0uDlZI8WP3JA3kAy0MerP7kgTyA5SEPVn/yQB50laq+UqJfv36pra3NzJkzF9s+c+bMDBgwYKnHDBgwoEPrV0bl9P2e8847L+eee27uuuuu7LDDDiuyzC7X0b5feOGFTJ8+PQcddFD7tra2tiRJXV1dnn322Wy66aYrtuguUM73e+DAgamvr09tbW37tq233jozZszIwoUL09DQsEJr7grl9P2tb30rRx11VL74xS8mSbbffvvMnTs3X/7yl/PNb34zNTWr5xx3WT/XevfuXRW/BZXIA3mwiDxYkjyQB/JgEXmwbPJAHsiD1Y88kAfyYBF5sCR5IA+6Kg9Wz6/YcmpoaMhOO+2Uu+++u31bW1tb7r777uy8885LPWbnnXdebH2S3HnnnctcvzIqp+8k+d73vpezzz47U6ZMyfDhwytRapfqaN9bbbVVnnjiiTz++OPtfz71qU9lr732yuOPP55BgwZVsvyylfP93nXXXfP888+3h2qS/OlPf8rAgQNXiYBJyut73rx5SwTJe0G76B5Aq6fV4edaZ8kDeSAP5MEHyYP3rWo/1zpLHsgDeSAPPkgevG9V+7nWWfJAHsgDefBB8uB9XfZzrdO3yl7FXXfddUVjY2Nx9dVXF0899VTx5S9/uejTp08xY8aMoiiK4qijjipOOeWU9vX3339/UVdXV5x33nnF008/XUycOLGor68vnnjiie5qoSwd7fvcc88tGhoaismTJxd/+ctf2v/MmTOnu1ooS0f7/nvjxo0rPv3pT1eo2q7T0b5feeWVYs011yxOOOGE4tlnny1uueWWYr311iv+7d/+rbtaKEtH+544cWKx5pprFv/3//7f4sUXXyzuuOOOYtNNNy0+85nPdFcLZZkzZ07x2GOPFY899liRpDj//POLxx57rHj55ZeLoiiKU045pTjqqKPa17/44otFz549i5NOOql4+umni4suuqiora0tpkyZ0l0tdAt5IA/kgTx4jzyQB/JAHsgDeVAU8kAeyAN5IA/kwSLyoOvzoOqHEkVRFBdeeGExePDgoqGhoRgxYkTx0EMPte/bY489inHjxi22/v/9v/9XbLHFFkVDQ0Ox7bbbFrfeemuFK+4aHel7o402KpIs8WfixImVL7yTOvr9/qBVNWSKouN9P/DAA8XIkSOLxsbGYpNNNinOOeecoqWlpcJVd15H+m5ubi7OPPPMYtNNNy2ampqKQYMGFccdd1zx9ttvV77wTrjnnnuW+np9r9dx48YVe+yxxxLHDBs2rGhoaCg22WST4qqrrqp43SsDeSAP3iMP3icP5EE1kgfy4D3y4H3yQB5UI3kgD94jD94nD+RBVygVxWp8jQkAAAAAALDSqOp7SgAAAAAAAJVjKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAALAKGDJkSD75yU92dxkAAACdYigBAAAs1bx583LmmWdm6tSp3V0KAACwmjCUAAAAlmrevHk566yzDCUAAIAuYygBAADdZO7cud1dAgAAQEUZSgAAQAWceeaZKZVKeeqpp3LEEUekb9++2W233dLS0pKzzz47m266aRobGzNkyJCcdtppWbBgwVLPc8cdd2TYsGFpamrKNttskxtvvHGpj/P3rr766pRKpUyfPr192+9///uMHj06/fr1S48ePbLxxhvnC1/4QpJk+vTpWXfddZMkZ511VkqlUkqlUs4888wkyec///n06tUrf/7znzNmzJj06tUr6667br7xjW+ktbV1scdua2vLBRdckG233TZNTU3p379/jjnmmLz99tuLrftH9bznuuuuy0477ZQ111wzvXv3zvbbb58f/OAHH/4NAAAAVgp13V0AAABUk8MOOyybb755vvOd76Qoinzxi1/MNddck0MPPTRf//rX89vf/jaTJk3K008/nZtuummxY5977rkcfvjh+cpXvpJx48blqquuymGHHZYpU6Zkn3326VAdr7/+evbdd9+su+66OeWUU9KnT59Mnz69fcix7rrr5sc//nGOPfbYHHzwwfmnf/qnJMkOO+zQfo7W1taMHj06I0eOzHnnnZe77ror3//+97Ppppvm2GOPbV93zDHH5Oqrr8748ePzz//8z3nppZfyox/9KI899ljuv//+1NfXf2g9SXLnnXfms5/9bD7xiU/ku9/9bpLk6aefzv33359/+Zd/6dg3AgAA6BaGEgAAUEFDhw7NT3/60yTJH/7whxx//PH54he/mMsuuyxJctxxx2W99dbLeeedl3vuuSd77bVX+7F/+tOf8rOf/ax9QHD00Udnq622ysknn9zhocQDDzyQt99+O3fccUeGDx/evv3f/u3fkiRrrLFGDj300Bx77LHZYYcd8rnPfW6Jc8yfPz+HH354vvWtbyVJvvKVr2THHXfMFVdc0T6UmDZtWi6//PJce+21OeKII9qP3WuvvbLffvvlhhtuyBFHHPGh9STJrbfemt69e+f2229PbW1th/oFAABWDt6+CQAAKugrX/lK+8e33XZbkmTChAmLrfn617+eZNE/wn/Q+uuvn4MPPrj98969e2fs2LF57LHHMmPGjA7V0adPnyTJLbfckubm5g4d+0Ef7CdJdt9997z44ovtn99www1Za621ss8+++TNN99s/7PTTjulV69eueeee5a7nj59+mTu3Lm58847y64XAADoXoYSAABQQRtvvHH7xy+//HJqamqy2WabLbZmwIAB6dOnT15++eXFtm+22WZL3C9iiy22SJLF7hWxPPbYY48ccsghOeuss9KvX798+tOfzlVXXbXMe1ksTVNTU/t9J97Tt2/fxe4V8dxzz+Wdd97Jeuutl3XXXXexP++++25ef/315a7nuOOOyxZbbJH9998/G264Yb7whS9kypQpHeobAADoXt6+CQAAKqhHjx5LbFvajanLtaxz/f3Np0ulUiZPnpyHHnooN998c26//fZ84QtfyPe///089NBD6dWr14c+1vK8hVJbW1vWW2+9XHvttUvd/95QY3nqWW+99fL444/n9ttvz69+9av86le/ylVXXZWxY8fmmmuu+dBaAACA7udKCQAA6CYbbbRR2tra8txzzy22febMmZk1a1Y22mijxbY///zzKYpisW1/+tOfkiRDhgxJsuhKhSSZNWvWYuv+/qqL93zsYx/LOeeck9///ve59tpr81//9V+57rrrknTNsGTTTTfNX//61+y6664ZNWrUEn+GDh263PUkSUNDQw466KBcfPHFeeGFF3LMMcfkJz/5SZ5//vlO1woAAKx4hhIAANBNDjjggCTJBRdcsNj2888/P0ly4IEHLrb9tddey0033dT++ezZs/OTn/wkw4YNy4ABA5IsGgIkyb333tu+bu7cuUtcSfD2228vMeAYNmxYkrS/ZVLPnj2TLDng6IjPfOYzaW1tzdlnn73EvpaWlvZzL089f/3rXxfbX1NTkx122GGxNQAAwMrN2zcBAEA3GTp0aMaNG5dLL700s2bNyh577JGHH34411xzTcaMGZO99tprsfVbbLFFjj766Pzud79L//79c+WVV2bmzJm56qqr2tfsu+++GTx4cI4++uicdNJJqa2tzZVXXpl11103r7zySvu6a665JhdffHEOPvjgbLrpppkzZ04uu+yy9O7du31Y0qNHj2yzzTa5/vrrs8UWW2TttdfOdtttl+222265e9xjjz1yzDHHZNKkSXn88cez7777pr6+Ps8991xuuOGG/OAHP8ihhx66XPV88YtfzFtvvZW99947G264YV5++eVceOGFGTZsWLbeeuvOfCsAAIAKMZQAAIBudPnll2eTTTbJ1VdfnZtuuikDBgzIqaeemokTJy6xdvPNN8+FF16Yk046Kc8++2w23njjXH/99Rk9enT7mvr6+tx000057rjj8q1vfSsDBgzI1772tfTt2zfjx49vX/feAOS6667LzJkzs9Zaa2XEiBG59tprF7sZ9+WXX56vfvWrOfHEE7Nw4cJMnDixQ0OJJLnkkkuy00475T/+4z9y2mmnpa6uLkOGDMnnPve57Lrrrstdz+c+97lceumlufjiizNr1qwMGDAghx9+eM4888zU1LgIHAAAVgWl4u+vkQYAAAAAAFgB/DoRAAAAAABQEYYSAAAAAABARRhKAAAAAAAAFWEoAQAAAAAAVIShBAAAAAAAUBGGEgAAAAAAQEUYSgAAAAAAABVhKAEAAAAAAFSEoQQAAAAAAFARhhIAAAAAAEBFGEoAAAAAAAAVYSgBAAAAAABUhKEEAAAAAABQEf8fBoGOjWI3Ge0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_changes(r_teaser, param_name='cons_preds', height=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we see the same behavior on the first subplot. But in other cases model demonstrates the greater hesitation for the most early points. Further, the greater amount of consecutive predictions is required, the lefter point shift. So, one may conclude the OneClassSVM approach is not as stable as the thresholding is or the underfitting of classifiers is present." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EconomyK" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "EconomyK algorithm widens the boundaries of basic classsifiers queue with the estimation of prefixes clustering results. Clustering is conducted with fast KMeans during the training phase and for prefixes the required length is cropped from centroids' coordinates. \n", + "\n", + "The accessed values of probability of being labeled as a cluster member is recalculated and used to ensure the slave prediction. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import fedot_ind.core.models.early_tc.economy_k as ECONOMYK\n", + "params = [1e-3, 1, 1e5, 1e7]\n", + "r_economy_k = eval_param_influence(ECONOMYK.EconomyK,\n", + " 'lambda_', params,\n", + " prediction_mode='all', interval_percentage=INTEVAL_PERCENTAGE)" + ] + }, + { + "cell_type": "code", + "execution_count": 201, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABiUAAAHgCAYAAADKeW7zAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACds0lEQVR4nOzdd3xV9f3H8fe9N8m92TuMsFH2li24Ki5EbcVdoFTbKuKitQpV0FrFUSfiqAOtuCpaaatVwRkkDBUEZSN7ZQ8yb+45vz/yyy2XBMi69+bmvJ6PRx6YMz/fQ3LfeD73nq/NNE1TAAAAAAAAAAAAfmYPdgEAAAAAAAAAAMAaaEoAAAAAAAAAAICAoCkBAAAAAAAAAAACgqYEAAAAAAAAAAAICJoSAAAAAAAAAAAgIGhKAAAAAAAAAACAgKApAQAAAAAAAAAAAoKmBAAAAAAAAAAACAiaEgAAAAAAAAAAICBoSqBFeOWVV2Sz2bRz586An/uee+6RzWZTTk5Oo49hs9l0zz33NF9RAIAmI1sAILh4HQaA1ovXeABNQVMCsIDly5drzJgxioqKUtu2bXXzzTfr8OHD9d7/pZdeUu/eveVyuXTyySdr3rx5dW63b98+XX755UpISFBcXJwuvvhi/fTTT7W2e/bZZ3XZZZepU6dOstls+tWvftXYoQEAguCTTz7Rtddeq379+snhcKhLly7BLgkALKUhr8OGYejhhx9W165d5XK5NGDAAL355pt1brtx40add955iomJUVJSkiZNmqTs7OwWd8y6FBQU6Le//a1SU1MVHR2tM888U99991299weAlsLqr/HkhjXQlABaubVr1+pnP/uZSktL9dhjj+m6667T3/72N1122WX12v/555/Xddddp759+2revHkaNWqUbr75Zj300EM+2x0+fFhnnnmmvvzyS82aNUv33nuv1qxZo9NPP125ubk+2z700EP67LPP1LdvX4WFhTXbWAEAgfHGG2/ojTfeUHx8vNq3bx/scgDAchryOvynP/1Jd9xxh8aNG6d58+apU6dOuvrqq/XWW2/5bLd3716ddtpp2rZtmx544AH94Q9/0AcffKBx48apsrKyxRyzLoZhaPz48XrjjTc0ffp0Pfzww8rKytIZZ5yhrVu3nnB/AGhJrPwaT25YiAm0AAsWLDAlmTt27Aj4uefMmWNKMrOzsxt9DEnmnDlzmq+oZnT++eeb7dq1MwsLC73LXnjhBVOS+fHHHx9339LSUjM5OdkcP368z/JrrrnGjI6ONvPy8rzLHnroIVOSuWrVKu+yjRs3mg6Hw5w5c6bP/jt37jQNwzBN0zSjo6PNKVOmNHZ4AHBMZIv/7Nu3z6ysrDRN0zTHjx9vdu7cObgFAWiReB32n/q+Du/du9cMDw83b7zxRu8ywzDMsWPHmh06dDCrqqq8y2+44QYzMjLS3LVrl3fZkiVLTEnm888/32KOWZe3337blGS+88473mVZWVlmQkKCedVVVx13XwCNw2u8/1j5NZ7csA4+KYEWa/HixRo/frzat28vp9Op7t2767777pPH4/HZ7owzzlC/fv20bt06nX766YqKitJJJ52kRYsWSZK+/PJLjRgxQpGRkerZs6eWLl1a5/lycnJ0+eWXKy4uTsnJybrllltUXl7us01FRYVuu+02paamKjY2VhdddJH27t1b61i7du3StGnT1LNnT0VGRio5OVmXXXZZwJ+1WFRUpCVLluiXv/yl4uLivMsnT56smJgY/eMf/zju/p9//rlyc3M1bdo0n+U33nijSkpK9MEHH3iXLVq0SMOGDdOwYcO8y3r16qWf/exntc7TuXNn2Wy2pgwNABqFbGke7du3V3h4eMDPCyD08TrcPOr7Orx48WK53W6ff8/bbDbdcMMN2rt3rzIzM73L3333XV144YXq1KmTd9nZZ5+tHj16+Px7PtjHrMuiRYvUpk0b/eIXv/AuS01N1eWXX67FixeroqLihNcKQNPxGt88rPwaT25YB00JtFivvPKKYmJiNGPGDD355JM65ZRTNHv2bN155521ts3Pz9eFF16oESNG6OGHH5bT6dSVV16pt99+W1deeaUuuOACPfjggyopKdHEiRNVXFxc6xiXX365ysvLNXfuXF1wwQV66qmn9Nvf/tZnm+uuu05PPPGEzjnnHD344IMKDw/X+PHjax1r9erVWr58ua688ko99dRTuv766/Xpp5/qjDPOUGlp6QnHnp+fr5ycnBN+nehY69evV1VVlYYOHeqzPCIiQoMGDdKaNWuOu3/N+qP3P+WUU2S3273rDcPQunXram0nScOHD9f27dvrvOYAEGhkS9OzBQCagtfhwL4Or1mzRtHR0erdu7fP8uHDh3vXS9Vzw2VlZR3z3/NH/n9DMI95vHEOGTJEdrvvLY7hw4ertLRUW7ZsOe7+AJoHr/G8xre0Yx4LudECBPujGoBp1v2xv9LS0lrb/e53vzOjoqLM8vJy77LTTz/dlGS+8cYb3mWbNm0yJZl2u91csWKFd/nHH39sSjIXLFjgXVbzsb+LLrrI51zTpk0zJZnff/+9aZqmuXbtWlOSOW3aNJ/trr766lof+6ur9szMTFOS+fe///34F8M0zc6dO5uSTvh1oo8avvPOO6Yk86uvvqq17rLLLjPbtm173P1vvPFG0+Fw1LkuNTXVvPLKK03TNM3s7GxTkvnnP/+51nbz5883JZmbNm2q8zg8vgmAv5AtvporW47G45sAHAuvw76C8To8fvx4s1u3brWWl5SUmJLMO++80zRN01y9evUxx3H77bebkrx/P8E85rFER0ebv/71r2st/+CDD0xJ5kcffXTc/QE0HK/xvniNb5nHPBZyI/iYYRYtVmRkpPe/i4uLVVFRobFjx+r555/Xpk2bNHDgQO/6mJgYXXnlld7ve/bsqYSEBKWnp2vEiBHe5TX//dNPP9U634033ujz/U033aRnnnlGH374oQYMGKAPP/xQknTzzTf7bHfrrbfqjTfeOGbtbrdbRUVFOumkk5SQkKDvvvtOkyZNOu7YX3/9dZWVlR13G0nq1q3bcdfXHMPpdNZa53K5TniOsrIyRURE1LnuyP1PdJ4jtwGAYCJbmp4tANAUvA4H9nW4rKysXv9Gr++/551OZ1CPeSxN3R9A8+A1ntf4lnbMYyE3go+mBFqsH3/8UXfddZc+++wzFRUV+awrLCz0+b5Dhw615iiIj49Xx44day2Tqj9Wd7STTz7Z5/vu3bvLbrd7nx+4a9cu2e12de/e3We7nj171jpWWVmZ5s6dqwULFmjfvn0yTfOYtdfl1FNPPeE29VETqnU9C6+8vNwndI+1f2VlZZ3rjtz/ROc5chsACCayBQCCi9fhwIqMjKzXv9Eb8u/5YB7zWJq6P4DmwWt8YLW213hyw1poSqBFKigo0Omnn664uDj9+c9/Vvfu3eVyufTdd9/pjjvukGEYPts7HI46j3Os5UeGy7E0ZSLmm266SQsWLNCtt96qUaNGKT4+XjabTVdeeWWt2uuSnZ1dayKousTExCgmJuaY69u1aydJOnDgQK11Bw4cUPv27Y97/Hbt2snj8SgrK0tpaWne5ZWVlcrNzfXun5SUJKfTeczzSDrhuQDA38iW5skWAGgsXocD/zrcrl07ff755zJN02fsR/8b/UT/31Dz7/1gH/N44+T/RYDg4jWe1/iWeMxjITeCj6YEWqQvvvhCubm5eu+993Taaad5l+/YscNv59y6dau6du3q/X7btm0yDENdunSRJHXu3FmGYWj79u0+XfXNmzfXOtaiRYs0ZcoUPfroo95l5eXlKigoqFctw4YN065du0643Zw5c3TPPfccc32/fv0UFhamb775Rpdffrl3eWVlpdauXeuzrC6DBg2SJH3zzTe64IILvMu/+eYbGYbhXW+329W/f3998803tY6xcuVKdevWTbGxsSccDwD4E9nSPNkCAI3F63DgX4cHDRqkF198URs3blSfPn28y1euXOldL0np6elKTU2t89/zq1at8m4X7GMeb5wZGRkyDMNn0tKVK1cqKipKPXr0OO7+AJqO13he41viMY+F3Ag++4k3AQKvpjN+ZCe8srJSzzzzjN/OOX/+fJ/v582bJ0k6//zzff586qmnfLZ74oknah3L4XDU6uLPmzevXl1zqfpZhEuWLDnh1+TJk497nPj4eJ199tlauHChiouLvctfe+01HT58WJdddpl3WWlpqTZt2qScnBzvsrPOOktJSUl69tlnfY777LPPKioqSuPHj/cumzhxolavXu0THps3b9Znn33mcx4ACBaypXmyBQAai9fhwL8OX3zxxQoPD/e5xqZp6rnnnlN6erpGjx7tXX7ppZfqP//5j/bs2eNd9umnn2rLli0+/54P9jEPHDigTZs2ye12e5dNnDhRhw4d0nvvveddlpOTo3feeUcTJkyo87nhAJoXr/G8xrfUY5IbLROflECLNHr0aCUmJmrKlCm6+eabZbPZ9Nprr9Xr43qNtWPHDl100UU677zzlJmZqYULF+rqq6/2TsQ0aNAgXXXVVXrmmWdUWFio0aNH69NPP9W2bdtqHevCCy/Ua6+9pvj4ePXp00eZmZlaunSpkpOT61VLcz6L8P7779fo0aN1+umn67e//a327t2rRx99VOecc47OO+8873arVq3SmWee6dO1j4yM1H333acbb7xRl112mc4991xlZGRo4cKFuv/++5WUlOTdf9q0aXrhhRc0fvx4/eEPf1B4eLgee+wxtWnTRr///e99avr3v/+t77//XlL1BFLr1q3TX/7yF0nSRRddpAEDBjTb+AGgBtnSfNmybt06/etf/5JU/Y60wsJC7+v4wIEDNWHChGY7F4DWg9fhwL8Od+jQQbfeeqseeeQRud1uDRs2TO+//74yMjL0+uuv+zwmZdasWXrnnXd05pln6pZbbtHhw4f1yCOPqH///po6dap3u2Afc+bMmXr11Ve1Y8cO77uhJ06cqJEjR2rq1KnasGGDUlJS9Mwzz8jj8ejee+9ttusO4Nh4jec1vqUek9xooUygBViwYIEpydyxY4d32ddff22OHDnSjIyMNNu3b2/+8Y9/ND/++GNTkvn55597tzv99NPNvn371jpm586dzfHjx9daLsm88cYbvd/PmTPHlGRu2LDBnDhxohkbG2smJiaa06dPN8vKynz2LSsrM2+++WYzOTnZjI6ONidMmGDu2bPHlGTOmTPHu11+fr45depUMyUlxYyJiTHPPfdcc9OmTWbnzp3NKVOmNPo6NVZGRoY5evRo0+VymampqeaNN95oFhUV+Wzz+eef1xpHjb/97W9mz549zYiICLN79+7m448/bhqGUWu7PXv2mBMnTjTj4uLMmJgY88ILLzS3bt1aa7spU6aYkur8WrBgQXMNG4DFkS3+U3Nt6/oKRs4BaJl4HfafhrwOezwe84EHHjA7d+5sRkREmH379jUXLlxY53F/+OEH85xzzjGjoqLMhIQE85prrjEPHjxYa7tgHrPm/yWO/LkyTdPMy8szr732WjM5OdmMiooyTz/9dHP16tXHuIIAmorXeP+x8mu8P45JbrRMNtP0Y8sSAAAAAAAAAADg/zGnBAAAAAAAAAAACAiaEgAAAAAAAAAAICBoSgAAAAAAAAAAgICgKQEAAAAAAAAAAAKCpgQAAAAAAAAAAAgImhIAAAAAAAAAACAgwoJdQCAYhqH9+/crNjZWHo9HFRUVwS4JFhQZGSm7nT4gQp9pmiouLlb79u1D7mf6yDyw2WzBLgcAQhp5AACQyAMAQLWG5IElmhL79+9X586dNW/ePA0bNkwOhyPYJcGCysrKdN1112nTpk3BLgVoFnv27FGHDh2CXUaD7N+/Xx07dgx2GQDQqpAHAACJPAAAVKtPHliiKREbG6t58+ZpzJgxatu2raKiohrUAfd4PJZsZDDu5mMYhvbt26e3335bnTp1anHvHnG73frkk090zjnnKDw8PNjlBAzjbty4i4qK1LFjR8XGxvqhOv+qqXnPnj2Ki4tr0L78vDBuK2DcjLshyAN+XqyAcTNuKyAPyIOGYNyM2woYt//zwBJNiaqqKg0bNkxt27ZVWlpag/Y1TdN7k9pKH+Vj3M0/7rS0NO3Zs0dOp1ORkZHNeuymcrvdioqKUlxcnOVebBl348cdiq8NNTXHxcU16n86+Hlh3K0d42bcjUEeWAPjZtxWwLjJg4bg54VxWwHjZtyNUZ88aFlv1/aTyspKORwORUVFBbsUWJjT6ZTNZlNlZWWwSwEAAAAAAACAoLBEU6JGKHbt0Xrw8wcAAAAAAADA6izVlAAAAAAAAAAAAMFDU8IC0tPTdd9993m/t9lsWrhwYRArAgAAAAAAAABYEU0JC9q9e7cuvfTSYJdRb3PmzFFSUpKSkpJ0zz33+Kz7/PPP1bdvX7nd7uAUBwAAAAAAAACot7BgFwD/KS8vl8vlqrW8Y8eOQaimcVauXKmHHnpI77zzjkzT1OWXX64LLrhAw4cPl9vt1rRp0/T88883aUZ4AAAAAAAAAEBg8EmJFsLj8WjWrFlKT0+Xy+VSz5499corr3jXV1VV6YorrvCu79q1q/7yl7/4HGPixIkaN26c7rzzTqWlpal79+51nuvIxzdt3rxZNptNf//73zVixAjvuT/77DOffT755BMNHTpULpdLbdu21dSpU1VUVORd/9BDD6lz585yOp1KTk7Weeed5133yiuvqEePHnK5XEpISNDo0aN99j2eH3/8UT179tSECRN00UUXqUePHvrxxx8lVX+CYtSoUTrttNPqdSwAAAAAAAAAQHDxSYkW4k9/+pP+8Y9/aN68eerdu7eWLl2q3/3ud2rTpo3OP/98eTwetW/fXm+99ZZSU1P1xRdf6LbbblP79u3161//2nuc5cuXKzY2Vv/9738bdP577rlHc+fOVZ8+ffTHP/5RkydP1rZt2+RwOLRhwwZdcsklmjlzpl555RUdPHhQN998s379619r0aJFysjI0J/+9Cc988wzOuOMM5STk6PPP/9ckrRr1y5dd911mj17tq644goVFhbq888/l2ma9apr8ODB2rlzp7Zu3SrTNLVz504NGjRIGzZs0BtvvKG1a9c2aJwAAAAAAAAAgOChKdEClJWV6cknn9R//vMf/exnP5Mk9e7dW8uWLdOzzz6r888/X06nU48//rh3n169eikzM1P/+Mc/fJoSkZGReuONN+p8bNPx3HzzzbriiiskSffff79OOeUUbdy4UYMHD9a9996rSy65RHfffbckqV+/fnriiSd0/vnnq7S0VDt27FBkZKQuv/xyJSQkqEePHho9erQkae/evfJ4PLryyivVo0cPSdLw4cPrXdfgwYN111136ZxzzpEk3X333Ro8eLBGjx6tv/zlL3r//fd1//33KywsTI8//rjPJzQAAAAAAAAAAC0LTYkWYMOGDSovL9eECRN8lrvdbvXu3dv7/YMPPqjXXntN+/fvV0VFRa31ktSzZ88GNyQkaciQId7/rplz4sCBAxo8eLB+/PFHbdmyRe+//753G9M0ZRiGNm/erIsuukh/+ctf1LVrV51xxhk699xzdc011yg2NlYjRozQqFGjNGTIEI0dO1bjxo3TpEmTlJqaWu/abr/9dt1+++3e759++mnFxMTojDPOUO/evZWZmamdO3dq8uTJ2rVrlyIjIxs8fgAAAAAAAACA/wV8TomvvvpKEyZMUPv27WWz2XxudB/LF198oSFDhsjpdOqkk07ymWuhNaiZX+Hdd9/V6tWrvV9r167VP//5T0nSiy++qHvuuUeTJ0/WBx98oNWrV+uyyy5TZWWlz7GioqIaVcORE0XbbDZJ8j5iqbS0VNdcc41Pbd98841++OEH9e7dWwkJCfrxxx/1yiuvqG3btrr//vvVr18/5eTkKCwsTMuWLdM///lP9erVS88995x69uypTZs2NarOAwcO6MEHH9Rzzz2nr776Sl27dlW/fv104YUXyu12a/369Y06LoDAIw8AABJ5AACoRh4AgHUEvClRUlKigQMHav78+fXafseOHRo/frzOPPNMrV27Vrfeequuu+46ffzxx36uNHAGDx6siIgI7dy5U3379vX5qpmsetmyZRo8eLDuuOMOjR49Wn379tXOnTsDUl///v21efPmWrX17dvX+6mM8PBwXXzxxXr22Wf1448/at++ffrwww8lSXa7XePGjdPjjz+uDRs2KDw8XG+99Vajapk2bZpuuOEGdevWTR6PR26327vO4/HI4/E0fcAAAoI8AABI5AEAoBp5AADWEfDHN51//vk6//zz6739c889p65du+rRRx+V9L+5Fh5//HGde+65/iozoBISEnT99ddr1qxZ8ng8Ouuss5Sfn68vvvhC8fHxmj59uk4++WS9++67eu+993TyySfrpZde0vr165Wenu73+mbNmqXTTz9dU6ZM0fXXX6/Y2FitW7dOH3/8sV599VW99dZb2r59u8466ywlJyfr/fffl2ma6tu3rz7//HN98sknuuCCC9S2bVstW7ZM+fn56tevX4PreP/997V9+3YtWrRIknTqqadqx44dWrRokXbt2iWHw6EBAwY09/AB+Al5AACQyAMAQDXyAACso8XPKZGZmamzzz7bZ9m5556rW2+99Zj7VFRUqKKiwvt9aWmp979rHknUGE3Z90Qef/xxpaam6tFHH9Vtt92m2NhY9e3bV3/6059kmqZmzJihNWvWaMqUKbLZbLr44os1ZcoULV26tFZdx6rzRNsda7/hw4fr448/1p/+9CedffbZMk1THTt21C9+8QuZpqmkpCQ99thjevjhh1VRUaHOnTvrhRde0JAhQ7R27VotW7ZMzz//vEpKStS+fXvdc889uvTSS2Wapj788ENdeOGF2rRpk3ci7LqUlJTotttu0xtvvCG73S7TNNW1a1fNnTtX119/vSIiIvTcc88pKiqq2f6emvvvu+Z4VVVVPp/waAlq6mlpdfkb427cuIN1vZojD2oel+d2uxs8Dn5eGLcVMG7G3Zj9A408CA7GzbitgHGTBw3BzwvjtgLGzbgbs399tPimxMGDB9WmTRufZW3atFFRUZHKysrqnNR47ty5uvfee73fjx49WvPmzWvS430C8VigO++8U3feeWed5z7eI49qaqtZf3Stu3bt8lle8wPi8XjUvXt3n++l6k9uHL3s1FNP1RdffFHnuc866ywtX768znX9+/fX559/fsyat23bpk6dOqljx47HvcYul0tbt26tNb6bbrpJN910U63jNpU//r4Nw5Bpmlq9erXPP4JakiVLlgS7hKBg3A1zZKM3kJojD2p88sknjZ6Dh58Xa2Hc1sK4G4Y84OfFShi3tTDuhiEP+HmxEsZtLYy7YRqSBy2+KdEYM2fO1IwZM7zfZ2dnq7CwUA6HQw6Ho8HH83g8jdov1AVi3B999JHuueeeOv+xECz+GrfdbpfNZtOwYcMUFxfX7MdvCrfbrSVLlmjcuHE+k563doy7ceOueTdRKDg6D4qKitSxY0edc845Df495OeFcVsB42bcDUEe8PNiBYybcVsBeUAeNATjZtxWwLj9nwctvinRtm1bHTp0yGfZoUOHFBcXd8wb2U6nU06n0/t9SUmJCgsLJUk2m61B5z/yET4N3TeUBWrc//3vf/127Mbw57hrjhcWFtZiX9DCw8NbbG3+xLgbvl8wNEce1GjK3zk/L9bCuK2FcTd8v2AgD4KLcVsL47YW8oA8aAjGbS2M21oCkQf2Bh89wEaNGqVPP/3UZ9mSJUs0atSoIFUEAAgG8gAAIJEHAIBq5AEAhK6ANyUOHz6stWvXau3atZKkHTt2aO3atdq9e7ek6o/STZ482bv99ddfr59++kl//OMftWnTJj3zzDP6xz/+odtuuy3QpQMAmhF5AACQyAMAQDXyAACsI+BNiW+++UaDBw/W4MGDJUkzZszQ4MGDNXv2bEnSgQMHvIEjSV27dtUHH3ygJUuWaODAgXr00Uf14osv6txzzw106QCAZkQeAAAk8gAAUI08AADrCPicEmeccYbPc/uP9sorr9S5z5o1a/xYFQAg0MgDAIBEHgAAqpEHAGAdLX5OCQAAAAAAAAAA0DrQlAAAAAAAAAAAAAFBUwIAAAAAAAAAAAQETQkAAAAAAAAAABAQNCUawOMx9OMPWfo6Y7d+/CFLHo/h93N+9NFHOuuss5SWliabzaaFCxfW2ubBBx9Uenq6nE6nBgwYoC+//NLvdTUXw+NR+a59Kt2wVeW79snweAJy3h07duiSSy5RQkKCXC6XevTooYyMDO/6hx56KGSvKQAAAAAAAAC0VGHBLiBUrFqxT39/dZ22bslTRUWVnM4wndwjSZN/NUAjRnbw23kPHz6s/v37a+rUqZo8eXKt9S+99JJmz56tv/71rxozZoweeeQRTZgwQRs3blR6errf6moOpVt+UuFny1V5IEumu0q28DBFtEtT/FmjFdWjm9/Om52drTFjxmjUqFFavHix2rRpo40bNyo5OVmStGDBgpC9pgAAAAAAAADQkvFJiXpYvWq/HvjLMq1fd0gJCU517ZqghASn1q87pAfuW6aVK/b67dwTJ07Uk08+qUmTJtW5/sknn9RVV12lm2++WUOGDNHChQvlcrk0f/58v9XUHEq3/KTcf3ygil375IiOUnibZDmio1Sxa59y//GBSrf85Ldz33PPPWrXrp0WLVqk008/Xb169dLPf/5z9enTR5I0b968kLymAAAAAAAAANDS0ZQ4AcMw9dqr65WfX6aTTkpSbJxTjjC7YuOcOumkJOXnl+m1V9YF5FFORysvL9eGDRs0btw47zKHw6GxY8dq1apVAa+nvgyPR4WfLZenpFTh7VJlj3TKZrfLHulUeLtUeUpKVfh5pt8e5fTRRx9p0KBBOv/885WUlKTevXvrsccekyRVVFSE5DUFAAAAAAAAgFBAU+IENm3M1rateWrTNkY2u81nnc1uU9u2MdqyJU+bNuYEvLaDBw/K4/GoXbt2PsvT0tKUlZUV8Hrqq3LvQVUeyFJYYrxstqOuqc2msMR4Ve4/pMq9B/1y/r179+q1115T9+7d9Z///EfXXXedZs2apaeffjpkrykAAAAAAAAAhALmlDiBgvwKVVR4FBVV96WKjArXoUMlKsgvD3BlocsoKa2eQ8IZXud6W0S4zIIqGSWl/jm/Yahfv356+umnJUmjR4/WDz/8oBdffFEXXnihX84JAAAAAAAAAOCTEieUkOiU0+lQaWlVnevLSt1yOsOUkOgKcGVS27Zt5XA4dODAAZ/lWVlZSktLC3g99WWPjpItPExmhbvO9WalW7bwMNmjo/xy/tTUVPXo0cNnWe/evbV///6QvaYAAAAAAAAAEApoSpxAr96pOunkJB06eFimYfqsMw1TBw8eVo8eSerVOyXgtblcLvXp00dLly71LvN4PFq2bJmGDx8e8HrqK6JDW0W0S1NVfqFM86hrapqqyi9URPs2iujQ1i/nHzp0qLZv3+6zbMuWLUpPT5fT6QzJawoAAAAAAAAAoYCmxAnY7TZNmtJfiYmR2rYtT8VFFaqqMlRcVKFt2/KUmBipSb8aIIfDP5eysLBQmZmZyszMlCT99NNPyszM1NatWyVJt9xyi9588009/fTTWrNmjSZNmqSysjJNmzbNL/U0B7vDofizRssRHSX3gWwZZRUyPYaMsgq5D2TLEROt+DNHye5w+OX8f/jDH7R27VrNnDlTP/74o55//nm9/vrr+t3vfidJuummm0LumgIAAAAAAABAKGBOiXoYNry9Zt01Rn9/dZ22bsnToUMlcjrDNGBAG0361QCNGNnBb+f++uuvNX78eO/3c+bM0Zw5c3TppZdq0aJFuvbaa5WVlaUHHnhAOTk56tWrlxYvXqwOHfxXU3OI6tFNuny8Cj9brsoDWTILqmQLD5OzSwfFnzmqer2fnHbaaVq4cKFmz56txx57TOnp6br//vt1/fXXyzRNTZ06VdnZ2SF3TQEAAAAAAACgpaMpUU/DR6Zr6PD22rQxRwX55UpIdKlX7xS/fUKixgUXXFDrEUdHmzlzpmbOnOnXOvwhqkc3ubp3VuXegzJKSmWPjlJEh7Z++4TEka688kpdeeWVx1w/c+ZMzZo1y+91AAAAAAAAAICV0JRoAIfDrr79mOy4OdkdDrk6pwe7DAAAAAAAAABAADCnBAAAAAAAAAAACAiaEgAAAAAAAAAAICBoSgAAAAAAAAAAgICgKQEAAAAAAAAAAAKCpgQAAAAAAAAAAAgImhIAAAAAAAAAACAgaEoAAAAAAAAAAICAoCkBAAAAAAAAAAACgqYEAAAAAAAAAAAICJoSDeDxGNqxPk/rvzqoHevz5PEYfj/nrFmz1K9fP0VHRyspKUnjxo3TunXrfLZ58MEHlZ6eLqfTqQEDBujLL7/0e13NxfB4VHVgl9w7flTVgV0yPB6/nzM9PV02m63W1+TJk73bPPTQQyF7TQEAAAAAAACgpQoLdgGhYmNmlj5asFV7NhfJXeFRuNOhjj3jdN6ve6jPqDS/nTcjI0O/+93vNHr0aLndbt15550677zztGnTJsXFxemll17S7Nmz9de//lVjxozRI488ogkTJmjjxo1KT0/3W13Nwb1rsyq++VRGzn6ZVW7ZwsJlT2kv59CfKbxzT7+dd/Xq1fIc0fz47rvvdMkll+jKK6+UJC1YsCBkrykAAAAAAAAAtGR8UqIeNq3M1t/vXavta/MVmxihdt1iFZsYoe1r8/X3e9ZoQ2aW386dkZGhm266SaeccopGjhypN954QwcOHNDy5cslSU8++aSuuuoq3XzzzRoyZIgWLlwol8ul+fPn+62m5uDetVlln74tz4GdsrmiZU9sI5srWp4DO1X26dty79rst3O3b99eHTt29H7961//UseOHXXeeedJkubNmxeS1xQAAAAAAAAAWjqaEidgGKY+XrBdxXmV6tAjVlGx4XI4bIqKDVeHHrEqzqvURwu2BuRRTpKUn58vSUpJSVF5ebk2bNigcePGedc7HA6NHTtWq1atCkg9jWF4PKr45lOZpYdlT24nmzNSNrtdNmek7MntZJYeVsU3nwXkUU7l5eV67733dM0118hut6uioiIkrykAAAAAAAAAhAKaEiew68cC7d1SqOT2kbLZbD7rbDabkttHas+mQu3eUOD3Wjwej6ZPn64hQ4Zo6NChOnjwoDwej9q1a+ezXVpamrKy/PfpjaYysvbKyNkve2xindfUHpsoI2efjKy9fq/ljTfeUHFxsX73u99JUsheUwAAAAAAAAAIBcwpcQKHCyrlrjDkjKr7UkVEOeQ+6NHh/Eq/1zJlyhRt2bJFGRkZfj+XP5nlh6vnkAh31r1BuFPm4QKZ5Yf9XsuCBQt02mmnqUuXLn4/FwAAAAAAAABYHZ+UOIGYhAiFO+2qKK2qc31lafWk1zGJEX6tY8qUKVq6dKk+/fRTdevWTZLUtm1bORwOHThwwGfbrKwspaX5b/LtprK5YmQLC5fcFXVv4K6QLSxcNleMX+vYsmWLli9fruuuu867LFSvKQAAAAAAAACEApoSJ9C5b4I69IhX7v4ymabps840TeXuL1PHXvHq1CfBL+c3DENTpkzRf//7Xy1dulS9evXyrnO5XOrTp4+WLl3qXebxeLRs2TINHz7cL/U0B3taB9lT2ssozq/zmhrF+bKnpMue1sGvdTz//PNKSkrSZZdd5l3mdDpD8poCAAAAAAAAQCigKXECdrtN507trtikCO3dUqzSYreqPIZKi93au6VYcckROm/qyXI4/HMpp0yZovfee0+vvvqq4uPjtWfPHu3Zs0clJSWSpFtuuUVvvvmmnn76aa1Zs0aTJk1SWVmZpk2b5pd6moPd4ZBz6M9ki4qRkXtAZkWZTMOQWVEmI/eAbFExcg49S3aHw281eDwevfnmm7r88ssVHh7us+6mm24KuWsKAAAAAAAAAKGAOSXqodeIVE2eM0gfLdiqPZuL5D5Y/cim7oOTdN7Uk9VnlP8e67Nw4UJJ0gUXXOCz/KmnntJNN92ka6+9VllZWXrggQeUk5OjXr16afHixerQwb+fMmiq8M49pZ9doYpvPpWRs1/m4QLZwsLlaNdVzqFnVa/3o3/96186cOCArr/++lrrpk6dquzs7JC7pgAAAAAAAADQ0tGUqKfeo9LUc0Sqdm8o0OH8SsUkRqhTnwS/fUKixtGPN6rLzJkzNXPmTL/W4Q/hnXvK0eEkGVl7ZZYfls0VU/1oJz9+QqLGz3/+8+Ne25kzZ2rWrFl+rwMAAAAAAAAArISmRAM4HHZ17Z8U7DJaFbvDIXu7zsEuAwAAAAAAAAAQAMwpAQAAAAAAAAAAAoKmBAAAAAAAAAAACAiaEgAAAAAAAAAAICBoSgAAAAAAAAAAgICgKQEAAAAAAAAAAAKCpgQAAAAAAAAAAAgImhIAAAAAAAAAACAgaEoAAAAAAAAAAICAoCkBAAAAAAAAAAACgqZEA3g8hvavL9D2r7K0f32BPB7D7+d8+OGH1aNHD8XExCgmJkaDBg3SokWLfLZ58MEHlZ6eLqfTqQEDBujLL7/0e13NxTA8MvK2yTi4pvpPw+P3c1ZVVenWW29Venq6XC6XOnbsqNtvv12G8b+/z4ceeihkrykAAAAAAAAAtFRhwS4gVOzKzNXKl39S1uZiVVV4FOZ0KK1nrEZc201dRqX47bwdO3bU/fffr969e8s0Tb3wwgu66qqr1LVrV51yyil66aWXNHv2bP31r3/VmDFj9Mgjj2jChAnauHGj0tPT/VZXczCy1svY8h+paLfkqZQcEVJcJ6nHhbKn9ffbee+++269+uqreu655zRo0CBlZmbqxhtvVEJCgmbNmqUFCxaE7DUFAAAAAAAAgJaMT0rUw64Vufronh+0b22+IhMjlNwtRpGJEdq3Nl8fzflBOzNz/Hbuq666Spdddpn69eun/v3766mnnlJUVJQyMjIkSU8++aSuuuoq3XzzzRoyZIgWLlwol8ul+fPn+62m5mBkrZex5iUpf4sUESPFplf/mb9FxpqXZGSt99u5V6xYoXPOOUdXXHGFevbsqV/96lcaM2aMVq9eLUmaN29eSF5TAAAAAAAAAGjpaEqcgGGYWr1gp0rzKpTaI1au2DDZHTa5YsOU2iNWpXkVWvnyTwF5lFNVVZVefPFFlZWV6bTTTlN5ebk2bNigcePGebdxOBwaO3asVq1a5fd6GsswPNWfkKgslGI7yRYeLZvNLlt4tBTbSaoslLH1P357lNPIkSO1bNkyrV9f3fhYsWKFvvnmG5133nmqqKgIyWsKAAAAAAAAAKGAxzedwKEfC5W95bDi2kfJZrP5rLPZbIprH6WsTcU6tKFI7fsn+KWGVatW6YwzzlBlZaUiIyP1+uuva8iQIdq5c6c8Ho/atWvns31aWpq2bt3ql1qaRcGO6kc2RabUeU3NyBSpcHf1dkknNfvp//KXv6ioqEgDBw6U3W6XYRi68847df3114fuNQUAAAAAAACAEEBT4gTK8itVVeFRRJSjzvURUQ4VH/SoLL/SbzUMGDBAq1evVn5+vt566y399re/Vffu3ZWUlOS3c/pVZXH1HBJhkXWvd7gkT171dn6wYMECvfvuu/rb3/6mAQMG6Ntvv9XMmTPVvn17XXjhhX45JwAAAAAAAACApsQJRSZGKMzpUGWpR67Y2persrR60uvIxAi/1eByudS3b19J0pgxY/Tdd9/pr3/9q15++WU5HA4dOHDAZ/usrCylpaX5rZ4mi4itntS6qkwKj6693lNevT4i1i+nv+uuu3TrrbfquuuukyQNHz5cO3bs0KOPPqrrrrsuNK8pAAAAAAAAAIQA5pQ4gTZ945XaI0ZF+0tlmqbPOtM0VbS/VGm9YtWmT1zAajIMQ5WVlXK5XOrTp4+WLl3qXefxeLRs2TINHz48YPU0WEJXKa6TVJZT5zVVWY4U36l6Oz8oLy+X3e77ox8WFibTNOV0OkPzmgIAAAAAAABACKApcQJ2u03DpnZRVJJT2VuKVV5cJcNjqry4StlbihWd5NSIX3eTw+GfSzl9+nR99NFH2rx5s1atWqXp06dr1apV+uUvfylJuuWWW/Tmm2/q6aef1po1azRp0iSVlZVp2rRpfqmnOdjtDtl7XChFxEvFu2W6S2QaHpnuEql4txQRL/vJF8pur/uRWU119tln69FHH9Xbb7+tzZs367XXXtOzzz6rCy64QJJ00003hdw1BQAAAAAAAIBQwOOb6qHzyGSdd08/rXz5J2VtLlbxwepHNqUPTtSIX3dTl1Epfjt3dna2rr32WmVnZysmJka9evXSe++9p0suuUSSdO211yorK0sPPPCAcnJy1KtXLy1evFgdOnTwW03NwZ7WXxp8rYwt/6me9NqTV/3IpqQe1Q2JtP5+O/eLL76o3//+97rtttuUl5en1NRUTZ48WQ899JAkaerUqcrOzg65awoAAAAAAAAALV1QPikxf/58denSRS6XSyNGjNCqVauOu/0TTzyhnj17KjIyUh07dtRtt92m8vLyAFVbrfOoZE18fqgufeYUXfTXQbr0mVM08bmhfm1ISNLbb7+tffv2qbKyUnl5eVq+fLm3IVFj5syZ2r9/vyorK7Vu3TqdeeaZfq2pudjT+ss++o+yj/y97MOmV/856o9+bUhIUkJCgl566SXt379f5eXl2rNnj5588km5XC7vNqF6TYFQE4p5AABofuQBAEAiDwDAKgL+SYm3335bM2bM0HPPPacRI0boiSee0LnnnqvNmzfXOZHwG2+8oTvvvFMvv/yyRo8erS1btuhXv/qVbDabHnvssYDW7nDY1b5/QkDP2drZ7Q4p6aRglwEgCEI5DwAAzYc8AABI5AEAWEnAPynx2GOP6Te/+Y2mTp2qPn366LnnnlNUVJRefvnlOrdfvny5Tj31VF199dXq0qWLzjnnHF111VUn7JYDAFo28gAAIJEHAIBq5AEAWEdAPylRWVmpb7/9VjNnzvQus9vtOvvss5WZmVnnPqNHj9bChQu1atUqDR8+XD/99JM+/PBDTZo06ZjnqaioUEVFhff70tJS73+bptno+puybyhj3M17vKqqKrnd7mY9dlPV1NPS6vI3xt24cTfH9QpWHhQVFXnH0NBx8PPCuK2AcTPuxuzfFORB6GDcjNsKGDd50BD8vDBuK2DcjLsx+9dHQJsSOTk58ng8atOmjc/yNm3aaNOmTXXuc/XVVysnJ0djxoyRaZqqqqrS9ddfr1mzZh3zPHPnztW9997r/X706NGaN2+ePB6PPB5Po2pv7H6hjnE3H8MwZJqmVq9e7fOPoJZkyZIlwS4hKBh3wxzZ6G2sYOVBjU8++URRUVGNqp2fF2th3NbCuBuGPODnxUoYt7Uw7oYhD/h5sRLGbS2Mu2EakgcBn1Oiob744gs98MADeuaZZzRixAht27ZNt9xyi+677z7dfffdde4zc+ZMzZgxw/t9dna2CgsL5XA45HA4GlyDx+Np1H6hjnE3L7vdLpvNpmHDhikuLq7Zj98UbrdbS5Ys0bhx4xQeHh7scgKGcTdu3DXvJgq05siDoqIidezYUeecc06Dfw/5eWHcVsC4GXdDkAf8vFgB42bcVkAekAcNwbgZtxUwbv/nQUCbEikpKXI4HDp06JDP8kOHDqlt27Z17nP33Xdr0qRJuu666yRJ/fv3V0lJiX7729/qT3/6k+z22tNiOJ1OOZ1O7/clJSUqLCyUJNlstgbVfOQjfBq6byhj3M0/7prjhYWFtdgXtPDw8BZbmz8x7obv11TByoMjx9DYcfDzYi2M21oYd8P3ayryIPQwbmth3NZCHpAHDcG4rYVxW0sg8iCgE11HRETolFNO0aeffupdZhiGPv30U40aNarOfUpLS2sFSc272K061wEAhDryAAAgkQcAgGrkAQBYS8Af3zRjxgxNmTJFQ4cO1fDhw/XEE0+opKREU6dOlSRNnjxZ6enpmjt3riRpwoQJeuyxxzR48GDvx/HuvvtuTZgwwZKPFgKA1oI8AABI5AEAoBp5AADWEfCmxBVXXKHs7GzNnj1bBw8e1KBBg/TRRx95JzPavXu3T6f7rrvuks1m01133aV9+/YpNTVVEyZM0P333x/o0gEAzYg8AABI5AEAoBp5AADWEZSJrqdPn67p06fXue6LL77w+T4sLExz5szRnDlzAlAZACCQyAMAgEQeAACqkQcAYA0BnVMi1BkeQ7kb8nQg84ByN+TJ8BgBPf+sWbNks9l07bXX+ix/8MEHlZ6eLqfTqQEDBujLL78MaF1NYRgeGaWbZRStrv7T8Pj9nAUFBbr22mvVvn17uVwuDR48WF999ZXPNg899FDIXlMAAAAAAAAAaKmC8kmJUHTomyxtfWOrCrYXyqjwyO50KKF7vE6+uofaDkvz+/m/+uorvfrqq+rRo4fP8pdeekmzZ8/WX//6V40ZM0aPPPKIJkyYoI0bNyo9Pd3vdTWFUbxGyn5HqtghGZWSPUJydpWRepnssYP9dt5rrrlGmzdv1ssvv6yOHTvq5Zdf1vjx47Vu3Tp16dJFCxYsCNlrCgAAAAAAAAAtGZ+UqIfsb7O15pE1yv0hT874CMV2jpUzPkK5P+RpzSPf6eDqLL+ev7CwUJMnT9Yzzzyj+Ph4n3VPPvmkrrrqKt18880aMmSIFi5cKJfLpfnz5/u1pqYyitdI+56SyjZKjnjJ2bH6z7KN0r6nqtf7QUlJiT7++GPdf//9Ou+889S3b189+uij6tSpk5544glJ0rx580LymgIAAAAAAABAS0dT4gRMw9TWt7apIr9Scd1jFR4TIZvDrvCYCMV1j1VFfqW2vrnFr49y+vWvf62zzz5bF198sc/y8vJybdiwQePGjfMuczgcGjt2rFatWuW3eprKMDzVn5DwFEjOrpIjWrI5qv90dq1enrPIL49ycrvd8ng8ioyM9FnucrmUmZmpioqKkLymAAAAAAAAABAKaEqcQP7mAhVuK1JU20jZbL6Xy2azK6ptpAq2FSp/c4Ffzv/iiy9q/fr1euqpp2qtO3jwoDwej9q1a+ezPC0tTVlZ/v30RpOUb6t+ZFN4G8lm811ns1UvL/+pertmlpCQoEGDBukvf/mLdu7cqaqqKj377LNau3atsrOzQ/eaAvVgmqZ+ysjWezd9K0l6/5bvtGN5jkzTDHJlAIBAIg8AABJ5AACoFow8oClxApWFFTIqPQqLrHv6DUekQ0aFR5WFFc1+7u3bt+uOO+7Qa6+9pqioqGY/ftBUFf3/HBKuutfbXdXrq4r8cvrXX39dpmmqa9eucrlcevbZZzVhwgTZjm6QAK1IVYVHr1z6teaf/pnWvLlbkvTt67v09JhP9dqVmfK4/fdpLwBAy0EeAAAk8gAAUC1YeUBT4gQi4p2yRzhUVVZV53pPWfWk1xHxzmY/d2ZmpvLy8nTqqacqLCxMYWFhWr16tRYsWKCwsDC1adNGDodDBw4c8NkvKytLaWn+n3y70cLiqie1NsrrXm+UV68Pi/PL6fv06aPVq1ersLBQ27dv17p16+R2u9WpUye1bds2NK8pcAL/+sNa/fivfZIko8r0+XPdoj36YOa6oNUGAAgc8gAAIJEHAIBqwcoDmhInkNgzQfEnxan0YJlM07czZJqGSg+WKeGkeCX2TGj2c1944YVavXq1VqxY4f3q27evLr74Yq1YsUKRkZHq06ePli5d6t3H4/Fo2bJlGj58eLPX02xcJ1XPHeE+JB39MSDTrF7u6la9nR/FxcWpc+fOys7O1ldffaUJEybI6XSG5jUFjqMkt0Ir/rZd5jGa26Ypff3MVpUVVga2MABAQJEHAACJPAAAVAtmHtCUOAGb3aaTrzxJzsQIFW0vlvtwpQyPR+7DlSraXixnYoROvqqH7I7mv5QJCQkaOnSoz1dUVJSSkpI0dOhQSdItt9yiN998U08//bTWrFmjSZMmqaysTNOmTWv2epqL3e6QUi+THAnVc0t4SiTTU/1nxQ4pLEFKmVi9nR+89957evfdd7Vp0ya9//77Gjt2rLp166bp06dLkm666aaQu6bA8Wz7PEse9/8agIeMnfr444+VbezxLqsqN7T9y+xglAcACBDyAAAgkQcAgGrBzIO6J0qAj9RTUjX49sHa+sZWFWwvlJFV/cim5P5JOvmqHmo7LHiP9bn22muVlZWlBx54QDk5OerVq5cWL16sDh06BK2m+rDHDpaRfrOU/U51I8KdXf3Ipqg+1Q2J2MF+O3dBQYHmzJmjQ4cOKT4+XhdccIEee+wxOZ1OmaapqVOnKjs7O+SuKXAsnkrflvdBY5u6K1z7jc1KUsdjbgcAaF3IAwCARB4AAKoFMw9oStRTm6FpShuSqvzNBaosrFBEvFOJPRP88gmJ41m1alWtZTNnztTMmTMDWkdzsMcOlhE9QCrfVj2pdVic5DrJb5+QqPHrX/9av/71r4+7zcyZMzVr1iy/1gEESvqQxPptN7h+2wEAQhN5AACQyAMAQLVg5gGPb2oAu8Ou5D5JajeqnZL7JAW8IdEa2e0O2aN6yh43rPpPPzckACtq0ytO3c9IlT3MVud6u8Omnue0VUr3mABXBgAIJPIAACCRBwCAasHMA+6qA4AFXPnyCMWkOmVz+AaNzWFTXPtIXf7isCBVBgAIJPIAACCRBwCAasHKA5oSAGABSV2iNeO7c3XmH3opIiZckhQRG66f3dFbt317jhI6RAW5QgBAIJAHAACJPAAAVAtWHjCnBABYRGwbl8bPHaDdnfpr/fr1uuC+/jr/hv7BLgsAEGDkAQBAIg8AANWCkQd8UgIAAAAAAAAAAAQETQkAAAAAAAAAABAQNCUAAAAAAAAAAEBA0JQAAIsoL6/Sa69+r2fmr5YkPTt/tV5fuF4VFVVBrgwAEEjkAQBAIg8AANWCkQc0JQDAAvLzy3X2Ga/pxuv/q107CyVJO3cW6obffKDzzn5dRUUVQa4QABAI5AEAQCIPAADVgpUHNCUAwAJumf5f/fhDliTJNOXz59o1B/X7Wz8JUmUAgEAiDwAAEnkAAKgWrDygKdEAhsdQ8bYs5a/ZreJtWTI8ht/P+fvf/142m83nq2vXrj7bPPjgg0pPT5fT6dSAAQP05Zdf+r2u5mIYHlUZm1TlWakqY5MMw+P3c3700Uc666yzlJaWJpvNpoULF9ba5qGHHjrhNQ3l6w5r2bunSIv/uVkej1nneo/H1KJ/bNChg4cDXBkAIJDIAwCARB4AAKoFMw9oStRT4fq92vzgf7Xl4Y+17clPteXhj7X5wf+qYP1ev5/7pJNO0u7du71fy5cv96576aWXNHv2bN1xxx3KzMxU3759NWHCBO3bt8/vdTVVlfGdyj2zVF51t8o8D6i86m6Ve2apyvjOr+c9fPiw+vfvr0cffbTO9QsWLDjhNQ3l6w7rWblin7fLfSwej6lVK/n5BYDWjDwAAEjkAQCgWjDzgKZEPRT9sF87Xlim4q2HFBbrlKtDgsJinSreekg7/pbh98aEw+FQx44dvV/t2rXzrnvyySd11VVX6eabb9aQIUO0cOFCuVwuzZ8/3681NVV1Q+JxecyNkuJlU0dJ8fKYG1XuedyvjYmJEyfqySef1KRJk+pcP2/evBNe01C97rAm80QJAwCwBPIAACCRBwCAasHMA5oSJ2Aapg7+Z53cxWWK7JgkR7RTNrtdjminIjsmyV1cpgP//t6vj3LatWuX0tLS1KFDB1188cXaunWrJKm8vFwbNmzQuHHjvNs6HA6NHTtWq1at8ls9TWUYHlV63pZpFsqmLrLbY2S3h8luj5FNXWSahar0/CMgj3I6WkVFxQmvaahed1jXyFEdZLMdfxu73abhI9IDUxAAICjIAwCARB4AAKoFMw9oSpxAyY5sle7OlzM5Rraj/pZsNpucyTEq3Zmnkh05fjn/qFGj9Mwzz+jf//63nnrqKe3evVunn366CgoKdPDgQXk8Hp9PTkhSWlqasrKy/FJPczC0VYb5k2xKk93u+yNot9tlU5oMc7sMbQ14bfW5pqF63WFd6R1i1d4VIx2rAW5KHaNjldYmOqB1AQACizwAAEjkAQCgWjDzgKbECVQVV8isrJLdFV7nersrXEZllaqKy/1y/okTJ2rq1KkaMWKEfvGLX2jJkiUqLi7WK6+84pfzBYRZKFOVklzH2MBVvd4sDGRVQKu1YXmW2hdFKcoMqw6amrD5/z+jzTCl5bm09dvcYJUIAAgA8gAAIJEHAIBqwcwDmhInEBbrlC0iTEa5u871Rrlb9ogwhcUe6wZ780pJSVHnzp21bds2tW3bVg6HQwcOHPDZJisrS2lpaQGpp1Fs8bIpQtKxGjnl1ett8YGsSpLqdU1D9rrDsg7uPKww2dXfk6zunji5/v+l3yW7ulfFqZ8nWWGy68CO4iBXCgDwJ/IAACCRBwCAasHMA5oSJxDdNVVRnRJVkXu41uQfpmmqIveworokKbprSkDqKSws1J49e9SuXTu5XC716dNHS5cu9a73eDxatmyZhg8fHpB6GsOuk2W3dZOpLBmG71wchmHIVJbstu6y6+SA1+Z0Ok94TUP1usO6YhOdkiS7bEozo9TJrG6edTHbK82Mkl02n+0AAK0TeQAAkMgDAEC1YOYBTYkTsNltanvhAIXHRqpsT548JRUyPYY8JRUq25On8LhItZswUHaHfy7l7373O/33v//V5s2btXTpUl1wwQWy2+2aOnWqJOmWW27Rm2++qaefflpr1qzRpEmTVFZWpmnTpvmlnuZgtzsU4bhCNlu8TO2UYRyWYVTJMA7L1E7ZbAmKcFwuu93hl/MXFhYqMzNTmZmZkqSffvpJmZmZ3gnEb7rpphNe01C87rCuQT9rp5iECO/3LluiJMl5xKeREtJcGnB624DXBgAIHPIAACCRBwCAasHMg7BmP2IrFNevvbr+ZowO/HudSnflycgtkT0iTLE92qjdhIFK6N/Bb+fet2+fpkyZooKCAiUmJmrYsGH6+uuv1b59e0nStddeq6ysLD3wwAPKyclRr169tHjxYnXo4L+amkOYfYhcuk2VnrdlmD/JVLZsipDD1kcRjssVZh/it3N//fXXGj9+vPf7OXPmaM6cObr00kv1zjvvaOrUqcrOzj7uNQ3V6w5rinA6NOXPgzX/5pXH3GbKn4coLJw+NQC0ZuQBAEAiDwAA1YKZBzQl6im+fwfF9U1XyY4cVRWXKyzWpeiuKX77hESN//znPyfcZubMmZo5c6Zf6/CHMPsQ2TVQhrZWT2pti69+tJOfPiFR44ILLqj1KK4aNctnzpypWbNmHfc4oXrdYU0TbuilynKPXp295n8TF0lyRjl07QNDdcF1PYJXHAAgYMgDAIBEHgAAqgUrD2h7N4DdYVfsSWlKHNxJsSel+b0hYQV2u0Nh9l4Kc4xQmL2X3xsSgFXZbDZNnNFPb+2/QhNn9JUkXX57f72170pdPL13kKsDAAQKeQAAkMgDAEC1YOUBd9UBwEKi4yI09Jzqx4wNPTddUbHhQa4IABAM5AEAQCIPAADVAp0HNCUAAAAAAAAAAEBA0JQAAAAAAAAAAAABYammxLEmNwYCgZ8/tARGlaHs73MkSTnrcmV4jCBXBAAIBvIAACCRBwCAaoHOA0s0JSIiIuTxeFRaWhrsUmBhFRUVMk1TERERwS4FFrX9nz/prcFv65v7v5Ekrbp3ld4+5R3t+PfO4BYGAAgo8gAAIJEHAIBqwciDML8duQUJDw/X6tWr5XQ6JUnR0dGy2Wz12tc0TRmGIbvdXu99WgPG3bzj9ng8ysrKUlhYmPfnEAik7e//pC9u+LL6myN+tEsPluqz33yus148U10v7BKU2gAAgUMeAAAk8gAAUC1YeWCJpoQk3XTTTVq/fr0OHDjQ4JvNpmla6sZ8Dcbd/Mc96aSTZLdb4gNKaEGMKkMrZ6867jYr56xSlws6y2a33u88AFgFeQAAkMgDAEC1YOaBZZoShmGoQ4cOcrlcKisrq/d+VVVVWr16tYYNG6awMMtcLsbth3FHR0db6lqi5Tiw/KDKso7/uleyr0QHVx5Su1FtA1QVACDQyAMAgEQeAACqBTMPLHeHNCIiokHP9He73aqoqFBcXJzCw8P9WFnLwritNW60bicKmIZuBwAITeQBAEAiDwAA1YKZBzxHBgBauai2kfXaLrpdlJ8rAQAEE3kAAJDIAwBAtWDmAU0JAGjl2o5qq6i2xwkQmxTTKUZpQ9MCVxQAIODIAwCARB4AAKoFMw9oSgBAK2d32DXq/hGSTdVfR/r/70f9ZSST2AFAK0ceAAAk8gAAUC2YeUBTAgAsoMv4LvrZy2cpJj3aZ3lspxiNe/VsdTqnY5AqAwAEEnkAAJDIAwBAtWDlgeUmugYAq+pyfmd1PreTXC+69PHajzTy/hEa/6vxvAMKACyGPAAASOQBAKBaMPKAT0oAgIXY7DYl9U6UJCX1SuJ/OADAosgDAIBEHgAAqgU6D2hKAAAAAAAAAACAgKApAQAAAAAAAAAAAoKmBAAAAAAAAAAACAiaEgAAAAAAAAAAICBoSgAAAAAAAAAAgICgKQEAAAAAAAAAAAKCpgQAAAAAAAAAAAgImhIAAAAAAAAAACAgaEoAAAAAAAAAAICACEpTYv78+erSpYtcLpdGjBihVatWHXf7goIC3XjjjWrXrp2cTqd69OihDz/8MEDVAgD8hTwAAEjkAQCgGnkAANYQFugTvv3225oxY4aee+45jRgxQk888YTOPfdcbd68WWlpabW2r6ys1Lhx45SWlqZFixYpPT1du3btUkJCQqBLBwA0I/IAACCRBwCAauQBAFhHwJsSjz32mH7zm99o6tSpkqTnnntOH3zwgV5++WXdeeedtbZ/+eWXlZeXp+XLlys8PFyS1KVLl0CWDADwA/IAACCRBwCAauQBAFhHQJsSlZWV+vbbbzVz5kzvMrvdrrPPPluZmZl17vOvf/1Lo0aN0o033qjFixcrNTVVV199te644w45HI4696moqFBFRYX3+6KiIkmS2+2W2+1uUM012zd0v1DHuBm3FVh13FVVVd4/GzP25rhe5EHoYNyM2wqsOm7ygDxoCMbNuK3AquMmD8iDhmDcjNsKrDruQOZBQJsSOTk58ng8atOmjc/yNm3aaNOmTXXu89NPP+mzzz7TNddcow8//FDbtm3TtGnT5Ha7NWfOnDr3mTt3ru69995ayz/55BNFRUU1qvYlS5Y0ar9Qx7ithXFbQ83r7cqVK5Wfn9/g/UtLS5tcA3kQehi3tTBuayAPyIPGYNzWwritgTwgDxqDcVsL47aGQOZBwB/f1FCGYSgtLU1/+9vf5HA4dMopp2jfvn165JFHjhkyM2fO1IwZM7zfFxUVqWPHjjrnnHMUFxfXoPO73W4tWbJE48aN834c0AoYN+O2AquOOy4uTrt27dKIESM0atSoBu9f826iQCMPgoNxM24rsOq4yQPyoCEYN+O2AquOmzwgDxqCcTNuK7DquAOZBwFtSqSkpMjhcOjQoUM+yw8dOqS2bdvWuU+7du0UHh7u89G73r176+DBg6qsrFREREStfZxOp5xOZ63l4eHhjf5Basq+oYxxWwvjtoawsDDvn40Zd3NcK/Ig9DBua2Hc1kAekAeNwbithXFbA3lAHjQG47YWxm0NgcwDe4OP3gQRERE65ZRT9Omnn3qXGYahTz/99Jjdl1NPPVXbtm2TYRjeZVu2bFG7du3qDBgAwLEVfP2dds3/pyRp19PvqXDF90GpgzwAgOAiDwAAklS85ZD2vvudJGnvu9/p8LasoNRBHgBAcAU6DwLalJCkGTNm6IUXXtCrr76qjRs36oYbblBJSYmmTp0qSZo8ebLPxEY33HCD8vLydMstt2jLli364IMP9MADD+jGG28MdOkAELI8FW59/+tH9N3vP1b+5jJJUt7GMn1764da/9u/ygjC5E3kAQAEHnkAAJAko8rQhnv/rdWTX1bOV1slSTlfbNaqX76kjQ98KNNjnOAIzY88AIDAC1YeBHxOiSuuuELZ2dmaPXu2Dh48qEGDBumjjz7yTma0e/du2e3/65V07NhRH3/8sW677TYNGDBA6enpuuWWW3THHXcEunQACFmb73xWuRvckmySafv/pdV/Zq+r1NY/vaCeD08LaE3kAQAEHnkAAJCk7c98roMf/VD9zf9/0sA0TEnSgX99r4ikaHW//vSA1kQeAEDgBSsPgjLR9fTp0zV9+vQ6133xxRe1lo0aNUorVqzwc1UA0DpVZOfp0MpiHfvDcTYd+Dpf3fILFZ4YH8jSyAMACCDyAABaH0/uARnFBbLHJsqRXPfcC0dzF5dr36JvJfPY2+x5a7U6Tx6lsKjAPgaJPACAxgm1PAhKUwIAEDi5H34t06i+AWWapnaVZklKU6G7RPr/Od4Mj125Hy9X2yvPD16hAAC/Ig8AoPVw79ig0v/+XZ6927zLwjr2UOT4KQrv1PO4++Z/s0tGpee42xjlbhV8t0spY05ulnoBAP4RqnkQ8DklAACB5SmvkCQVukv1df5Geczqj+NlVxT6bGeUVwa8NgBA4JAHANA6uLevV/FL98izb7vP8qq921T8wmy5d2w47v5GRf3mDzrRjSoAQHCFch7QlACAVi6qXzdtPLxHy/J/VEHV4WNuFzuQd0EBQGtGHgBA6DNNUyX/fE4yzeovn5WGZBgqWfw3mUevO0LMyW3qda7o7qlNKRUA4Eehngc8vgkAWrHdu3dr2a5N2lW1R6YcaudM1v6q/KO2MuWKqVTs4D5BqREA4H/kAQC0DlW7NsnIOyRJqvQYyi2tUGGFW+F2m5wOh5xhdjnLdihs+wZFdevtMzF0jZjuqXLG21RRaEiy1XEWU5FJDkV3TvbvYAAAjRbqeUBTAgBaodLSUi1fvlw//fST3HmFSo5zq7Onp9LCk7Q/95sjtjRltxuKj89X5YEsRbRLC1rNAIDmRx4AQOtQVVWl3Nxc7Vu9Unt2ZSm3tFKFx3ncXsTf/67wDt0VEREhp9Mpl8slp9Mpp9MpW0Gxst3f63BZssIUocOeckmu/9+zOg/iIg+RBwDQArWWPKApAQCtiGma2rhxo1atWqXKykrZbDb1SkxW2y7dJMOtw4UlsuVVf3TPZjMVHVui2PhiOcIMVezax/90AEArQR4AQOgyDEO5ubnKzs72fuXn58s0TVUd3K2KvP89gi86IkyJrggZpqlyj6HKKkMVHo/sEU5JUmVlpSorK1VcXOzdp2zbThUWH5ThyFJFmVMVHqekdrLJIA8AoAVpzXlAUwIAWonc3FxlZGQoKytLkpSWlqaxY8cqYttu7f/8O8nuUUJyoU6LiNYnks7sEaOEmP9NbmoLJxIAoDUgDwAgdBiGoYKCAp8bTrm5uTIMo9a2UVFRShk6Sq4Da5XsMJQc5ZQrzFFrO1t0vOL+eJfcHkPl5eWqqKjw+cqNiNb+bzer0vCoIqZKZXLrYHy8BnTLVUI4eQAAwWC1PCBhACDEVVVV6dtvv9W6detkmqbCw8M1fPhw9enTRzabTR5npGxhYTKrqiRJPRKTZTv3LHV/5zOpyiNJskVEKKoPE5sCQCgjDwCgZTNNU4WFhbVuOFX9/+vykVwul1JSUpSWlqaUlBSlpqYqOjpaklSeFKbSf71wzPNEnXu1HOERcoRXH+doni7dtH3pam8eeMIc2jpypJL2kAcAEAjkQROaEitXrtSIESOasxYAQAPt2bNHy5Yt8378rlu3bho1apQ3oCTJEROlhHPGKv+/X0imWedxEs8/XfbI2gEFAAgN5AEAtDzFxcU+N5xycnJUWVn7ud8RERHeG001X7Gxscc8rmvkuZLHrdJP3pTcFZLNLpmGFOFU1HmT5Bz6s+PWRR4AQGCRB7U1uikxatQonXTSSZo0aZKuueYadevWrTnrAgAcx5ETl0pSTEyMxowZo06dOtW5ferVl6iqsEjFX38r2e3VCx12qcqjuNNHKOXyCwNVOgCgGZEHANAylJSU1LrhVF5eXmu7sLAwpaSk+LzrNT4+XjabrUHnc516oZxDf6bKDatkFOfLHpukiL7DZYuo340j8gAA/IM8qJ9GNyUWLlyo119/Xffdd5/uuecejRw5UpMmTdLll1+upKSk5qwRAPD/6pq4tH///jrllFMUHh5+zP1sYQ61v2mqysf/TLlfrZIkJZx1qpJOGyFX146BKh8A0EzIAwAInvLycp8bTtnZ2SotLa21nd1uV3Jyss87XhMSEmSvuenTRDZnpJyDT2/cvuQBADQZedB4jW5KXH311br66quVk5Ojt956S2+88YamTZumW2+9Veedd55++ctf6qKLLlJERERz1gsAlpWXl6evvvrKO3Fpamqqxo4dq5SUlHofw9Wtk9I6tpM+/FCpV1983BtXAICWiTwAgMCprKysdcPp8OHDtbaz2WxKTEz0vts1LS1NiYmJcjhqTzzakpAHAFA/5EHzavJE1ykpKZo+fbqmT5+u7du364033tDrr7+uK664QvHx8Zo4caImT56sMWPGNEe9AGA5J5q4FABgDeQBAPhXVVWV8vPztX79ehUUFCg7O1uFhYV1bpuQkODzjtfk5GSFhTX5FgsAoAUgD/yvWa9QZGSkoqKi5HK5ZJqmbDabFi9erJdeeklDhgzRq6++qj59+jTnKQGgVTt64tKuXbtq9OjRPhOXAgBaP/IAAJqXx+NRbm6uzztec3NztW7dOpWVlfm8ozUuLs7nmd8pKSk8FQIAWgnyIDia3JQoLi7WokWL9Prrr+vLL7+U3W7X+eefr9mzZ2vChAmy2+365z//qd///veaOnWqVq5c2Rx1A0CrVlpaqszMTG3fvl1S9cSlp556qjp37hzkygAAgUQeAEDTGYah/Px8nxtOeXl5MgzDZzvTNOVyudS5c2e1a9dOqampSklJkctVv8lCAQAtG3nQcjS6KbF48WK9/vrr+s9//qPy8nINGzZMTzzxhK688kolJyf7bDtx4kTl5+frxhtvbHLBANCaNXbiUgBA60IeAEDjmKbpfdTGke949Xg8tbZ1uVw+z/yOj4/XF198oXHjxvFaCwAhjjxo2RrdlPj5z3+ujh076rbbbtPkyZPVs2fP424/cOBAXXPNNY09HQC0es0xcSkAIPSRBwBQf0VFRcrKylJOTo6ys7OVk5Mjt9tda7uIiAifZ36npqYqJibGZ5u69gMAhAbyILQ0uinx2Wef6Ywzzqj39sOHD9fw4cMbezoAaLWqqqr03Xffad26dTIMQ+Hh4Ro2bJj69u3LxKUAYCHkAQAc3+HDh33e8Zqdna3Kyspa24WFhdW64RQXFxeEigEA/kAehL5GNyUa0pAAANSNiUsBABJ5AABHKy0tVU5Ojs+7XsvKympt53A4lJyc7HPDKSEhgWYuALQS5EHr1OimxF133aX//Oc/Wrt2bZ3rBw8erEsuuURz5sxp7CkAoNVi4lIAgEQeAIAklZeXe2801XyVlJTU2s5utyspKcnnhlNiYqLsdnsQqgYANDfywDoa3ZRYtGiRfv7znx9z/QUXXKC3336bpgQAHIGJSwEAEnkAwLoqKyuVk5Pj867XoqKiWtvZbDYlJCT43HBKTk6Ww+EIQtUAgOZGHlhbo5sSu3fvVvfu3Y+5vmvXrtq1a1djDw8ArU5eXp4yMjJ06NAhSUxcCgBWRR4AsIqqqirl5ub6vOO1oKCgzm3j4+Nr3XCiSQsArQN5gKM1uikRExNz3KbDjh075HK5Gnt4AGg1mLgUACCRBwBaN8MwlJeX5323a1ZWlvLz82WaZq1tY2JifG44paSkyOl0BqFqAEBzIw9QH02a6Pr555/X9ddfr/T0dJ91e/bs0d/+9jedeeaZTS4QAELZ0ROXdunSRaeeeioTlwKAxZAHAFoTwzBUUFDg847X3NxcGYZRa9uoqKhaN5wiIyODUDUAoLmRB2isRjcl7rvvPg0fPlx9+/bVtddeq759+0qSfvjhB7388ssyTVP33XdfsxUKAKGEiUsBABJ5ACD0maapwsJCn2d+5+TkqKqqqta2TqfT54ZTamoqzVcAaCXIAzSnRjclevbsqYyMDN100016/PHHfdaddtppeuqpp9S7d+8mFwgAocQ0TW3atEkrV670Tlzar18/DR06lGcgAoCFkAcAQlVxcbH279+vLVu2yDRNFRQUqLKystZ24eHhtW44xcbGBqFiAIA/kAfwp0Y3JSRpwIAB+vLLL5WTk6OffvpJktStWzcm6QNgSUdPXJqSkqLTTjuN10QAsBjyAECoKCkp8XnkRk5OjsrLy+XxeLRjxw7FxMTI4XAoLCxMycnJPjec4uPjmQ8HAFoJ8gCB1qSmRI2UlBT+JwuAZTFxKQBAIg8AtGzl5eU+N5yys7NVWlpaazu73a7ExER17NhRY8eOVfv27ZWQkCC73R6EqgEAzY08QEvQ5KbE3r17tWbNGhUWFtY5icnkyZObegoAaLH27t2rZcuWqaioSBITlwKAVZEHAFqSysrKWjecDh8+XGs7m82mxMREn3e8JiUlyTAMffjhh+rZsyePnAOAEEYeoKVqdFOivLxcU6ZM0bvvvivDMGSz2WSapiT5vBOMpgSA1qi0tFQrVqzQtm3bJEnR0dE69dRT1aVLl+AWBgAIKPIAQLBVVVUpJyfH54ZTYWFhndsmJCT43HBKTk5WWFjt2wJ1veEQANCykQcIJY1uSsyaNUvvvfee7r//fo0aNUpnnHGGXn31VbVr105PPPGE9u/fr7///e/NWSsABB0TlwIAJPIAQHB4PB7l5ub63HAqKCjwvkHwSLGxsT43nFJSUhQRERGEqgEAzY08QKhrdFNi0aJFmjp1qu644w7l5uZKktLT03XWWWfp7LPP1llnnaX58+fr2WefbbZiASCYmLgUACCRBwACwzAM5efn+9xwysvLq/Ndq9HR0bVuOLlcriBUDQBobuQBWqNGNyWysrI0fPhwSVJkZKSk6pnaa1x66aX685//TFMCQMg71sSlffr0YYInALAQ8gCAv5imqYKCAp8bTrm5ufJ4PLW2dblcSktLU0pKivemU1RUVBCqBgA0N/IAVtHopkSbNm28n5CIiopSYmKiNm/erAkTJkiSioqKVF5e3jxVAkCQ1DVx6ejRoxUTExPkygAAgUQeAGhORUVFPjeccnJy5Ha7a20XERHh847X1NRUXncAoBUhD2BVjW5KjBgxQsuWLdMdd9whSZowYYIeeeQRtWvXToZh6PHHH9fIkSObrVAACKSysjJlZmYycSkAWBx5AKCpDh8+7HPDKTs7W5WVlbW2CwsLU0pKis+7XuPi4mSz2YJQNQCguZEHwP80uilx880365133lFFRYWcTqfuu+8+ZWZmatKkSZKk7t2766mnnmq2QgEgEEzT1ObNm7VixQomLgUACyMPADRGWVlZrRtOZWVltbZzOBxKTk72ecdrQkICN5wAoJUgD4Dja3RTYsyYMRozZoz3+44dO2rjxo1av369HA6HevXqpbCwRh8eAAIuPz9fGRkZOnjwoCQmLgUAqyIPANRHRUVFrRtOR86zWMNutyspKcnnXa9JSUnMRQMArQR5ADRco7oGpaWl+uUvf6lLL71U11xzjXe53W7XwIEDm604AAiEqqoqrV27Vt9//70Mw1BYWJiGDRumvn378o8DALAQ8gDAsbjdbuXk5PjccKqZY+ZoiYmJPu94TUpK4g17ANBKkAdA82jUb0JUVJSWLl2q888/v7nrAYCAysnJ0bvvvqvS0lJJTFwKAFZFHgCoUVVVpdzcXGVnZ+vAgQNatmyZ9u/fL4fDUWvbuLg477td09LSlJyczCPeAKCVIA8A/2nS45syMzP1m9/8pjnrAYCAKCsrU0ZGhr799lv1799fcXFxTFwKABZEHgDWZhiG8vLylJWV5X3na15enkzTlCR5PB7vIzhiYmJ83vGakpIip9MZzPIBAM2EPAACq9FNiaefflrnnnuu7rrrLl1//fXq0KFDc9YFAH5x5MSlZWVlstls6tu3r0aNGsW7GADAQsgDwHoMw1BBQYHPIzdyc3NlGEatbSMjI72P2ggPD9fll1+uuLi4IFQNAGhu5AEQfI1uSgwcOFBVVVWaO3eu5s6dq7CwsFpdQZvNpsLCwiYXCQDN4eiJS5OTkzVixAhuQAGAxZAHQOtnmqYKCwuVk5PjfddrTk6Oqqqqam3rdDp93vGampqq6OhoSdXPDj9w4IAiIyMDPQQAQDMgD4CWqdFNiUsvvVQ2m605awEAv6iqqtKaNWtqTVzao0cPffTRR8EuDwAQIOQB0HoVFxf7vOM1JydHlZWVtbYLDw/3Pmqj5tnfvOMVAFoP8gAIDY1uSrzyyivNWAYA+MfevXu1bNkyFRUVSZI6d+6sU089VTExMXK73UGuDgAQKOQB0HqUlpZ63+1a82d5eXmt7cLCwpScnOzzjtf4+HjeXAcArQR5AISuRjclAKAlKysrU2ZmprZt2yZJio6OZuJSALAg8gAIbeXl5T7veM3OzlZpaWmt7ex2e60bTgkJCbLb7UGoGgDQ3MgDoHVpdFPi73//e722mzx5cmNPAQANVjNx6cqVK1VRUSFJ6tevn4YOHaqIiIggVwcACBTyAAg9lZWVPu92zcrK0uHDh2ttZ7PZlJiY6HPDKSkpSQ6HIwhVAwCaG3kAtH6Nbkr86le/Oua6Iz/+RFMCQKAcPXFpSkqKxo4dq9TU1CBXBgAIJPIAaPmqqqqUk5Pj847XwsLCOrdNSEjwueGUnJyssDA+9A8ArQF5AFhTo39zd+zYUWuZx+PRzp079cwzz2j37t169dVXm1QcANRHVVWV1q5dq7Vr1/pMXNq3b18+ogkAFkIeAC2Tx+NRbm6uz7te8/PzZZpmrW1jY2N9bjilpKTw6SYAaCXIAwA1Gt2U6Ny5c53Lu3XrprPOOkvjx4/X008/rfnz5ze6OAA4kX379ikjI6POiUsBANZBHgAtg2EYys/P93nHa15engzDqLVtdHR0rRtOLpcrCFUDAJobeQDgePz2GacLL7xQd999N00JAH5RVlamFStWaOvWrZKYuBQArIo8AILHNE0VFBT43HDKzc2Vx+Opta3L5fK54ZSamqqoqKggVA0AaG7kAYCG8ltTYvv27d5JBQGguTBxKQBAIg+AYCgtLdX27du9N55ycnLkdrtrbRcREVHrhhOfWgKA1oM8ANBUjW5KfPXVV3UuLygo0FdffaWnnnpKl1xySWMPDwC1MHEpAEAiD4BAOHz4sPfdrjk5OTpw4IC+/fZbFRQUyOFweLcLCwtTSkqKzw2nuLg42Wy2IFYPAGgu5AEAf2h0U+KMM86o84XFNE05HA5ddtllmjdvXpOKAwCpejKsNWvW+ExcOnToUPXr14+JSwHAQsgDwD/Kysp8HrmRnZ2tsrIyn208Ho/sdrtSU1PVrl077w2nhIQEbjgBQCtBHgAIlEY3JT7//PNay2w2mxITE9W5c2fFxcU1qTAAkGpPXNqpUyeNGTOGj3wCgMWQB0DzqKioqHXDqaSkpNZ2NptNSUlJPjeb2rZtqwsvvFDh4eFBqBwA0JzIAwDB1OimxOmnn96cdQCAj/LycmVmZnonLo2KitKpp56qrl27BrkyAEAgkQdA47ndbuXk5PjccKpp7B0tMTHR55EbSUlJCgsL8zkWn0gCgNBEHgBoaRrdlNixY4d++OEHTZgwoc71//73v9W/f3916dKlsacAYFGbN2/WihUrmLgUACyOPIAVmGX5MnculVmWJ1tkimxdfyabK6HBx6mqqlJubq7PDaeCgoI6t42Li/O54ZSSksK7XQEgyMgDAFbS6KbEH/7wBxUVFR2zKTF//nwlJCTorbfeqnPdI488ooMHD2rgwIGaN2+ehg8ffsJzvvXWW7rqqqt08cUX6/33329s6QBaqIKCAmVkZOjAgQOSpOTkZI0dO1ZpaWlBrgz+Qh4AqAt5YD1WzAPTNGV+v0DG969IpiHZHDJNj7T6KdkHXStb/0nHfDa3YRjKy8vzueGUl5cn0zRrbRsTE1PrhpPT6fTz6ACgccgD8gCANTS6KZGZmalbb731mOt/9rOf6Yknnqi1/O2339aMGTP03HPPacSIEXriiSd07rnnavPmzcf9H82dO3fqD3/4g8aOHdvYkgG0UExcak3kAYCjkQfWZNU8MH98Q8bal45YUFX9p1El47vnZQ+Pkq33RBmGoYKCAp8bTrm5uTIMo9YxIyMjfW44paamKjIyMkAjAoCmIQ9qFpAHAFq/Rjcl8vPzFRsbe8z1MTExys3NrbX8scce029+8xtNnTpVkvTcc8/pgw8+0Msvv6w777yzzmN5PB5dc801uvfee5WRkXHMj50BCD379u3TsmXLVFhYKImJS62EPABwJPLAuqyYB2ZVefU7Yo9ebpoqKvMou9it7H/8VXkD7crNK1BVVVWtbZ1OZ60bTtHR0QGoHgD8gzw4Yjl5AKCVa3RTolOnTvr66691ww031Lk+IyNDHTp08FlWWVmpb7/9VjNnzvQus9vtOvvss5WZmXnMc/35z39WWlqarr32WmVkZJywtoqKCu+zhyV5J+9xu91yu90n3P9INds3dL9Qx7gZt7+Vl5drxYoV2rZtm6Tqd3KMHj3aO3FpIGrh77tx426O60UehA7Gzbj9jTwIHvIgeHlg7F0lw10lqXp+lPwSt1ZuK1J2caUqq2oeuVEim3OV7PGdFR4erpSUFO9Xamqq4uLijlmbP/B7writgHGTBw1BHvB7YgWMm3E3Zv/6aHRT4qqrrtJ9992n4cOHa/r06d6P1Hs8Hj399NN6++239ac//clnn5ycHHk8HrVp08ZneZs2bbRp06Y6z7Ns2TK99NJLWrt2bb1rmzt3ru69995ayz/55BNFRUXV+zhHWrJkSaP2C3WM21oCNe59+/Zp8+bN3herTp06KTU1VRs3btTGjRsDUsOR+PtumNLS0iafmzwIPYzbWsgDayEPgpQHibd6//PH/T9qb8VeKUKyu+yKi4tTfHy84uLiFBcXp+joaNlsNuXm5io3N1ebN29u2rmbgN8Ta2Hc1kIekAcNwe+JtTBuawlEHjS6KTFz5kwtW7ZMt956q+6//3717NlTkrR582ZlZ2frjDPOqNWUaKji4mJNmjRJL7zwglJSUhpU24wZM7zfFxUVqWPHjjrnnHPq7CAfj9vt1pIlSzRu3DiFh4c3aN9QxrgZtz8UFBRo2bJlstls6tWrl5KSkjRmzJigTVzK33fjxl3zbqJAIg+Ch3Ezbn8gD1oG8iB4eWBmb5Tnk1u831ceylFcaaVGdI9Tn/Ro2e3VE5o6zn9GtqSTGnWO5sbvCeO2AsZNHjQEecDviRUwbsbdEA3Jg0Y3JZxOpz755BO9+uqreu+997R9+3ZJ0vDhw3XppZdq8uTJtSYkTElJkcPh0KFDh3yWHzp0SG3btq11ju3bt2vnzp2aMGGCd1nNBD5hYWHavHmzunfvXmdtTqez1vLw8PBG/yA1Zd9QxritxV/jPnriUqfT2aImLuXvu+H7NRV5EHoYt7WQB9ZCHgQ+D8x2/WWPby8V7ZZMQ4dLy+WQoY4JNjntbslmlxJPUlib3o06vj/xe2ItjNtayAPyoCH4PbEWxm0tgciDRjclpOrn+02dOtU7CdGJRERE6JRTTtGnn36qSy65RFJ1aHz66aeaPn16re179eql9evX+yy76667VFxcrCeffFIdO3ZsSvkAAmD//v3KyMhg4lL4IA8A6yEPUBer5oHNZpPj1JnyfHyTyssrVe6uvpEWH+movgFlD5dj9B1BrhIAAoc8IA8AWEujmxJ5eXnau3evBgwYUOf69evXq0OHDkpMTPRZPmPGDE2ZMkVDhw7V8OHD9cQTT6ikpMTb2Jg8ebLS09M1d+5cuVwu9evXz2f/hIQESaq1HEDLUjNx6ZYtWyRJUVFROvXUU70TlwLkAWAN5AFOxKp5YEvrJ8cFz6lgyaOSDirG6VB4mENKHynHkN+1mMd0AECgkAfkAQDraHRT4rbbbtPmzZu1YsWKOtf/7ne/U+/evfXSSy/5LL/iiiuUnZ2t2bNn6+DBgxo0aJA++ugj72RGu3fvbhEf3wfQeDWvDRUVFZKkvn37atiwYYqIiAhyZWhJyAOg9SMPUB9WzgNbck8dHjRD9pzeSkqLk2PCL2SLTDzxjgDQCpEH5AEA62h0U+Kzzz7TDTfccMz1EyZM0HPPPVfnuunTp9f58TtJ+uKLL4573ldeeaW+JQIIsIKCAmVkZOjAgQOSpKSkJJ122mlBm7gULR95ALRO5AEaysp5UFBQIFtEtBI79OYGFADLIw/IAwDW0OimRHZ2tlJSUo65Pjk5WVlZWY09PIAQcvTEpWFhYS1q4lIAQGCQB0DDFRQUSPrfY0cAANZEHgCwkkY3Jdq1a6c1a9Ycc/23336r1NTUxh4eQIioa+LSU089VbGxsUGuDAAQSOQB0DjchAIASOQBAGtpdFPikksu0fz583X++efroosu8lm3ePFiLViw4LiPdwIQ2uqauHT06NHq1q1bkCsDAAQSeQA0nsfjUXFxsSRuQgGAlZEHAKym0U2Je+65R0uXLtXPf/5zDRw4UP369ZMk/fDDD/r+++/Vu3dv3Xvvvc1WKICWY8uWLVqxYoXKy8slMXEpAFgVeQA0TWFhoUzTVEREhKKiooJdDgAgSMgDAFbT6KZEfHy8VqxYoYcffljvvfeeFi1aJEnq3r277r77bt1+++2Kjo5utkIBBB8TlwIAJPIAaC48qgMAIJEHAKyn0U0JSYqOjta9997LJyKAVs7j8Wjt2rVas2aNd+LSU045Rf3792fiUgCwEPIAaF7chAIASOQBAOtpUlMCQOvHxKUAAIk8APyBm1AAAIk8AGA9TWpKlJeX691339V3332nwsJCGYbhs95ms+mll15qUoEAgoOJSwEAEnkA+FPNTajExMTgFgIACCryAIDVNLopsWvXLp155pnauXOnEhISVFhYqKSkJBUUFMjj8SglJUUxMTHNWSuAADl64tI+ffpo+PDhTFwKABZDHgD+Y5om74wFAJAHACyp0U2J22+/XYWFhVqxYoW6deumtLQ0vf322zr11FP11FNP6emnn9bHH3/cnLUC8LOCggKtXLlS+/fvl8TEpQBgVeQB4H8lJSWqqqqS3W7nMWgAYGHkAQAranRT4rPPPtO0adM0fPhw5eXlSaru7jqdTt1+++3auHGjbr31Vn3wwQfNViwA//B4PNq2bZsOHjwom83GxKUAYFHkARA4Ne+KjY+P5/cLACyMPABgRY1uSpSWlqpLly6SpLi4ONlsNu/Eh5I0atQo/eEPf2hygQD8a//+/friiy+0fft29e/fX126dNGYMWN4hwYAWAx5AAQWj+oAAEjkAQBranRTolOnTtq7d2/1QcLClJ6erhUrVugXv/iFJGnDhg1yuVzNUyWAZnfkxKUej0dOp1NnnXWWevbsGezSAAABRB4AwZGfny+Jm1AAYHXkAQAranRT4qyzztLixYs1Z84cSdKvfvUrzZ07V/n5+TIMQ6+99pomT57cbIUCaD5HT1zau3dvpaamqlu3bkGuDAAQSOQBEDy8MxYAIJEHAKyp0U2JO++8U6tXr1ZFRYWcTqdmzZql/fv3a9GiRXI4HLr66qv12GOPNWetAJqosLBQGRkZPhOXjh07VklJSfrwww+DXB0AIFDIAyD4uAkFAJDIAwDW1KTHN3Xq1Mn7vcvl0osvvqgXX3yxWQoD0Hw8Ho++//57fffddzIMo9bEpW63O9glAgACgDwAWoaKigqVlZVJ4iYUAFgZeQDAqhrdlAAQGg4cOKCMjAzvuy86duzIxKUAYEHkAdBy1PweRkdHKzw8PLjFAACChjwAYFU0JYBWqry8XCtXrtTmzZslSVFRURo9ejTPCQcAiyEPgJaHR3UAACTyAIB10ZQAWqGjJy7t06ePhg8froiIiCBXBgAIJPIAaJm4CQUAkMgDANZFUwJoRY41cWmbNm2CXBkAIJDIA6Blq7kJlZiYGNxCAABBRR4AsCqaEkArUNfEpUOGDNGAAQNkt9uDXR4AIEDIAyA08M5YAIBEHgCwLpoSQIg7euLSDh06aMyYMYqLiwtuYQCAgCIPgNDg8XhUVFQkiZtQAGBl5AEAK6MpAYSooycujYyM1OjRo9W9e/cgVwYACCTyAAgtRUVFMk1TERERioqKCnY5AIAgIQ8AWBlNCSAEbd26VZmZmUxcCgAWRx4AoYdHdQAAJPIAgLXRlABCCBOXAgAk8gAIZfn5+ZK4CQUAVkceALAymhJACKiZuHTNmjXyeDxMXAoAFkUeAKGPd8YCACTyAIC10ZQAWjgmLgUASOQB0FpwEwoAIJEHAKyNpgTQQjFxKQBAIg+A1sQ0TW5CAQDIAwCWR1MCaIGOnri0d+/eGj58uJxOZ5ArAwAEEnkAtC4lJSWqqqqS3W7nU04AYGHkAQCroykBtCCFhYVatmyZ9u3bJ0lKTEzUaaedxsSlAGAx5AHQOtW8KzYuLo55YADAwsgDAFZHUwJoAY6euNThcOiUU05h4lIAsBjyAGjdeFQHAEAiDwCApgQQZAcPHtRXX33FxKUAYHHkAdD6cRMKACCRBwBAUwIIkoqKCq1cuVKbNm2SxMSlAGBV5AFgHdyEAgBI5AEA0JQAgoCJSwEAEnkAWA03oQAAEnkAADQlgACqa+LSsWPHqm3btkGuDAAQSOQBYD2VlZUqLS2VxE0oALAy8gAAaEoAAWEYhr7//nt99913TFwKABZGHgDWVfOu2KioKEVERAS3GABA0JAHAEBTAvA7Ji4FAEjkAWB1PKoDACCRBwAg0ZQA/KauiUtHjRqlk046KciVAQACiTwAIP3vJlRiYmJwCwEABBV5AAA0JQC/2LZtm5YvX+6duLRXr14aMWIEE5cCgMWQBwBq8M5YAIBEHgCARFMCaFZFRUXKyMhg4lIAsDjyAMDR8vPzJXETCgCsjjwAAJoSQLOoa+LSIUOGaODAgUxcCgAWQh4AqIthGCoqKpLETSgAsDLyAACq0ZQAmujgwYPKyMjwvtuBiUsBwJrIAwDHUlRUJNM0FR4erujo6GCXAwAIEvIAAKrRlAAaiYlLAQASeQDgxHhUBwBAIg8AoAZNCaARtm3bpszMTJWVlUli4lIAsCryAEB9MKkpAEAiDwCgBk0JoAGKioq0bNky7d27VxITlwKAVZEHABqCm1AAAIk8AIAaNCWAemDiUgCARB4AaBxuQgEAJPIAAGrQlABO4OiJS9PT0zV27FgmLgUAiyEPADRWzU2oxMTE4BYCAAgq8gAAqtGUAI7h6IlLXS6XRo8ezcSlAGAx5AGApigpKZHb7ZbNZqOJCQAWRh4AwP/QlADqsG3bNn3zzTdMXAoAFkceAGiqmnfFxsXF8Zg3ALAw8gAA/oemBHCEoqIiffPNN9q/f78cDocSEhJ02mmnMXEpAFgMeQCgufCoDgCARB4AwJFoSgCqnrh03bp1WrVqlXJzc9WxY0cNGzaMiUsBwGLIAwDNjUlNAQASeQAAR6IpAcs7cuJSj8ej5ORkXXrppUpOTg52aQCAACIPAPhDfn6+JG5CAYDVkQcA8D80JWBZFRUVWrVqlTZu3CipeuLSYcOGacuWLUw6BQAWQh4A8CfeGQsAkMgDADgSTQlY0rZt25SZmVlr4lK73a4tW7YEuToAQKCQBwD8qbKyUqWlpZK4CQUAVkYeAIAvmhKwlKKiIi1btkx79+6VVP2PgbFjx6pdu3aSJLfbHczyAAABQh4ACISad8VGRUUpIiIiuMUAAIKGPAAAXzQlYAk1E5d+++238ng8cjgcGjx4sAYOHCiHwxHs8gAAAUIeAAgkHtUBAJDIAwA4mj0YJ50/f766dOkil8ulESNGaNWqVcfc9oUXXtDYsWOVmJioxMREnX322cfdHjjaoUOH9N5772nVqlXyeDxKT0/XxIkTNWTIEG5AAUFGHiCQyAOg5WqtecBNKABoGPIAAKwh4E2Jt99+WzNmzNCcOXP03XffaeDAgTr33HOVlZVV5/ZffPGFrrrqKn3++efKzMxUx44ddc4552jfvn0BrhyhpqKiQhkZGVq8eLHy8vLkcrl05plnavz48YqPjw92eYDlkQcIFPIAaNlacx5wEwoA6o88AADrCHhT4rHHHtNvfvMbTZ06VX369NFzzz2nqKgovfzyy3Vu//rrr2vatGkaNGiQevXqpRdffFGGYejTTz8NcOUIJdu3b9c//vEPbdy4UVL1xKWXX365Tj755CBXBqAGeYBAIA+Alq815wE3oQCg/sgDALCOgM4pUVlZqW+//VYzZ870LrPb7Tr77LOVmZlZr2OUlpbK7XYrKSnJX2UihJ1o4lIALQN5AH8jD4DQ0JrzwDAMFRUVSeImFACcCHkAANYS0KZETk6OPB6P2rRp47O8TZs22rRpU72Occcdd6h9+/Y6++yzj7lNRUWFKioqvN/XvPi73W653e4G1VyzfUP3C3WhNu6aiUvXrFnjnbh00KBBGjBggBwOR73HEWrjbi6Mm3E3Zv+mIA9CR6iNmzxoGsbNuBuzf1O05jwoKCiQ2+1WWFiYnE5nyP988XvCuK2AcZMHDUEehPY4GopxM24rCGQeBLQp0VQPPvig3nrrLX3xxRdyuVzH3G7u3Lm69957ay3/5JNPFBUV1ahzL1mypFH7hbpQGHdBQYE2bNig4uJiSVJSUpL69OmjAwcO6MCBA406ZiiM2x8Yt7U0dtylpaXNXEnDkQeBFwrjJg+aD+O2FvLAP3lw6NAhrV+/XnFxcfrwww8bdY6WiN8Ta2Hc1kIekAcNwe+JtTBuawlEHgS0KZGSkiKHw6FDhw75LD906JDatm173H3/+te/6sEHH9TSpUs1YMCA4247c+ZMzZgxw/t9UVGRd8KjuLi4BtXsdru1ZMkSjRs3TuHh4Q3aN5SFwrgrKyu1evVq7d+/X126dJHT6dTIkSOb9JzwUBi3PzBuxt0QNe8magryIHSEwrjJg+bDuBl3Q5AHx79ua9eulcfjUffu3XXmmWc26BwtEb8njNsKGDd50BDkAb8nVsC4GXdDNCQPAtqUiIiI0CmnnKJPP/1Ul1xyiSR5JyGaPn36Mfd7+OGHdf/99+vjjz/W0KFDT3gep9Mpp9NZa3l4eHijf5Casm8oa6nj3r59u5YvX66ysjI5HA717NlTI0aMOO47IhqipY7b3xi3tTR23M1xrciD0NNSx00e+AfjthbywD95UFJSIofDoZSUlFb1c8XvibUwbmshD8iDhuD3xFoYt7UEIg8C/vimGTNmaMqUKRo6dKiGDx+uJ554QiUlJZo6daokafLkyUpPT9fcuXMlSQ899JBmz56tN954Q126dNHBgwclSTExMYqJiQl0+Qiy4uJiLVu2THv27JHExKVAKCMP0BTkAdB6tNY8KCgokCQlJiYGtxAACBHkAQBYR8CbEldccYWys7M1e/ZsHTx4UIMGDdJHH33kncxo9+7dstvt3u2fffZZVVZWauLEiT7HmTNnju65555Alo4gqpm49LvvvlNVVZXsdruGDBmigQMHyuFwBLs8AI1AHqAxyAOg9WmteVBzEyohISGodQBAqCAPAMA6gjLR9fTp04/58bsvvvjC5/udO3f6vyC0aIcOHVJGRoby8vIkSe3bt9fYsWMVHx8f5MoANBV5gIYgD4DWq7XlQWlpqSorK2Wz2Rr8jHIAsDLyAACsIShNCaA+KisrtWrVKm3YsEGS5HK5NHLkSPXo0SPIlQEAAok8ABBqat4VGxcXx6e4AMDCyAMAqBtNCbRIP/30k5YvX67S0lJJavaJSwEAoYE8ABCKeFQHAEAiDwDgWGhKoEWpa+LSMWPGqH379kGuDAAQSOQBgFCWn58viZtQAGB15AEA1I2mBFoEwzC0fv16ffvtt0xcCgAWRh4AaA14ZywAQCIPAOBYaEog6OqauHTMmDGENgBYDHkAoLXgJhQAQCIPAOBYaEogaJi4FAAgkQcAWhe3262SkhJJ3IQCACsjDwDg2GhKICiOnri0R48eGjlyJBOXAoDFkAcAWpuad8VGRkbK6XQGtxgAQNCQBwBwbDQlEFBMXAoAkMgDAK0Xj+oAAEjkAQAcD00JBERdE5cOHjxYgwYNYuJSALAQ8gBAa8dNKACARB4AwPHQlIDfZWVl6auvvmLiUgCwOPIAgBXU3IRKTEwMbiEAgKAiDwDg2GhKwG+YuBQAIJEHAKyFd8YCACTyAACOh6YE/IKJSwEAEnkAwFoMw1BhYaEkbkIBgJWRBwBwfDQl0KyKi4v19ddfa/fu3ZKk+Ph4jR07lolLAcBiyAMAVlRcXCzDMBQWFqbo6OhglwMACBLyAACOj6YEmgUTlwIAJPIAgLUd+agOm80W3GIAAEFDHgDA8dGUQJMdPXFpu3btNHbsWD6iCAAWQx4AsLr8/HxJPKoDAKyOPACA46MpgUZj4lIAgEQeAEANJjUFAEjkAQCcCE0JNAoTlwIAJPIAAI7ETSgAgEQeAMCJ0JRAgzBxKQBAIg8AoC7chAIASOQBAJwITQnUi2EY+uGHH/TNN98wcSkAWBh5AAB1Ky0tVWVlpWw2m+Lj44NdDgAgSMgDADgxmhI4oaysLGVkZCg3N1cSE5cCgFWRBwBwbDXvio2NjaVJCwAWRh4AwInRlMAxud1uLV++XJs3b5YkOZ1OjRw5Uj179gxyZQCAQCIPAODEeFQHAEAiDwCgPmhKoE4//fSTvv76a/Xo0UMOh4OJSwHAosgDAKgfbkIBACTyAADqg6YEfBw+fFjLli3Tjh07VFFRobi4OJ155plMXAoAFkMeAEDDcBMKACCRBwBQHzQlIKnuiUu7d++uSy+9lHfDAoCFkAcA0DjchAIASOQBANQHTQnUOXHpyJEjtXz5ciZlAgALIQ8AoHHcbrcOHz4siZtQAGBl5AEA1A9NCQurrKzU6tWr9eOPP0rynbjU7XYHuToAQKCQBwDQNIWFhZIkl8vFp8oAwMLIAwCoH5oSFrVjxw59/fXXKi0tlSQmLgUAiyIPAKDpeFQHAEAiDwCgvmhKWEzNxKW7d++WJMXHx2vs2LFMXAoAFkMeAEDz4SYUAEAiDwCgvmhKWERdE5cOGjRIgwcP5jnhAGAh5AEANL+am1CJiYnBLQQAEFTkAQDUD00JC8jOztZXX33lnbi0bdu2Ou200+jcA4DFkAcA4B/5+fmSeGcsAFgdeQAA9UNTohWrrKzUN998ox9++EHS/yYu7dGjh2w2W5CrAwAECnkAAP5jGIZ3YlNuQgGAdZEHAFB/NCVaqaMnLj355JM1cuRIRUZGBrkyAEAgkQcA4F+HDx+WYRhyOByKiYkJdjkAgCAhDwCg/mhKtDKHDx/W119/rV27dkmS4uLiNHbsWKWnpwe5MgBAIJEHABAYRz6qg0+fAYB1kQcAUH80JVoJJi4FAEjkAQAEWs2kpjyqAwCsjTwAgPqjKdEKZGdnKyMjQzk5OZKYuBQArIo8AIDA4yYUAEAiDwCgIWhKhLCaiUt//PFHmaYpp9OpESNGqGfPnnxUEAAshDwAgODhJhQAQCIPAKAhaEqEqJ07d+rrr79WSUmJJCYuBQCrIg8AILi4CQUAkMgDAGgImhIhholLAQASeQAALUFZWZkqKiokSfHx8UGuBgAQLOQBADQMTYkQYRiGfvzxR61evdpn4tJBgwYpLIy/RgCwCvIAAP6vvXsPjqq+/z/+2t1kEyIJCT9ygYIgQQTlkgI/IiBlsEgs1oIWzIhARFCurRWLIkgDRcU66E8HQSvIxRlsLBb8toIBpDAMN7kY+lVBAcOlahOKBRMSIJvs5/dHmoWQANlN9myS83x0MpKTc3bf7012X9Pz3j2f+qPiXbHR0dG8BgOAjZEHAOAfXikbgOoWLu3fv7/i4uJCXBkAwErkAQDUL1yqAwAgkQcA4C+GEvUYC5cCACTyAADqq4qTUAyHAcDeyAMA8A9DiXrqyoVLO3TooD59+rBwKQDYDHkAAPXXmTNnJPHOWACwO/IAAPzDUKKeOXfunHbu3Knjx49LKl+49I477lDr1q1DWxgAwFLkAQDUf1yuAwAgkQcA4C+GEvVEdQuXdu/eXT/+8Y9ZJAkAbIQ8AICGobS0VOfOnZPESSgAsDPyAAD8x9mNeuD06dPatm0bC5cCgM2RBwDQcFS8KzYyMlKRkZGhLQYAEDLkAQD4j6FECHk8Hu3du9e3cKnb7dbtt9/OwqUAYDPkAQA0PFyqAwAgkQcAEAiGEiHCwqUAAIk8AICG6ocffpDESSgAsDvyAAD8x1DCYixcCgCQyAMAaOh4ZywAQCIPACAQDCUsUrFw6b59++TxeFi4FABsijwAgMaBk1AAAIk8AIBAcPbDAixcCgCQyAMAaCyMMSooKJDESSgAsDPyAAACw1AiiDwej/bt26fPP/+chUsBwMbIAwBoXM6fP6+ysjK53W5FR0eHuhwAQIiQBwAQGIYSQcLCpQAAiTwAgMao4jW9WbNmDJcBwMbIAwAIDEOJOsbCpQAAiTwAgMasqKhIbrebS3UAgM2RBwAQGIYSdcQYoy+++EJ79+5l4VIAsDHyAAAaP05CAQAk8gAAAsXZkTpw5cKliYmJ+slPfsLCpQBgM+QBANhDUVGR4uLiOAkFADZHHgBAYBhK1EJ1C5empqaqU6dOXEsQAGyEPAAAe6m4hjhDZwCwN/IAAALDUOIqyspK5P36OXmOfSJpsjy7H5Kz+wy5YnpIkk6cOKEdO3bo3LlzksoXLr399tsVFRUVwqoBAHWNPAAASJLXW6qygqUq/PZDlZTcqLIfshTdZKCk/xPq0gAAFiIPAKD2nKG400WLFqldu3aKjIxUamqq9uzZc839V69erU6dOikyMlJdu3bV+vXrg1pfWcF+ef/yEzl2bpbjXxclSY6vT8msmaqzO8Zo48aN2rBhg86dO6fo6GgNGTJEd955JyegAMBP5AEAQKr/eeAtOaKygz3kOvL/VHDiqCSpaUmuXAeHynNqVlDvGwDshDwAAHuwfCjx3nvvadq0acrMzNSnn36q7t27Ky0tTadOnap2/507d+rBBx/UuHHjlJOTo2HDhmnYsGH6/PPPg1JfWVmZvNlT5CiueGjKL7thjPT5N8V6f9lOff3JQjmdTqWkpGjEiBFq3bp1UGoBgMaMPAAASPU/D7xer8oOD5fz/AVJUkGhV5LUPDpMMkbOE6tVenZJUO4bAOyEPAAA+7B8KPHKK6/o0Ucf1dixY3XrrbfqzTffVFRUlJYtW1bt/q+99pruvvtuTZ8+XZ07d9a8efPUo0cPvf7660Gpz+Q+J0exSxUnnySpoKBAf/30tHYdLZSnzCih6LDuv/9+9e7dW2FhXAELAAJBHgAApPqfB96Ct+U6f0GO/+bBmcIySVKzaKdvm/nXH4Ny3wBgJ+QBANiHpWdQSkpKtH//fj3zzDO+bU6nU4MGDdKuXbuqPWbXrl2aNm1apW1paWn64IMPrno/Fy9e1MWLF33fFxQUSCpfiNTj8VyzRs+xXXIoXJJDZV6j3bnF2n1mt24pMooMC9f/bR+jW1pGyek+J48n+jodN1wVj9P1Hq/Ghr7p2w5q23ddPF7kQcPB84S+7YC+yYNr1vj9R3J53b4TTv8pMJKkpjc0kcf73/d4FZTo4vl8OcOaX/O2GjKeJ/RtB/RNHlyzRvJAEs8T+rYH+g5+Hlg6lDh9+rTKysqUmJhYaXtiYqK+/PLLao/Jy8urdv+8vLyr3s/8+fM1d+7cKts3btxYg+t8T5Liyv/l9Xq127NbxhTqbPt0derUSScjInRSkjbnSMq5zm01fJs2bQp1CSFB3/ZC3/4pLi6u9X2TBw0PzxN7oW97IQ+ulwcZlb7b/u02Sef1vxee1Dd5l510ytt9ndtpHHie2At92wt5QB74g+eJvdC3vViRB43yWhPPPPNMpWl5QUGB2rRpo8GDBysmJuaax3r2Zshx+DtVXK7jx62lj5s8pPSw1Qor3iIVSwrzyjF8vVwudxC7CC2Px6NNmzbprrvuUnh4eKjLsQx907cd1LbvincTNQTkQe3xPKFvO6Bv8uBaPPnT5Px2q++dsb3SXFp3fJKGJb+uqIjyd4N5wxxydtktp7NR/t8rSTxP6Nse6Js8uBbyoBzPE/q2A/oOfh5Y+irZokULuVwu5efnV9qen5+vpKSkao9JSkrya39JioiIUERERJXt4eHh131And1myxwZJxlJcigx2q34uHiFnSlRuEokGXlvbC535A3XvJ3GoiaPWWNE3/ZC3/4fV1vkQcPD88Re6NteyINr9+FKfEomb6NkjBxyqEWzCCUmJioqwqNw50UZGZUlpsod0eSat9NY8DyxF/q2F/KAPPAHzxN7oW97sSIPLF3o2u12q2fPntq8ebNvm9fr1ebNm9WnT59qj+nTp0+l/aXyj5Bcbf/acjXtLJOS8t/vzGU/MZKMzA1euVLfDsp9A4BdkAcAAKlh5IEzvI28bUdJksxleWD++z9vVJTCWi4Myn0DgF2QBwBgL5YOJSRp2rRpWrJkiVauXKlDhw5p0qRJKioq0tixYyVJY8aMqbSw0eOPP67s7Gy9/PLL+vLLLzVnzhzt27dPU6dODVqN4d0XS/2HycRKvhNRYUbe5BZy/uJ/5IpIvMbRAICaIA8AAFIDyYP42TIdZ8sb3dx3IsqEOVXWsq9cnbbK6YoN2n0DgF2QBwBgH5Zf5C49PV3//ve/9bvf/U55eXlKSUlRdna2b3GikydPyum8NCvp27ev3n33XT377LOaOXOmbr75Zn3wwQfq0qVLUOsMS35KSn5KZUX50se75Ri+Qe7IyKDeJwDYCXkAAJAaUB40Gy01Gy3vhTNS3jY5u+yRu5pLgAAAAkMeAIB9hGTlnalTp151cr1169Yq20aMGKERI0YEuarqudzNy//rcoXk/gGgMSMPAABSw8oDp6tp+X+dln/oHAAaPfIAAOyBV04AAAAAAAAAAGAJhhIAAAAAAAAAAMASDCUAAAAAAAAAAIAlGEoAAAAAAAAAAABLMJQAAAAAAAAAAACWYCgBAAAAAAAAAAAswVACAAAAAAAAAABYgqEEAAAAAAAAAACwBEMJAAAAAAAAAABgibBQF2AFY4wkqaCgwO9jPR6PiouLVVBQoPDw8Lourd6ib/q2A/oOrO+K19KK19aGhDzwH33Ttx3QN3ngD/5e6NsO6Ju+/UEe8PdiB/RN33ZgZR7YYihRWFgoSWrTpk2IKwGAxqOwsFDNmjULdRl+IQ8AoO6RBwAAiTwAAJSrSR44TEMcZfvJ6/Xqu+++U3R0tBwOh1/HFhQUqE2bNvrnP/+pmJiYIFVY/9A3fdsBfQfWtzFGhYWFatWqlZzOhnUVQPLAf/RN33ZA3+SBP/h7oW87oG/69gd5wN+LHdA3fduBlXlgi09KOJ1OtW7dula3ERMTY6s/wgr0bS/0bS+16buhvQOqAnkQOPq2F/q2F/IgMPy92At92wt9+4884O/FLujbXujbfzXNg4Y1wgYAAAAAAAAAAA0WQwkAAAAAAAAAAGAJhhLXERERoczMTEVERIS6FEvRN33bAX3bq+/asuvjRt/0bQf0ba++a8uujxt907cd0Le9+q4tuz5u9E3fdkDfwe/bFgtdAwAAAAAAAACA0OOTEgAAAAAAAAAAwBIMJQAAAAAAAAAAgCUYSgAAAAAAAAAAAEswlAAAAAAAAAAAAJZgKCFp0aJFateunSIjI5Wamqo9e/Zcc//Vq1erU6dOioyMVNeuXbV+/XqLKq1b/vS9ZMkS9e/fX3FxcYqLi9OgQYOu+zjVV/7+vitkZWXJ4XBo2LBhwS0wSPzt++zZs5oyZYpatmypiIgIdezYsUH+rfvb96uvvqpbbrlFTZo0UZs2bfTEE0/owoULFlVbN7Zt26Z7771XrVq1ksPh0AcffHDdY7Zu3aoePXooIiJCHTp00IoVK4JeZ31EHpAHNUEekAcNBXkQOPKAPKgJ8oA8aCjIg8CRB+RBTZAH5EFDUa/ywNhcVlaWcbvdZtmyZeaLL74wjz76qImNjTX5+fnV7r9jxw7jcrnMSy+9ZA4ePGieffZZEx4ebj777DOLK68df/seOXKkWbRokcnJyTGHDh0yDz/8sGnWrJn55ptvLK68dvztu8KxY8fMj370I9O/f38zdOhQa4qtQ/72ffHiRdOrVy8zZMgQs337dnPs2DGzdetWc+DAAYsrrx1/+161apWJiIgwq1atMseOHTMbNmwwLVu2NE888YTFldfO+vXrzaxZs8yaNWuMJLN27dpr7p+bm2uioqLMtGnTzMGDB83ChQuNy+Uy2dnZ1hRcT5AH5AF5UBV5QB6QB+QBeVA98oA8aEjIg8CQB+QBeVAVeUAe1FUe2H4o0bt3bzNlyhTf92VlZaZVq1Zm/vz51e7/wAMPmHvuuafSttTUVDNhwoSg1lnX/O37SqWlpSY6OtqsXLkyWCUGRSB9l5aWmr59+5qlS5eajIyMBhky/vb9xhtvmPbt25uSkhKrSgwKf/ueMmWKufPOOyttmzZtmunXr19Q6wymmoTMU089ZW677bZK29LT001aWloQK6t/yINy5AF5cDny4BLywD7Ig3LkAXlwOfLgEvLAPsiDcuQBeXA58uAS8qB2bH35ppKSEu3fv1+DBg3ybXM6nRo0aJB27dpV7TG7du2qtL8kpaWlXXX/+iiQvq9UXFwsj8ej5s2bB6vMOhdo37///e+VkJCgcePGWVFmnQuk77/+9a/q06ePpkyZosTERHXp0kUvvPCCysrKrCq71gLpu2/fvtq/f7/vI3u5ublav369hgwZYknNodIYXtdqizwgD8gD8uBy5MElDe11rbbIA/KAPCAPLkceXNLQXtdqizwgD8gD8uBy5MEldfW6FlbrW2jATp8+rbKyMiUmJlbanpiYqC+//LLaY/Ly8qrdPy8vL2h11rVA+r7S008/rVatWlX5w6zPAul7+/btevvtt3XgwAELKgyOQPrOzc3V3//+dz300ENav369jh49qsmTJ8vj8SgzM9OKsmstkL5Hjhyp06dP64477pAxRqWlpZo4caJmzpxpRckhc7XXtYKCAp0/f15NmjQJUWXWIQ/IA4k8qA55QB6QB+XIg+sjDxoO8oA8uB7ygDwgD8qRB1WRB+RBXeWBrT8pgcC8+OKLysrK0tq1axUZGRnqcoKmsLBQo0eP1pIlS9SiRYtQl2Mpr9erhIQEvfXWW+rZs6fS09M1a9Ysvfnmm6EuLai2bt2qF154QYsXL9ann36qNWvWaN26dZo3b16oSwPqJfKg8SMPyAOgJsiDxo88IA+AmiAPGj/ygDyoK7b+pESLFi3kcrmUn59faXt+fr6SkpKqPSYpKcmv/eujQPqusGDBAr344ov6+OOP1a1bt2CWWef87fvrr7/W8ePHde+99/q2eb1eSVJYWJi++uorJScnB7foOhDI77tly5YKDw+Xy+XybevcubPy8vJUUlIit9sd1JrrQiB9z549W6NHj9b48eMlSV27dlVRUZEee+wxzZo1S05n45zjXu11LSYmxhbvgpLIA/KgHHlQFXlAHpAH5ciDqyMPyAPyoPEhD8gD8qAceVAVeUAe1FUeNM5HrIbcbrd69uypzZs3+7Z5vV5t3rxZffr0qfaYPn36VNpfkjZt2nTV/eujQPqWpJdeeknz5s1Tdna2evXqZUWpdcrfvjt16qTPPvtMBw4c8H394he/0MCBA3XgwAG1adPGyvIDFsjvu1+/fjp69KgvVCXp8OHDatmyZYMIGCmwvouLi6sESUXQlq8B1Dg1hte12iIPyAPygDy4HHlwSUN7Xast8oA8IA/Ig8uRB5c0tNe12iIPyAPygDy4HHlwSZ29rtV6qewGLisry0RERJgVK1aYgwcPmscee8zExsaavLw8Y4wxo0ePNjNmzPDtv2PHDhMWFmYWLFhgDh06ZDIzM014eLj57LPPQtVCQPzt+8UXXzRut9u8//775l//+pfvq7CwMFQtBMTfvq+UkZFhhg4dalG1dcffvk+ePGmio6PN1KlTzVdffWU+/PBDk5CQYJ577rlQtRAQf/vOzMw00dHR5k9/+pPJzc01GzduNMnJyeaBBx4IVQsBKSwsNDk5OSYnJ8dIMq+88orJyckxJ06cMMYYM2PGDDN69Gjf/rm5uSYqKspMnz7dHDp0yCxatMi4XC6TnZ0dqhZCgjwgD8gD8qACeUAekAfkAXlAHhhDHpAH5AF5QB6QB+XIg7rPA9sPJYwxZuHChebGG280brfb9O7d2+zevdv3swEDBpiMjIxK+//5z382HTt2NG6329x2221m3bp1FldcN/zpu23btkZSla/MzEzrC68lf3/fl2uoIWOM/33v3LnTpKammoiICNO+fXvz/PPPm9LSUourrj1/+vZ4PGbOnDkmOTnZREZGmjZt2pjJkyebM2fOWF94LWzZsqXa52tFrxkZGWbAgAFVjklJSTFut9u0b9/eLF++3PK66wPygDyoQB5cQh6QB3ZEHpAHFciDS8gD8sCOyAPyoAJ5cAl5QB7UBYcxjfgzJgAAAAAAAAAAoN6w9ZoSAAAAAAAAAADAOgwlAAAAAAAAAACAJRhKAAAAAAAAAAAASzCUAAAAAAAAAAAAlmAoAQAAAAAAAAAALMFQAgAAAAAAAAAAWIKhBAAAAAAAAAAAsARDCQAAAAAAAAAAYAmGEgAAAAAAAAAAwBIMJQAAAAAAAAAAgCUYSgAAAAAAAAAAAEswlAAAAAAAAAAAAJZgKAEAAAAAAAAAACzBUAIAAAAAAAAAAFiCoQQAAAAAAAAAALAEQwkAAAAAAAAAAGAJhhIAAAAAAAAAAMASDCUAAAAAAAAAAIAlGEoAAAAAAAAAAABLMJQAAAAAAAAAAACWYCgBAAAAAAAAAAAswVACAAAAAAAAAABYgqEEAAAAAAAAAACwBEMJAAAAAAAAAABgCYYSAAAAAAAAAADAEgwlAAAAAAAAAACAJRhKAAAAAAAAAAAASzCUAAAAABqAdu3a6ec//3moywAAAACAWmEoAQAAAKBaxcXFmjNnjrZu3RrqUgAAAAA0EgwlAAAAAFSruLhYc+fOZSgBAAAAoM4wlAAAAABCpKioKNQlAAAAAIClGEoAAAAAFpgzZ44cDocOHjyokSNHKi4uTnfccYdKS0s1b948JScnKyIiQu3atdPMmTN18eLFam9n48aNSklJUWRkpG699VatWbOm2vu50ooVK+RwOHT8+HHftn379iktLU0tWrRQkyZNdNNNN+mRRx6RJB0/flzx8fGSpLlz58rhcMjhcGjOnDmSpIcfflhNmzbVt99+q2HDhqlp06aKj4/Xb3/7W5WVlVW6b6/Xq1dffVW33XabIiMjlZiYqAkTJujMmTOV9rtWPRWysrLUs2dPRUdHKyYmRl27dtVrr712/V8AAAAAgHohLNQFAAAAAHYyYsQI3XzzzXrhhRdkjNH48eO1cuVKDR8+XE8++aQ++eQTzZ8/X4cOHdLatWsrHXvkyBGlp6dr4sSJysjI0PLlyzVixAhlZ2frrrvu8quOU6dOafDgwYqPj9eMGTMUGxur48eP+4Yc8fHxeuONNzRp0iTdd999uv/++yVJ3bp1891GWVmZ0tLSlJqaqgULFujjjz/Wyy+/rOTkZE2aNMm334QJE7RixQqNHTtWv/71r3Xs2DG9/vrrysnJ0Y4dOxQeHn7deiRp06ZNevDBB/XTn/5Uf/jDHyRJhw4d0o4dO/T444/794sAAAAAEBIMJQAAAAALde/eXe+++64k6R//+IemTJmi8ePHa8mSJZKkyZMnKyEhQQsWLNCWLVs0cOBA37GHDx/WX/7yF9+AYNy4cerUqZOefvppv4cSO3fu1JkzZ7Rx40b16tXLt/25556TJN1www0aPny4Jk2apG7dumnUqFFVbuPChQtKT0/X7NmzJUkTJ05Ujx499Pbbb/uGEtu3b9fSpUu1atUqjRw50nfswIEDdffdd2v16tUaOXLkdeuRpHXr1ikmJkYbNmyQy+Xyq18AAAAA9QOXbwIAAAAsNHHiRN+/169fL0maNm1apX2efPJJSeUn4S/XqlUr3Xfffb7vY2JiNGbMGOXk5CgvL8+vOmJjYyVJH374oTwej1/HXu7yfiSpf//+ys3N9X2/evVqNWvWTHfddZdOnz7t++rZs6eaNm2qLVu21Lie2NhYFRUVadOmTQHXCwAAACC0GEoAAAAAFrrpppt8/z5x4oScTqc6dOhQaZ+kpCTFxsbqxIkTlbZ36NChynoRHTt2lKRKa0XUxIABA/TLX/5Sc+fOVYsWLTR06FAtX778qmtZVCcyMtK37kSFuLi4SmtFHDlyRD/88IMSEhIUHx9f6evcuXM6depUjeuZPHmyOnbsqJ/97Gdq3bq1HnnkEWVnZ/vVNwAAAIDQ4vJNAAAAgIWaNGlSZVt1C1MH6mq3deXi0w6HQ++//752796tv/3tb9qwYYMeeeQRvfzyy9q9e7eaNm163fuqySWUvF6vEhIStGrVqmp/XjHUqEk9CQkJOnDggDZs2KCPPvpIH330kZYvX64xY8Zo5cqV160FAAAAQOjxSQkAAAAgRNq2bSuv16sjR45U2p6fn6+zZ8+qbdu2lbYfPXpUxphK2w4fPixJateunaTyTypI0tmzZyvtd+WnLircfvvtev7557Vv3z6tWrVKX3zxhbKysiTVzbAkOTlZ33//vfr166dBgwZV+erevXuN65Ekt9ute++9V4sXL9bXX3+tCRMm6J133tHRo0drXSsAAACA4GMoAQAAAITIkCFDJEmvvvpqpe2vvPKKJOmee+6ptP27777T2rVrfd8XFBTonXfeUUpKipKSkiSVDwEkadu2bb79ioqKqnyS4MyZM1UGHCkpKZLku2RSVFSUpKoDDn888MADKisr07x586r8rLS01HfbNann+++/r/Rzp9Opbt26VdoHAAAAQP3G5ZsAAACAEOnevbsyMjL01ltv6ezZsxowYID27NmjlStXatiwYRo4cGCl/Tt27Khx48Zp7969SkxM1LJly5Sfn6/ly5f79hk8eLBuvPFGjRs3TtOnT5fL5dKyZcsUHx+vkydP+vZbuXKlFi9erPvuu0/JyckqLCzUkiVLFBMT4xuWNGnSRLfeeqvee+89dezYUc2bN1eXLl3UpUuXGvc4YMAATZgwQfPnz9eBAwc0ePBghYeH68iRI1q9erVee+01DR8+vEb1jB8/Xv/5z3905513qnXr1jpx4oQWLlyolJQUde7cuTa/CgAAAAAWYSgBAAAAhNDSpUvVvn17rVixQmvXrlVSUpKeeeYZZWZmVtn35ptv1sKFCzV9+nR99dVXuummm/Tee+8pLS3Nt094eLjWrl2ryZMna/bs2UpKStJvfvMbxcXFaezYsb79KgYgWVlZys/PV7NmzdS7d2+tWrWq0mLcS5cu1a9+9Ss98cQTKikpUWZmpl9DCUl688031bNnT/3xj3/UzJkzFRYWpnbt2mnUqFHq169fjesZNWqU3nrrLS1evFhnz55VUlKS0tPTNWfOHDmdfAgcAAAAaAgc5srPSAMAAAAAAAAAAAQBbycCAAAAAAAAAACWYCgBAAAAAAAAAAAswVACAAAAAAAAAABYgqEEAAAAAAAAAACwBEMJAAAAAAAAAABgCYYSAAAAAAAAAADAEgwlAAAAAAAAAACAJRhKAAAAAAAAAAAASzCUAAAAAAAAAAAAlmAoAQAAAAAAAAAALMFQAgAAAAAAAAAAWIKhBAAAAAAAAAAAsARDCQAAAAAAAAAAYIn/DyyW7XUrDNiTAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_changes(r_economy_k, param_name='lambda', height=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Extremely interesting behaviour is registered for EconomyK model: its confidence rockets with the earliness drop resulting in acceptance of almost all the predictions, however, the resulting accuracy is not as large as it is expected for such level of confidence." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ECEC" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The definition of a common confidence threshold for different classifiers lacks of logic since the ratio of available classes and features changes along the time series length and effect the classifiers' performances. ECEC model aims to eliminate this drawback evaluating the confidence thresholds separately and automatically. \n", + "\n", + "Obtained values are stored in *confidence_thresholds* attribute after training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot_ind.core.models.early_tc import ecec as ECEC\n", + "reload(ECEC)\n", + "accuracy_importance = [0, 0.1, 0.2, 1]\n", + "r_ecec = eval_param_influence(ECEC.ECEC,\n", + " 'accuracy_importance', accuracy_importance,\n", + " prediction_mode='all', interval_percentage=INTEVAL_PERCENTAGE)" + ] + }, + { + "cell_type": "code", + "execution_count": 202, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABiUAAAHgCAYAAADKeW7zAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAC3kklEQVR4nOzdd3yV5f3/8dc5J8k52QmEEAgQwgh7Q9hDZSOtdVsriFvrrq0Fq+DP7bdqVXC0VbTiqmjVihNFhoS99wwbkpBN9jn37480kZhA1tl5Px+PPELu+z73/bnuk3O/c3Hdw2QYhoGIiIiIiIiIiIiIiIiLmT1dgIiIiIiIiIiIiIiINA0alBAREREREREREREREbfQoISIiIiIiIiIiIiIiLiFBiVERERERERERERERMQtNCghIiIiIiIiIiIiIiJuoUEJERERERERERERERFxCw1KiIiIiIiIiIiIiIiIW2hQQkRERERERERERERE3EKDEiIiIiIiIiIiIiIi4hYalBBpgNTUVEwmE2+99ZanSxERERfS8V5EREB5ICIidaO8EKkbDUqINFGvvPKKQvI8Vq5cyYgRIwgJCSEuLo67776b/Px8T5clIlJvOt6fX2OO96+++ipXXHEF7dq1w2Qycf3117u2WBGRRlAenF9D8+DIkSM8+uijJCcnEx0dTUxMDGPGjGHx4sVuqFpExPmUF+f27bffcuONN9KzZ08sFgvt27f3dEk+y2QYhuHpIkR8jWEYFBcXExgYiMVi8XQ5DdKzZ09iYmL48ccfPV2K19m0aRNDhw6lW7du3HLLLRw9epS//vWvXHDBBXz11VeeLk9E3EjHe//W2ON9+/btycvLIzk5mcWLF3PttdeqAyfip5QH/q0xeTB37lz+9Kc/cckllzB8+HDKysr417/+xYYNG3jzzTeZMWOGm1ohIt5AeeHfrr/+ej788EP69+/P4cOHsVgspKamerosnxTg6QJEzuXMmTOEhoZ6uowamUwmbDabp8tokIKCAkJCQjxdhlebNWsW0dHR/Pjjj0RERADl//F088038+233zJ+/HgPVyjiX3S8dw0d72vX2OP90qVLK6+SCAsLc0fJIn5NeeAayoPaNSYPLrjgAg4fPkxMTEzltNtuu42+ffvyyCOPaFBCxAWUF66hvKjdk08+yT/+8Q8CAwO5+OKL2bZtm6dL8lm6fVMTcujQIe644w66dOlCcHAwzZs354orrqhxRC87O5v77ruP9u3bY7VaadOmDdOmTSMjI6NymaKiIubMmUNSUhI2m41WrVpx6aWXsn//fgB+/PFHTCZTtZHVmu6vd/311xMWFsb+/fuZPHky4eHhXHvttQAsX7688tYIVquVtm3bct9991FYWFit7l27dnHllVfSokULgoOD6dKlCw899BAAS5YswWQy8Z///Kfa69577z1MJhMpKSl12pfna8Phw4e5+OKLCQsLIz4+nnnz5gGwdetWLrzwQkJDQ0lISOC9996rss633noLk8nEsmXLuPXWW2nevDkRERFMmzaNrKysajW88sor9OjRA6vVSuvWrfn9739PdnZ2lWXGjBlDz549Wb9+PaNGjSIkJIRZs2bRvn17tm/fztKlSzGZTJhMJsaMGQNAZmYmDzzwAL169SIsLIyIiAgmTZrE5s2bq6y74v3997//zRNPPEGbNm2w2WxcdNFF7Nu3r1q9q1evZvLkyURHRxMaGkrv3r158cUXqyyza9cuLr/8cpo1a4bNZmPgwIF8/vnndXpPnCU3N5fvvvuO3/3ud5UdEoBp06YRFhbGv//9b7fWI9IQOt7reK/jfe2ccbxPSEjAZDK5skyRRlEeKA+UB7VrbB706NGjyoAEgNVqZfLkyRw9epS8vDyX1C3iTMoL5YXyom5at25NYGCg27frj3SlRBOydu1aVq5cydVXX02bNm1ITU3l1VdfZcyYMezYsaNyNDQ/P5+RI0eyc+dObrjhBvr3709GRgaff/45R48eJSYmBrvdzsUXX8z333/P1VdfzT333ENeXh7fffcd27Zto2PHjvWur6ysjAkTJjBixAj++te/Vtbz0UcfUVBQwO23307z5s1Zs2YNL7/8MkePHuWjjz6qfP2WLVsYOXIkgYGB3HLLLbRv3579+/fz3//+lyeeeIIxY8bQtm1b3n33XX7zm99U2fa7775Lx44dGTp0aCP2MNjtdiZNmsSoUaN49tlneffdd7nzzjsJDQ3loYce4tprr+XSSy/ltddeY9q0aQwdOpTExMQq67jzzjuJiopizpw57N69m1dffZVDhw5VHtQB5syZw6OPPsrYsWO5/fbbK5dbu3YtP/30U5UD5OnTp5k0aRJXX301v/vd72jZsiVjxozhrrvuIiwsrDKEW7ZsCcCBAwf49NNPueKKK0hMTOTUqVO8/vrrjB49mh07dtC6desq9T799NOYzWYeeOABcnJyePbZZ7n22mtZvXp15TLfffcdF198Ma1ateKee+4hLi6OnTt38sUXX3DPPfcAsH37doYPH058fDx//vOfCQ0N5d///jeXXHIJH3/8cbX37JeysrKw2+21vkchISHnHfnfunUrZWVlDBw4sMr0oKAg+vbty8aNG2vdhoin6Xiv472O9zrei4DyQHmgPADP5cHJkydr3baIt1BeKC+UF7XnhTiZIU1GQUFBtWkpKSkGYPzrX/+qnPbII48YgPHJJ59UW97hcBiGYRhvvvmmARjPP//8OZdZsmSJARhLliypMv/gwYMGYMyfP79y2vTp0w3A+POf/1ynup966inDZDIZhw4dqpw2atQoIzw8vMq0s+sxDMOYOXOmYbVajezs7MppaWlpRkBAgDF79uxq2zmX87XhySefrJyWlZVlBAcHGyaTyfjggw8qp+/atcsAqmxz/vz5BmAMGDDAKCkpqZz+7LPPGoDx2WefVdYbFBRkjB8/3rDb7ZXLzZ071wCMN998s3La6NGjDcB47bXXqrWhR48exujRo6tNLyoqqrLeivZarVbj//2//1c5reL97datm1FcXFw5/cUXXzQAY+vWrYZhGEZZWZmRmJhoJCQkGFlZWVXWe/Z7c9FFFxm9evUyioqKqswfNmyY0blz52p1/lJCQoIB1PpV2/v80UcfGYCxbNmyavOuuOIKIy4urtZaRDxNx3sd78+m433NnH28Dw0NNaZPn16v14i4mvJAeXA25UHNXPH3/969ew2bzWZcd9119X6tiCcoL5QXZ1Ne1M2UKVOMhISEer1GfqbbNzUhwcHBlf8uLS3l9OnTdOrUiaioKDZs2FA57+OPP6ZPnz41jjRWjLx+/PHHxMTEcNddd51zmYa4/fbbz1v3mTNnyMjIYNiwYRiGUXnWSnp6OsuWLeOGG26gXbt256xn2rRpFBcXs3DhwsppH374IWVlZfzud79rcN1nu+mmmyr/HRUVRZcuXQgNDeXKK6+snN6lSxeioqI4cOBAtdffcsstVUaub7/9dgICAvjyyy8BWLx4MSUlJdx7772YzT9/hG+++WYiIiJYtGhRlfVZrdZ63cfUarVWrtdut3P69GnCwsLo0qVLld+TCjNmzCAoKKjy55EjRwJUtm3jxo0cPHiQe++9l6ioqCqvrXhvMjMz+eGHH7jyyivJy8sjIyODjIwMTp8+zYQJE9i7dy/Hjh07b93vvvsu3333Xa1f06ZNO+96Ki7ztFqt1ebZbLYaLwMV8TY63ut4Xxc63ut4L/5PeaA8qAvlgXPzoKCggCuuuILg4GCefvrper1WxFOUF8qLumjqeSHOpds3NSGFhYU89dRTzJ8/n2PHjmEYRuW8nJycyn/v37+fyy677Lzr2r9/P126dCEgwHm/QgEBAbRp06ba9MOHD/PII4/w+eefV7tXXkXdFQe0nj17nncbXbt2ZdCgQbz77rvceOONQPnBaciQIXTq1KnRbbDZbLRo0aLKtMjISNq0aVMtfCMjI2u891/nzp2r/BwWFkarVq0q7+V46NAhoDyozhYUFESHDh0q51eIj4+vEgK1cTgcvPjii7zyyiscPHiwyiVuzZs3r7b8L0M9OjoaoLJtFfeMPN97s2/fPgzD4OGHH+bhhx+ucZm0tDTi4+PPuY7hw4efc159VPxRU1xcXG1eUVFRlT96RLyVjvc63teFjvc63ov/Ux4oD+pCeeC8PLDb7Vx99dXs2LGDr776qtqtTES8lfJCeVEXTT0vxLk0KNGE3HXXXcyfP597772XoUOHEhkZiclk4uqrr8bhcDh9e+caAT/XfdzOHnE9e9lx48aRmZnJgw8+SNeuXQkNDeXYsWNcf/31Dap72rRp3HPPPRw9epTi4mJWrVrF3Llz672emlgslnpNPzvoXaW+/6ny5JNP8vDDD3PDDTfw2GOP0axZM8xmM/fee2+N+9sZbatY7wMPPMCECRNqXKa2PwLS09PrdI/AsLAwwsLCzjm/VatWAJw4caLavBMnTqhjIT5Bx/tyOt6fn473Ot6L/1MelFMenJ/ywHl5cPPNN/PFF1/w7rvvcuGFF9b5dSKeprwop7w4v6aeF+JcGpRoQhYuXMj06dN57rnnKqcVFRWRnZ1dZbmOHTuybdu2866rY8eOrF69mtLS0nM+db5iBPSX6//lyOz5bN26lT179vD2229XuYzqu+++q7Jchw4dAGqtG+Dqq6/m/vvv5/3336ewsJDAwECuuuqqOtfkanv37uWCCy6o/Dk/P58TJ04wefJkABISEgDYvXt3ZbsBSkpKOHjwIGPHjq3Tds71R8DChQu54IILeOONN6pMz87OJiYmpl5tASofYrVt27Zz1lbRjsDAwDrX/0uDBg2q0+/W7NmzmTNnzjnn9+zZk4CAANatW1flEsqSkhI2bdpUZZqIt9LxvpyO9+V0vK+ZjvfSFCgPyikPyikPauasPPjjH//I/Pnz+dvf/sY111xTp9eIeAvlRTnlRTnlhbiDninRhFgslmqjkS+//HK10cLLLruMzZs385///KfaOipef9lll5GRkVHjiHHFMgkJCVgsFpYtW1Zl/iuvvFKvms9eZ8W/X3zxxSrLtWjRglGjRvHmm29y+PDhGuupEBMTw6RJk1iwYAHvvvsuEydObNDB01X+/ve/U1paWvnzq6++SllZGZMmTQJg7NixBAUF8dJLL1Vp2xtvvEFOTg5Tpkyp03ZCQ0Or/QEANf+efPTRR7Xeo+9c+vfvT2JiIn/729+qba9iO7GxsYwZM4bXX3+9xjOU0tPTa92Os+4RGBkZydixY1mwYAF5eXmV09955x3y8/O54oor6tBqEc/S8b6cjvfldLyvWX2O9wUFBezatYuMjIxa6xPxJsqDcsqDcsqDmjkjD/7v//6Pv/71r8yaNYt77rmn1tpFvI3yopzyopzyQtxBV0o0IRdffDHvvPMOkZGRdO/enZSUFBYvXlztvm9//OMfWbhwIVdccQU33HADAwYMIDMzk88//5zXXnuNPn36MG3aNP71r39x//33s2bNGkaOHMmZM2dYvHgxd9xxB7/+9a+JjIzkiiuu4OWXX8ZkMtGxY0e++OIL0tLS6lxz165d6dixIw888ADHjh0jIiKCjz/+uMZ767300kuMGDGC/v37c8stt5CYmEhqaiqLFi1i06ZNVZadNm0al19+OQCPPfZY/XemC5WUlHDRRRdx5ZVXsnv3bl555RVGjBjBr371K6A8UGfOnMmjjz7KxIkT+dWvflW53KBBg+r8AKYBAwbw6quv8vjjj9OpUydiY2O58MILufjii/l//+//MWPGDIYNG8bWrVt59913q4yy14fZbObVV19l6tSp9O3blxkzZtCqVSt27drF9u3b+eabbwCYN28eI0aMoFevXtx888106NCBU6dOkZKSwtGjR9m8efN5t+PMewQ+8cQTDBs2jNGjR3PLLbdw9OhRnnvuOcaPH8/EiROdth0RV9Hx/mc63ut4fz51Pd6vWbOGCy64oNrZU//9738r6y0tLWXLli08/vjjAPzqV7+id+/eTqtVpCGUBz9THigPzqcxefCf//yHP/3pT3Tu3Jlu3bqxYMGCKuseN24cLVu2dFqtIq6gvPiZ8kJ5cT5btmzh888/B8qfd5GTk1P593+fPn2YOnWq07bl9wxpMrKysowZM2YYMTExRlhYmDFhwgRj165dRkJCgjF9+vQqy54+fdq48847jfj4eCMoKMho06aNMX36dCMjI6NymYKCAuOhhx4yEhMTjcDAQCMuLs64/PLLjf3791cuk56eblx22WVGSEiIER0dbdx6663Gtm3bDMCYP39+5XLTp083QkNDa6x7x44dxtixY42wsDAjJibGuPnmm43NmzdXW4dhGMa2bduM3/zmN0ZUVJRhs9mMLl26GA8//HC1dRYXFxvR0dFGZGSkUVhYWO99efDgwTq3YfTo0UaPHj2qTU9ISDCmTJlS+fP8+fMNwFi6dKlxyy23GNHR0UZYWJhx7bXXGqdPn672+rlz5xpdu3Y1AgMDjZYtWxq33367kZWVVadtG4ZhnDx50pgyZYoRHh5uAMbo0aMNwzCMoqIi4w9/+IPRqlUrIzg42Bg+fLiRkpJijB49unIZwzCMJUuWGIDx0Ucf1bpvDMMwVqxYYYwbN84IDw83QkNDjd69exsvv/xylWX2799vTJs2zYiLizMCAwON+Ph44+KLLzYWLlxYYxtcafny5cawYcMMm81mtGjRwvj9739v5Obmur0OkYbQ8f5nOt7reF+buhzvK/bB7Nmzq0yfPn26AdT49cv9IuIJyoOfKQ+UB7VpaB7Mnj37nFkAGEuWLHFvQ0QaQHnxM+WF8uJ8Kt6Lmr5++VmR8zMZhhuenCLiZcrKymjdujVTp06tdi88T3nrrbeYMWMGa9euZeDAgZ4uR0TEL+h4LyIioDwQEZG6UV6IuIeeKSFN0qeffkp6erruFyci4ud0vBcREVAeiIhI3SgvRNxDz5SQJmX16tVs2bKFxx57jH79+jF69Ogq80tKSsjMzDzvOiIjIwkODnZlmSIi0kg63ouICCgPRESkbpQXIu6lQQlpUl599VUWLFhA3759eeutt6rNX7lyJRdccMF51zF//nyuv/561xQoIiJOoeO9iIiA8kBEROpGeSHiXnqmhMhZsrKyWL9+/XmX6dGjB61atXJTRSIi4go63ouICCgPRESkbpQXIs6lQQkREREREREREREREXELPehaRERERERERERERETcokk8U8LhcHD8+HHCw8Ox2+0UFxd7uiRpgoKDgzGbNQ4ovs8wDPLy8mjdurXP/U6fnQcmk8nT5YiI+DTlgYiIgPJARETK1ScPmsSgxPHjx0lISODll19m0KBBWCwWT5ckTVBhYSE33XQTu3bt8nQpIk5x5MgR2rRp4+ky6uX48eO0bdvW02WIiPgV5YGIiIDyQEREytUlD5rEoER4eDgvv/wyI0aMIC4ujpCQkHqNgNvt9iY5kKF2O4/D4eDYsWN8+OGHtGvXzuvOHiktLeXbb79l/PjxBAYGeroct1G7G9bu3Nxc2rZtS3h4uAuqc62Kmo8cOUJERES9XqvfF7W7KVC71e76UB7o96UpULvV7qZAeaA8qA+1W+1uCtRu1+dBkxiUKCsrY9CgQcTFxREbG1uv1xqGUfmf1E3pUj612/ntjo2N5ciRI1itVoKDg5267sYqLS0lJCSEiIiIJnewVbsb3m5fPDZU1BwREdGgTod+X9Ruf6d2q90NoTxoGtRutbspULuVB/Wh3xe1uylQu9XuhqhLHnjX6douUlJSgsViISQkxNOlSBNmtVoxmUyUlJR4uhQRERERERERERERj2gSgxIVfHHUXvyHfv9ERERERERERESkqWtSgxIiIiIiIiIiIiIiIuI5GpRoAuLj43nssccqfzaZTCxYsMCDFYmIiIiIiIiIiIhIU6RBiSbo8OHDXHbZZZ4uo85mz55Ns2bNaNasGXPmzKkyb8mSJfTo0YPS0lLPFCciIiIiIiIiIiIidRbg6QLEdYqKirDZbNWmt23b1gPVNMzq1at55pln+OijjzAMgyuvvJLJkyeTnJxMaWkpd9xxB6+//nqjnggvIiIiIiIiIiIiIu6hKyW8hN1uZ9asWcTHx2Oz2ejSpQtvvfVW5fyysjKuuuqqyvmJiYk8/vjjVdZx+eWXM27cOP785z8TGxtLx44da9zW2bdv2r17NyaTiX/9618MHjy4cts//PBDldd8++23DBw4EJvNRlxcHDNmzCA3N7dy/jPPPENCQgJWq5XmzZszceLEynlvvfUWSUlJ2Gw2oqKiGDZsWJXXns/27dvp0qULU6dO5Ve/+hVJSUls374dKL+CYujQoYwaNapO6xIRERERERERERERz9KVEl7ioYce4t///jcvv/wy3bp1Y/Hixdx66620bNmSSZMmYbfbad26NR988AEtWrTgxx9/5L777qN169bccMMNletZuXIl4eHhfPXVV/Xa/pw5c3jqqafo3r07f/rTn5g2bRr79u3DYrGwY8cOLrnkEmbOnMlbb73FyZMnufvuu7nhhhtYuHAhy5cv56GHHuKVV15hzJgxZGRksGTJEgAOHTrETTfdxCOPPMJVV11FTk4OS5YswTCMOtXVr18/UlNT2bt3L4ZhkJqaSt++fdmxYwfvvfcemzZtqlc7RURERERERERERMRzNCjhBQoLC3nxxRf54osvuOiiiwDo1q0bK1as4NVXX2XSpElYrVZeeOGFytd07dqVlJQU/v3vf1cZlAgODua9996r8bZN53P33Xdz1VVXAfDEE08wYMAAdu7cSb9+/Xj00Ue55JJLePjhhwHo2bMnf/vb35g0aRIFBQUcPHiQ4OBgrrzySqKiokhKSmLYsGEAHD16FLvdztVXX01SUhIAycnJda6rX79+/OUvf2H8+PEAPPzww/Tr149hw4bx+OOP8+mnn/LEE08QEBDACy+8UOUKDRERERERERERERHxLhqU8AI7duygqKiIqVOnVpleWlpKt27dKn9++umneeeddzh+/DjFxcXV5gN06dKl3gMSAP3796/8d8UzJ06cOEG/fv3Yvn07e/bs4dNPP61cxjAMHA4Hu3fv5le/+hWPP/44iYmJjBkzhgkTJnDttdcSHh7O4MGDGTp0KP3792fkyJGMGzeO6667jhYtWtS5tj/+8Y/88Y9/rPx57ty5hIWFMWbMGLp160ZKSgqpqalMmzaNQ4cOERwcXO/2i4g4U/6+NI4uXE/WukNggmbJibS5fAChiTGeLk1ERNxIeSAiIqA8EBH5Jbc/U2LZsmVMnTqV1q1bYzKZqvxH97n8+OOP9O/fH6vVSqdOnao8a8EfVDxf4eOPP2bt2rWVX5s2beI///kPAP/85z+ZM2cO06ZNY9GiRaxdu5YrrriCkpKSKusKCQlpUA1nPyjaZDIBVN5iqaCggGuvvbZKbevWrWPbtm1069aNqKgotm/fzltvvUVcXBxPPPEEPXv2JCMjg4CAAFasWMF//vMfunbtymuvvUaXLl3YtWtXg+o8ceIETz/9NK+99hrLli0jMTGRnj17cvHFF1NaWsrWrVsbtF4RcT9/zYPjX2xhzXVvcOK/Wyg8mkXhkSyOf7qR1df+k5PfbPd0eSIiXkd5ICIioDwQEWlK3D4ocebMGfr06cO8efPqtPzBgweZMmUKF1xwAZs2beLee+/lpptu4ptvvnFxpe7Tr18/goKCSE1NpUePHlW+Kh5WvWLFCvr168eDDz7IsGHD6NGjB6mpqW6pr1evXuzevbtabT169Ki8KiMwMJBf//rXvPrqq2zfvp1jx47x5ZdfAmA2mxk3bhwvvPACO3bsIDAwkA8++KBBtdxxxx3cfvvtdOjQAbvdTmlpaeU8u92O3W5vfINFxC38MQ/y96ax64lFYIBhd1RON+wGOAx2PPpfzqSe9mCFIiLeR3kgIiKgPBARaUrcfvumSZMmMWnSpDov/9prr5GYmMhzzz0H/PyshRdeeIEJEya4qky3ioqK4rbbbmPWrFnY7XYuvPBCsrKy+PHHH4mMjOTOO++kc+fOfPzxx3zyySd07tyZN954g61btxIfH+/y+mbNmsXo0aOZPn06t912G+Hh4WzZsoVvvvmGt99+mw8++ID9+/dz4YUX0rx5cz799FMMw6BHjx4sWbKEb7/9lsmTJxMXF8eKFSvIysqiZ8+e9a7j008/Zf/+/SxcuBCA4cOHc/DgQRYuXMihQ4ewWCz07t3b2c0XERfxxzw4unA9JrOpvJNRA5MJjn28nqQ/jHdzZSIi3kt5ICIioDwQEWlKvP6ZEikpKYwdO7bKtAkTJnDvvfee8zXFxcUUFxdX/lxQUFD574pbEjVEY15bmxdeeIEWLVrw3HPPcd999xEeHk6PHj146KGHMAyD+++/n40bNzJ9+nRMJhO//vWvmT59OosXL65W17nqrG25c70uOTmZb775hoceeoixY8diGAZt27bl0ksvxTAMmjVrxvPPP8+zzz5LcXExCQkJ/OMf/6B///5s2rSJFStW8Prrr3PmzBlat27NnDlzuOyyyzAMgy+//JKLL76YXbt2VT4IuyZnzpzhvvvu47333sNsNmMYBomJiTz11FPcdtttBAUF8dprrxESEuK098nZ73fF+srKyqpc4eENKurxtrpcTe1uWLs9tb+ckQcVt8srLS2tdzvqst8yNqbisJjAYsIwDFZm7gADhjTvisVkwQAyNh0i0Yd+5/Q5UbubArVbeVAfygPfqdkZ1G61uylQHigP6kOfE7W7KVC7XZ8HJsOV/9Ne28ZNJv7zn/9wySWXnHOZpKQkZsyYwcyZMyunffnll0yZMoWCgoIaH2o8Z84cHn300cqfhw0bxssvv0ynTp0a/MwFcY158+bx/PPPs3PnToKCgjxdjksVFhZy4MABTpw4UeWPIBFfU1BQwG9/+1tycnKIiIhwyjrdlQcV3nvvPZfnQUlJCUuWLAGgffv2dOnSxaXbExFxN+VB3SgPRMTfKQ/qRnkgIv6uPnng9VdKNMTMmTO5//77K39OT08nJycHi8WCxWKp9/rsdnuDXufr3NHur7/+mjlz5tT4x4KnuKrdZrMZk8nEoEGDnPaHmrOUlpby3XffMW7cuCoPPfd3anfD2l1xNpEv+GUe5Obm0rZtW8aPH1/vz2Fd9tueF77j1FfbMOwGhfZirGknATjJKTrF5BNpC6P1r/rS6c4LGt4oN9PnRO1uCtRu5UF9KA/0OWkK1G61uz6UB8qDpkDtVrubAnfmgdcPSsTFxXHq1Kkq006dOkVERMQ5/yPbarVitVorfz5z5gw5OTlA+Wh7fZx9IUl9X+vL3NXur776ymXrbghXtrtifQEBAV57QAsMDPTa2lxJ7a7/6zzBGXlQoTHv+flem3DpQE59ugWTw8BRVoap8t6xBltO72dE8x60u3SAT/6+6XPStKjdTYvyQHlQH/qcNC1qd9OiPFAe1Ic+J02L2t20uCMPzPVeu5sNHTqU77//vsq07777jqFDh3qoIhER8QRfyIOwTrF0e2gymE04zOUdjgCThSBLALn2QszTuxKa0NzDVYqI+DblgYiIgPJARMSXuf1Kifz8fPbt21f588GDB9m0aRPNmjWjXbt2zJw5k2PHjvGvf/0LgNtuu425c+fypz/9iRtuuIEffviBf//73yxatMjdpYuIiBP5ax60mtKb8G6t2PTWYgIXHSQs0Mag0UPZG5XDwcAscnNzve4WbiIinqQ8EBERUB6IiDQlbr9SYt26dfTr149+/foBcP/999OvXz8eeeQRAE6cOMHhw4crl09MTGTRokV899139OnTh+eee45//vOfTJgwwd2li4iIE/lzHoR1aEHiraPoeOsouj0wkYnP3EhiryTsdjvLly/3dHkiIl5FeSAiIqA8EBFpStx+pcSYMWOq3Lf/l956660aX7Nx40YXViUiIu7m73lQVlYGlD9HBmDUqFF89NFHHDt2jD179pCUlOTJ8kREvIbyQHkgIgLKA+WBiDQlXv9MCREREV/0y05HREQEAwcOBCAlJYXCwkKP1SYiIu6jPBAREVAeiIicTYMSIiIiLvDLTgdAr169aN68OcXFxaxcudJTpYmIiBspD0REBJQHIiJn06CEiIiIC9TU6TCbzYwePRqTycT+/fur3BNXRET8k/JARERAeSAicjYNSoiIiLhATZ0OgJiYGHr16gXAihUrKC0tdXttIiLiPsoDEREB5YGIyNk0KFEPdruD7dvS+Gn5YbZvS8Nud7h8m19//TUXXnghsbGxmEwmFixYUG2Zp59+mvj4eKxWK71792bp0qUur8tZHHY7RYeOUbBjL0WHjuGw292y3YMHD3LJJZcQFRWFzWYjKSmJ5cuXV85/5plnfHafioh3sP/vePbLTgfAwIEDCQ8PJz8/n7Vr17q7NBERcSPlgYiIgPJARORsGpSoozWrjnH377/mvru/Zdaff+C+u7/l7t9/zepVR1263fz8fHr16sVzzz1X4/w33niDRx55hAcffJCUlBR69OjB1KlTOXbsmEvrcoaCPQc49Y/3OfXmh6Qt+A+n3vyQU/94n4I9B1y63fT0dEaMGEFAQACfffYZmzZt4plnnqF58+YAzJ8/32f3qYh4j3OdCVUxbeTIkQBs27aNtLQ0t9YmIiLuozwQERFQHoiInE2DEnWwds1xnnx8BVu3nCIqykpiYhRRUVa2bjnFk4+tcOnAxOWXX86LL77IddddV+P8F198kWuuuYa7776b/v37s2DBAmw2G/PmzXNZTc5QsOcAp/+9iOJDx7CEhhDYsjmW0BCKDx3j9L8XuXRgYs6cObRq1YqFCxcyevRounbtym9+8xu6d+8OwMsvv+yT+1REvEtFp8NisdQ4v02bNnTu3BmAZcuW4XC4/uo7ERFxP+WBiIiA8kBE5GwalKiFw2HwzttbycoqpFOnZoRHWLEEmAmPsNKpUzOysgp5560tbrmV0y8VFRWxY8cOxo0bVznNYrEwcuRI1qxZ4/Z66spht5Pzw0rsZwoIbNUCc7AVk9mMOdhKYKsW2M8UkLMkxWW3cvr666/p27cvkyZNolmzZnTr1o3nn38egOLiYp/cpyLifc53JlSFoUOHYrPZyMzMZNOmTW6qTERE3El5ICIioDwQETmbBiVqsWtnOvv2ZtIyLgyT2VRlnslsIi4ujD17Mtm1M8PttZ08eRK73U6rVq2qTI+NjfXqS/1Kjp6k5EQaAdGRmEy/2KcmEwHRkZQcP0XJ0ZMu2f7Ro0d555136NixI1988QU33XQTs2bNYu7cuT67T0XE+9Sl02Gz2Rg2bBgAGzZsIDs72x2liYiIGykPREQElAciImfToEQtsrOKKS62ExJSc2gEhwRSXFxGdlaRmyvzXY4zBRilZZisgTXONwUFYpSW4ThT4JrtOxx0796duXPnMmzYMP7whz9wzTXX8M9//tMl2xORpqkunQ6ATp060bZtWxwOB8uWLcMwDHeUJyIibqI8EBERUB6IiJxNgxK1iIq2YrVaKCgoq3F+YUEpVmsAUdE2N1cGcXFxWCwWTpw4UWV6WloasbGxbq+nrsyhIZgCAzCKS2ucb5SUYgoMwBwa4pLtt2jRgqSkpCrTunXrxvHjx312n4qI96lrpwNgxIgRBAQEcPLkSXbt2uXq0kRExI2UByIiAsoDEZGzaVCiFl27taBT52acOpmP4ag6Om04DE6ezCcpqRldu8W4vTabzUb37t1ZvHhx5TS73c6KFStITk52ez11FdQmjqBWsZRl5VQb8TcMg7KsHIJatySoTZxLtj9w4ED2799fZdqePXuIj4/HarX65D4VEe9Tn05HeHg4gwYNAmD16tUUFLjmSjEREXE/5YGIiIDyQETkbBqUqIXZbOK66b2Ijg5m375M8nKLKStzkJdbzL59mURHB3Pd9b2xWFyzK3NyckhJSSElJQWAAwcOkJKSwt69ewG45557eP/995k7dy4bN27kuuuuo7CwkDvuuMMl9TiD2WIh8sJhWEJDKD2RjqOwGMPuwFFYTOmJdCxhoUReMBSzxeKS7T/wwANs2rSJmTNnsn37dl5//XXeffddbr31VgDuuusun9unIuJ9Kjodljoey3r27EmLFi0oKSnhp59+cmVpIiLiRsoDEREB5YGIyNk0KFEHg5JbM+svI+jVuyXZ2cWkHswmO7uY3r1bMuvhEQwe0sZl2/7pp58YNmxY5YOOZs+ezbBhw5g5cyYAN954I3PmzOHJJ59k8ODBbNu2jc8++4w2bVxXkzOEJHWg+ZVTsCbEYz9TQGnaaexnCrC2b0PzKyYTktTBZdseNWoUCxYs4OOPP6Z///4888wzPPHEE9x2220AzJgxg9mzZ/vcPhUR72K324G6nQkFYDKZGDVqFGazmYMHD5KamurC6kRExF2UByIiAsoDEZGz1e1IKCQPiWdgcmt27cwgO6uIqGgbXbvFuOwKiQqTJ0+u9aFGM2fOrByk8CUhSR2wdUyg5OhJHGcKMIeGENQmzmVXSJzt6quv5uqrrz7n/JkzZzJr1iyX1yEi/qs+l2dXaN68Ob1792bTpk2sWLGC1q1bExQU5KoSRUTEDZQHIiICygMRkbPpSol6sFjM9OgZy/CR7ejRM9blAxJNgdliwZYQT0j3ztgS4t0yICEi4g4N6XQA9O/fn8jISAoKClizZo0rShMRETdSHoiICCgPRETOpv9VFxERcYGGdjoCAgIYOXIkADt27ODkyZNOr01ERNxHeSAiIqA8EBE5mwYlREREXKChnQ6A1q1b07VrVwCWLVtWef9ZERHxPcoDEREB5YGIyNk0KCEiIuJkDoej8nlADel0AAwePJjg4GCys7PZuHGjM8sTERE3UR6IiAgoD0REfkmDEiIiIk5WcRYUNLzTYbVaGT58OACbNm0iMzPTKbWJiIj7KA9ERASUByIiv6RBCRERESer6HSYTCbM5oZHbYcOHUhISMDhcLBs2bLKs6tERMQ3KA9ERASUByIiv6RBCRERESdrzP1if2nEiBEEBgaSlpbGjh07Gr0+ERFxH+WBiIiA8kBE5Jc0KCEiIuJkzux0hIaGMnjwYADWrFlDfn5+o9cpIiLuoTwQERFQHoiI/JIGJURERJzMmZ0OgG7dutGyZUtKS0tZsWKFU9YpIiKupzwQERFQHoiI/JIGJURERJzM2Z0Ok8nEqFGjMJvNHD58mP379ztlvU1daZGdde+kMv/SFbw+4Uc+f2AT6XvyPF2WiPgR5YFvUB6IiKspD3yD8kDEfTQoUQ92u4ODWzPZuuwkB7dmYrc7XL7NWbNm0bNnT0JDQ2nWrBnjxo1jy5YtVZZ5+umniY+Px2q10rt3b5YuXeryupzFYbdTduIQpQe3U3biEA673eXbjI+Px2QyVfuaNm1a5TLPPPOMz+5TEfG8ik6HxWJx2jqjo6Pp168fACtXrqS4uNhp626KMlPP8GyPr3h/+mq2f36MPd+dYvmLe3i625cs+esuT5cnIn5CeeD9lAci4g7KA++nPBBxLw1K1NHOlDRevG0lL/1+Fa//cS0v/X4VL962kh0paS7d7vLly7n11ltZtmwZX375JaWlpUycOJHc3FwA3njjDR555BEefPBBUlJS6NGjB1OnTuXYsWMurcsZSg/tpuDT1yn4/B8UfPmv8u+fvk7pod0u3e7atWs5fPhw5denn34KwNVXXw3A/PnzfXafioh3sP9vgNVZZ0JV6Nu3L1FRURQWFrJq1Sqnrrspcdgd/H3SUrKPFABgOCqmG2DAF3/azM4vT3iwQhHxF8oD76Y8EBF3UR54N+WBiPtpUKIOdq1O51+PbmL/pizCo4No1SGc8Ogg9m/K4l9zNrp0YGL58uXcddddDBgwgCFDhvDee+9x4sQJVq5cCcCLL77INddcw913303//v1ZsGABNpuNefPmuawmZyg9tJvC7z/EfiIVky0Uc3RLTLZQ7CdSKfz+Q5cOTLRu3Zq2bdtWfn3++ee0bduWiRMnAvDyyy/75D4VEe/h7MuzK1gsFkaNGgXA7t27NVjaQLu+Okn67jwcZUaN801mEz8+r7OhRKTxlAfeTXkgIu6iPPBuygMR99OgRC0cDoNv5u8nL7OENknhhIQHYrGYCAkPpE1SOHmZJXw9f69bbuUEkJWVBUBMTAxFRUXs2LGDcePGVc63WCyMHDmSNWvWuKWehnDY7RSv+x6jIB9z81aYrMGYzGZM1mDMzVthFORTvO4Ht9zKqaioiE8++YRrr70Ws9lMcXGxT+5TEfEurup0AMTFxdG9e3egfOC6YltSdzu/OoE5wFT58ynjIPuMtZQaRQAYDoOj67I8VZ6I+BHlgXdTHoiIuygPvJvyQMT9NChRi0Pbszm6J4fmrYMxmUxV5plMJpq3DubIrhwO78h2eS12u50777yT/v37M3DgQE6ePIndbqdVq1ZVlouNjSUtzbW3lWoMR9pRHBnHMYdH17hPzeHRODKO4Ug76vJa3nvvPfLy8rj11lsBfHafioh3cWWnAyA5OZnQ0FByc3PZsGGDS7bhz+wlP59IUGTkc4oDFJDDSfSAQBFxLuWBd1MeiIi7KA+8m/JAxP00KFGL/OwSSosdWENqDo6gEAulxXbys0pcXsv06dPZs2cPH330kcu35UpGUT5GWSkEWmteINCKUVaKUZTv8lrmz5/PqFGjaN++vcu3JSJNh6s7HUFBQYwYMQKAzZs3c/r0aZdsx1+1S25Wfn9Y4AT7gPJ/Z3GCQiMPgMg2wZ4qT0T8iPLAuykPRMRdlAfeTXkg4n4alKhFWFQQgVYzxQU1X/5WUmAn0GohLDrIpXVMnz6dxYsX8/3339OhQweg/BI9i8XCiRNVH7aTlpZGbGysS+tpDJMtDFNAIJQW17xAaTGmgEBMtjCX1rFnzx5WrlzJTTfdVDnNV/epiHgXV3c6ABISEujQoQOGYbB06VIcDvfcRtAf9LumHdawAM6YssgjAzARShQAJ9kHJhh+WyeP1igi/kF54N2UByLiLsoD76Y8EHE/DUrUIqFHFG2SIjl9vBDDqPrAG8MwOH28kLZdI2nXPcol23c4HEyfPp2vvvqKxYsX07Vr18p5NpuN7t27s3jx4sppdrudFStWkJyc7JJ6nMEc2wZzTGsceVk17lNHXhbmmHjMsW1cWsfrr79Os2bNuOKKKyqnWa1Wn9ynIuJd7P97Jo4rOx0Aw4YNIygoiIyMDLZt2+bSbfkTa1gg0z8azinzfkwWE82Jpw3dMZnM5HGaVhcGMuS2jp4uU0T8gPLAuykPRMRdlAfeTXkg4n4alKiF2WxiwoyOhDcL4uiePArySimzOyjIK+XonjwimgcxcUZnLBbX7Mrp06fzySef8PbbbxMZGcmRI0c4cuQIZ86cAeCee+7h/fffZ+7cuWzcuJHrrruOwsJC7rjjDpfU4wxmiwXrwIswhYThOH0Co7gQw+HAKC7EcfoEppAwrAMvxGyxuKwGu93O+++/z5VXXklgYGCVeXfddZfP7VMR8S7uOBMKICQkhCFDhgCwbt068vLyXLo9fxLYqYBhD7Whw7BY2kUkEWINo0unrgy4rj3tb3FgtphqX4mISC2UB95PeSAi7qA88H7KAxH3cu3R0E90HdyCabP78vX8vRzZnUvpyfJbNnXs14yJMzrTfajrbuuzYMECACZPnlxl+ksvvcRdd93FjTfeSFpaGk8++SQZGRl07dqVzz77jDZtXHuVQWMFJnSBi66ieN33ODKOY+RnYwoIxNIqEevAC8vnu9Dnn3/OiRMnuO2226rNmzFjBunp6T63T0XEe1R0OiwuHFyt0LVrV/bt28fx48dZvnx5tbyQ6hwOB2vWrCGydTA3v/AbBgwYAEBRURHvv/8+mVmZ7N+vh9qJSOMpD7yb8kBE3EV54N2UByLup0GJOuo2NJYug1tweEc2+VklhEUH0a57lMuukKjwy9sb1WTmzJnMnDnTpXW4QmBCFyxtOuFIO4pRlI/JFlZ+ayc3hPRvfvOb8+7bmTNnMmvWLJfXISL+yV1nQlUYOXIkCxcu5OjRo+zdu5fOnTu7Zbu+aufOneTm5hIcHEzv3r0rp9tsNvr27cvatWtZt24doaGhHqxSRPyB8sC7KQ9ExF2UB95NeSDifrp9Uz1YLGYSezWj16g4Ens1c/mARFNgtlgIaJVAYGIPAloluGVAQkTE1dzd6YiMjKw8myclJYWioiK3bNcXlZaWsn79egAGDBhQ7RZ+vXr1IiQkhPz8fI4cOeKJEkXEjygPvJfyQETcSXngvZQHIp6h/1UXERFxMnd3OgB69+5Ns2bNKCoqYuXKlW7brq/ZvHkzRUVFREZG0rVr12rzAwICGDhwIAAHDhygpKTE3SWKiB9RHngv5YGIuJPywHspD0Q8Q4MSIiIiTuaJTofZbGbUqFGYTCb27duns3hqUFBQwJYtWwAYNGgQZnPNfwYlJSURFRVFaWkpmzdvdmeJIuJnlAfeSXkgIu6mPPBOygMRz9GghIiIiJN5otMBEBsbS8+ePQFYvnw5paWlbt2+t1u/fj1lZWXExsbSoUOHcy5nNpsZNGgQANu2bSM/P99dJYqIn1EeeCflgYi4m/LAOykPRDxHgxIiIiJOVtHpsHjgOTkDBw4kLCyM/Px81q1b5/bte6vs7Gx27doFwJAhQ2pdPiEhgejoaOx2e+U9ZkVE6kt54H2UByLiCcoD76M8EPEsDUqIiIg4mafOhAIIDAxk5MiRQPlZPOnp6W6vwRutWbMGwzBISEggLi6uTq9JSkoCYPfu3WRmZrqyPBHxU8oD76M8EBFPUB54H+WBiGdpUEJERMTJ7HY74JlOB0Dbtm3p1KkThmGwdOlSHA6HR+rwFqdOnSI1NRWTyURycnKdXxcVFUViYiJQ3mkREakv5YF3UR6IiKcoD7yL8kDE8zQoISIi4mSePBOqwrBhw7DZbGRmZlY+vK2pWrVqFQBdunQhOjq6Xq+teODd4cOHOX78uCvKExE/pjzwLsoDEfEU5YF3UR6IeJ4GJURERJzIMAyPnwkFYLPZGDp0KFD+ALecnByP1eJJqampnDp1ioCAAAYMGFDv10dERNCtWzcAVq9ejWEYzi5RRPyU8sC7KA9ExFOUB95FeSDiHTQoISIi4kQVHQ7wbKcDoHPnzrRp0wa73c7y5cs9WosnOByOysuqe/XqRWhoaIPW079/fwIDA0lPT+fAgQPOLFFE/JjywHsoD0TEk5QH3kN5IOI9NChRD3a7g+Nbs9m/LI3jW7Ox211/D75nn32WpKQkwsLCCAsLo2/fvixcuLDKMk8//TTx8fFYrVZ69+7N0qVLXV6XszgcdhyZ+3Cc3Fj+3WGv/UWNVFZWxr333kt8fDw2m422bdvyxz/+sco9FZ955hmf3aci4lkVl2YDWCwWD1ZSbsSIEQQEBHD8+HF27drl6XLcavfu3WRnZ2Oz2ejTp0+D1xMcHFz5+jVr1lTpWIqInIvywHsoD0TEk5QH3kN5IOI9NChRR4dSTrPw1nV8fMd6Pn9gEx/fsZ6Ft64jNSXDpdtt27YtTzzxBKtWrSIlJYVRo0ZxzTXXsH79egDeeOMNHnnkER588EFSUlLo0aMHU6dO5dixYy6tyxkcaVtxrHwWx6rncKydW/595bM40ra6dLsPP/wwb7/9Ns8//zybN2/mscce45VXXuGpp54CYP78+T67T0XE8yo6HRaLBZPJ5OFqyi8vHjhwIFB+79SCggIPV+QeZWVllVnZv39/goKCGrW+3r17ExISQl5eHjt37nRGiSLi55QH3kF5ICKepjzwDsoDEe+iQYk6OLTqNF/P2caxTVkERwfRvEMYwdFBHNuUxdezt7l0YOKaa67hiiuuoGfPnvTq1YuXXnqJkJCQysvsXnzxRa655hruvvtu+vfvz4IFC7DZbMybN89lNTmDI20rjo1vQNYeCAqD8Pjy71l7cGx8w6UDE6tWrWL8+PFcddVVdOnSheuvv54RI0awdu1aAF5++WWf3Kci4h284SF2v9SzZ09iYmIoKSlh5cqVni7HLbZs2UJBQQHh4eGV93xtjLPvObthwwZKSkoavU4R8W/KA++gPBART1MeeAflgYh30aBELRwOg7XzUynILKZFUji28ADMFhO28ABaJIVTkFnM6jcPuOVWTmVlZfzzn/+ksLCQUaNGUVRUxI4dOxg3blzlMhaLhZEjR1beI88bORx2HHu+gJIcCG+HKTAUk8mMKTAUwttBSQ6OvV+47FZOQ4YMYcWKFWzdWj7wsWrVKtatW8fEiRMpLi72yX0qIt7DGzsdZrOZUaNGYTKZOHDgAIcOHfJ0SS5VWFjI5s2bARg0aJDTLpPv0qULUVFRFBUVsWnTJqesU0T8l/LA85QHIuINlAeepzwQ8T4alKjFqe05pO/JJ6J1SLXL7EwmExGtQ0jblcepHbkuq2HNmjWEhIRgs9m47777ePfdd+nfvz8nT57EbrfTqlWrKsvHxsaSlpbmsnoaLfsg5B6G4Jga9ynBMZBzuHw5F3j88ce55JJL6NOnDwEBAQwbNoxbb72V2267zXf3qYh4DW/sdADExMTQu3dvAFasWOHXZ/Js2LCB0tJSYmJi6Nixo9PWazabSU5OBmDr1q2cOXPGaesWEf+jPPA85YGIeAPlgecpD0S8jwYlalGYVUJZsZ2gkJpHUYNCLJQV2ynMct3Bu3fv3qxdu5Yff/yR6dOnc8stt7BhwwaXbc/lSvLAXgIBwTXPt9jK55fkuWTz8+fP5+OPP+bvf/87K1euZN68ebzyyivMnTvXJdsTkabFWzsdAAMGDCAiIoIzZ85U3rLO3+Tm5lbe03XIkCFOv29v+/btiYuLw263s27dOqeuW0T8i/LAs5QHIuItlAeepTwQ8U4alKhFcHQQAVYLJQU130qopMBOgNVCcHTjHpBzPjabjR49ejBixAjmzp1Lt27d+Otf/0pcXBwWi4UTJ05UWT4tLY3Y2FiX1dNoQeFgCYKywprn24vK5weFu2Tzf/nLX7j33nu56aabSE5O5vbbb+eWW27hueee8919KiJew5s7HQEBAYwcORKA7du3c+rUKQ9X5Hxr1qzB4XDQtm1bWrdu7ZJtDB48GIA9e/aQlZXlkm2IiO9THniW8kBEvIXywLOUByLeSYMStWjZI5IWSWHkHi/AMIwq8wzDIPd4AbFdw2nZPcJtNTkcDkpKSrDZbHTv3p3FixdXzrPb7axYsaLy8jGvFJUIEe2gMKPGfUphBkS2K1/OBYqKijCbq/7qBwQEYBgGVqvVN/epiHiNik6Hs+5T6mzx8fF06dIFgKVLl2K3u+b5PZ6Qnp7OgQMHgJ87Bq7QsmVLEhMTMQyD1atXu2w7IuLblAeeozwQEW+iPPAc5YGI99KgRC3MZhODZrQnpJmV9D15FOWV4bAbFOWVkb4nj9BmVgbf0AGLxTW78s477+Trr79m9+7drFmzhjvvvJM1a9bwu9/9DoB77rmH999/n7lz57Jx40auu+46CgsLueOOO1xSjzOYzRbMSRdDUCTkHcYoPYPhsGOUnoG8wxAUibnzxZjNrgnssWPH8txzz/Hhhx+ye/du3nnnHV599VUmT54MwF133eVz+1REvEfFH/HeeCZUhSFDhhAcHEx2drZfPZBt1apVACQlJdGsWTOXbis5ORmTycThw4erXV0nIgLKA09SHoiIN1EeeI7yQMR7aVCiDhKGNGfinJ7E942mMKuE0wfyKcwqIb5fNBMe7Un7oTEu23Z6ejo33ngjvXr1YuLEiWzYsIFPPvmESy65BIAbb7yROXPm8OSTTzJ48GC2bdvGZ599Rps2bVxWkzOYY3th7ncjRCdBST7kHy//3iwJc78bMcf2ctm2//nPf3LxxRdz33330adPH2bNmsW0adN4/vnnAZgxYwazZ8/2uX0qIt7Bmy/PrmC1Whk2bBgAGzdu9ItLjCv++LdYLAwcONDl24uMjKRbt27Az50dEZGzKQ88Q3kgIt5GeeAZygMR7+aRQYl58+bRvn17bDYbgwcPZs2aNedd/m9/+xtdunQhODiYtm3bct9991FUVOSmasslDG3O5a8P5LJXBvCrv/blslcGcPlrA106IAHw4YcfcuzYMUpKSsjMzGTlypWVAxIVZs6cyfHjxykpKWHLli1ccMEFLq3JWcyxvTAP+xPmIX/APOjO8u9D/+TSAQmAqKgo3njjDY4fP05RURFHjhzhxRdfxGazVS7jq/tUxNf4Yh7Uxhc6HQAdO3akXbt2OBwOli1bVu12er7EMIzK350ePXoQFhbmlu0OGDCAwMDAKpeFi0jDKA88R3nQeMoDEedRHniO8qDxlAcidef2QYkPP/yQ+++/n9mzZ7Nhwwb69OnDhAkTSEtLq3H59957jz//+c/Mnj2bnTt38sYbb/Dhhx8ya9YsN1cOFouZ1r2i6Dgqlta9olx2y6amxGy2YG7WCXNcv/LvLrplk4h4H1/Og/PxlU4HwIgRIwgMDOTUqVPs3LnT0+U02J49e8jMzMRqtdKvXz+3bTc4OJg+ffoAPz9AT0TqT3ngecqDxlEeiDiH8sDzlAeNozwQqTu3/6/6888/z80338yMGTPo3r07r732GiEhIbz55ps1Lr9y5UqGDx/Ob3/7W9q3b8/48eO55pprah0tFxER7+aveeBLnY6wsDAGDRoEwOrVqzlz5oyHK6q/srIy1q1bB0C/fv2wWq1u3X6vXr0IDg4mNzeXHTt2uHXbIv5CeeB5yoPGUx6INJ7ywPOUB42nPBCpG7ceEUtKSli/fj0zZ86snGY2mxk7diwpKSk1vmbYsGEsWLCANWvWkJyczIEDB/jyyy+57rrrzrmd4uJiiouLK38uKCio/HdjLj/z5UvXGkPtdu76ysrKKC0tdeq6G6uiHm+ry9XU7oa12xn7y1N5kJubW9mG+rajrvutqKio8mF2vvC7lZSUxK5du0hPT+fHH39k/PjxVeZ7++dk06ZN5ObmEhoaSlJSktPqrE+7e/fuzcqVK1mzZg2JiYkEBQU5pQZP8Pb321XUbuVBfSgPvLMtygPn8vb321XUbuVBfSgPvLMtygPn8vb321XUbtfngVsHJTIyMrDb7bRs2bLK9JYtW7Jr164aX/Pb3/6WjIwMRowYgWEYlJWVcdttt533crynnnqKRx99tPLnYcOG8fLLL2O32yuDoL4a+jpfp3Y7j8PhwDAM1q5dW+WPIG/y3XffeboEj1C76+fsgd6G8lQeVPj2228JCQlpUO217bft27dz9OhRioqKOHXqVIO24W5FRUVs376drVu3cvToUeLi4qot442fk5KSEpYvX05ZWRk9e/bkm2++cfo26tJuh8PBgQMHOHPmDJmZmXTu3NnpdbibN77f7qB214/yQHngLZQHruON77c7qN31ozxQHngL5YHreOP77Q5qd/3UJw+8/tqxH3/8kSeffJJXXnmFwYMHs2/fPu655x4ee+wxHn744RpfM3PmTO6///7Kn9PT08nJycFisWCx1P+ZBXa7vUGv83Vqt3OZzWZMJhODBg0iIiLC6etvjNLSUr777jvGjRtHYGCgp8txG7W7Ye2uOJvI3ZyRB7m5ubRt25bx48fX+3NY1/0WHBxMdHQ0Q4YMoWfPnvXahid16tSJjRs3YrVaueiiiyovc/bmz8mqVavo1q0b0dHRXHrppZhMJqetu77t7tmzJ4sXLyYgIIAxY8Y0uFPrad78fruS2q08qA/lgfd9TpQHzufN77crqd3Kg/pQHnjf50R54Hze/H67ktrt+jxw66BETEwMFoul2sjwqVOnahx1BXj44Ye57rrruOmmm4Dye7OdOXOGW265hYceegizufpjMaxWa5V7xp05c4acnByAeh+Qzr6FjzMPZt5O7XZ+uyvWFxAQ4LUHtMDAQK+tzZXU7vq/rrE8lQdnt6Gh7ajLay0WCzabzad+rwYNGsThw4fJzs5mw4YNjBo1qsp8b/uc5OXlsXv3biwWC8OHD3fZJdF1bXfnzp3ZsWMHp06dYvPmzdX2n6/xtvfbXdTu+r+usZQH3kd5UDPlQdOidtf/dY2lPPA+yoOaKQ+aFrW7/q+rK7c+6DooKIgBAwbw/fffV05zOBx8//33DB06tMbXFBQUVAuSirPYm+qzDkREfJ0/54EvPcjubBaLpfIP5V27dnHixAkPV3R+a9euxeFwEB8fT9u2bT1dDgBDhgwBYPfu3WRlZXm4GhHfoDzwPsqDxlMeiNSf8sD7KA8aT3kgcm5uHZQAuP/++/nHP/7B22+/zc6dO7n99ts5c+YMM2bMAGDatGlVHmw0depUXn31VT744AMOHjzId999x8MPP8zUqVOb5K2FRET8hb/mga92OgDi4uLo1q0bAMuWLfPa5wplZGSwb98+AAYPHuzhan7WsmVL2rdvj2EYrFmzxtPliPgM5YH3UR40jvJApGGUB95HedA4ygORc3P7EfGqq64iPT2dRx55hJMnT9K3b1++/vrryocZHT58uMpI91/+8hdMJhN/+ctfOHbsGC1atGDq1Kk88cQT7i5dREScyF/zwJc7HVD+R/yhQ4fIyclhw4YN9O3b19MlVVPxB32nTp2IiYnxcDVVJScnc+jQIQ4dOsSJEydo1aqVp0sS8XrKA++kPGgc5YFI/SkPvJPyoHGUByI188gR8c477+TOO++scd6PP/5Y5eeAgABmz57N7Nmz3VCZiIi4kz/mga93OoKCghgxYgTffvstmzdvpl27dp4uqYqjR49y9OhRzGYzgwYN8nQ51URFRdGtWzd27NjB6tWrueSSSzxdkohPUB54H+VB4ygPRBpGeeB9lAeNozwQqZnbb9/kyxx2B6d3ZHIi5QSnd2TisDvcuv1Zs2ZhMpm48cYbq0x/+umniY+Px2q10rt3b5YuXerWuhrD4bDjKNiNI3dt+XeH6y8FzM7O5sYbb6R169bYbDb69evHsmXLqizzzDPP+Ow+FRHP8vVOB0D79u1JTEzE4XCwbNkyr7knr2EYrF69GoDu3bsTHh7u4Ypq1r9/fwICAkhLS+PAgQOeLkdEPER54DrKAxHxJcoD11EeiPguDUrU0al1afz0wEpW/jmFNXPWsvLPKfz0wEpOrk1zy/aXLVvG22+/TVJSUpXpb7zxBo888ggPPvggKSkp9OjRg6lTp3Ls2DG31NUYjryNkPowHJoDR54u/576cPl0F7r22mtZunQpb775JuvXr+fCCy9kypQpHDx4EID58+f77D4VEc+r6HR4031sG2L48OEEBQWRkZHB4cOHPV0OAPv27eP06dMEBQXRv39/T5dzTiEhIfTp0wcov5Tc4XDvSQwi4h2UB66jPBARX6I8cB3lgYjv0qBEHaSvT2fj/23k9LZMrJFBhCeEY40M4vS2TDb+3waXD0zk5OQwbdo0XnnlFSIjI6vMe/HFF7nmmmu4++676d+/PwsWLMBmszFv3jyX1tRYjryNcOwlKNwJlkiwti3/XrgTjr3ksoGJM2fO8M033/DEE08wceJEevTowXPPPUe7du3429/+BsDLL7/sk/tURLyDP5wJBeV/OFc8JG7v3r3k5+d7tB673c7atWsB6Nu3LzabzaP11KZ3794EBweTm5vLzp07PV2OiHiA8sA1lAci4muUB66hPBDxbRqUqIXhMNj7wT6Ks0qI6BhOYFgQJouZwLAgIjqGU5xVwt7397j0Vk433HADY8eO5de//nWV6UVFRezYsYNx48ZVTrNYLIwcObLyIT/eyOGwQ/pHYM8GayJYQsFkKf9uTSyfnrHQJbdyKi0txW63ExwcXGW6zWYjJSWF4uJin9ynIuI97PbyY5evdzoAunbtSsuWLbHb7axYscKjtWzfvp38/HxCQ0Pp2bOnR2upi8DAQAYMGADA+vXrKS0t9XBFIuJuygPXUB6IiK9RHriG8kDEt2lQohZZu7PJ2ZdLSFwwJlPV3WUymQmJCyZ7Xw5Zu7Ndsv1//vOfbN26lZdeeqnavJMnT2K322nVqlWV6bGxsaSluee2Ug1StA+KD0JgSzCZqs4zmcqnFx0oX87JoqKi6Nu3L48//jipqamUlZXx6quvsmnTJtLT0313n4rUgWEYHFiezid3rQfg03s2cHBlhtfcD9Qf2O32yv3pD50Ok8nEyJEjMZvNHD16lH37nH9crouSkhI2biy/gm7AgAE+s2+7du1KZGQkRUVFbN682dPliIgbKQ9cQ3kgIr5GeeAaygMR36dBiVqU5BTjKLETEFzzAc4SbMFRbKckp9jp296/fz8PPvgg77zzDiEhIU5fv8eU5YKjBMznuLTObCufX5brks2/++67GIZBYmIiNpuNV199lalTp2L65QCJiB8pK7bz1mU/MW/0D2x8v/z+n+vfPcTcEd/zztUp2Et1T0tnqLg0G/yj0wHlg7kdOnQAYOXKlRQVFbm9hk2bNlFcXEx0dHS1Zyt5M7PZTHJyMgBbtmyhoKDAwxWJiLsoD1xDeSAivkZ54BrKAxHfp0GJWgRFWjEHWSgrLKtxvr3QjtlqISjS6vRtp6SkkJmZyfDhwwkICCAgIIC1a9cyf/58AgICaNmyJRaLhRMnTlR5XVpaGrGxsU6vx2kCIsAcBI5zBJejqHx+QIRLNt+9e3fWrl1LTk4O+/fvZ8uWLZSWltKuXTvi4uJ8c5+K1OLzBzax/fPyh7U7yowq37csPMKimVs8Vps/qbg022w2Yzb7T8QmJiYSHR1NUVERq1atcuu28/Pz2bp1KwDJyck+t18TExOJjY2lrKyM9evXe7ocEXET5YHzKQ9ExBcpD5xPeSDiH3zrk+sB0V2iiOwUQcHJQgyj6pnEhuGg4GQhUZ0iie4S5fRtX3zxxaxdu5ZVq1ZVfvXo0YNf//rXrFq1iuDgYLp3787ixYsrX1NxX7+KkVevZOtU/uyI0lPwy9vGGEb5dFuH8uVcKCIigoSEBNLT01m2bBlTp07FarX65j4VOY8zp4tZ9ff9GOe4GMIw4KdX9lKYU+LewvxQxZlQFovFw5U4l9lsZuTIkQDs2bOHo0ePum3b69atw263ExcXR0JCgtu260xDhgwBYNeuXWRnZ3u2GBFxC+WB8ykPRMQXKQ+cT3kg4h80KFELk9lE56s7YY0OInd/HqX5JTjsdkrzS8jdn4c1OojO1yRhtjh/V0ZFRTFw4MAqXyEhITRr1oyBAwcCcM899/D+++8zd+5cNm7cyHXXXUdhYSF33HGH0+txFrPZAi2uAEtU+bMl7GfAsJd/Lz4IAVEQc3n5ci7wySef8PHHH7Nr1y4+/fRTRo4cSYcOHbjzzjsBuOuuu3xun4qcz74ladhLfx4APOVI5ZtvviHdcaRyWlmRg/1L0z1Rnl+p6HT4y6XZZ4uNja18gNzy5curXIruKpmZmezZswf4+Q93XxQXF0f79u0xDIM1a9Z4uhwRcQPlgXMpD0TEVykPnEt5IOI/NChRBy0GtKDfH/vRvGczinNKyD+UT3FOCc17NaPfH/sTN8hzt/W58cYbmTNnDk8++SSDBw9m27ZtfPbZZ7Rp08ZjNdWFObwfxN8Nwd3AngPFR8q/h3SH1neXz3eR7Oxs7r33Xnr37s3NN9/M4MGD+eGHH7Bay2/BNWPGDGbPnu1z+1TkXOwlVS+ROOkofxjZccfu8y4n9efPnQ6AQYMGERYWRl5eHuvWrXP59ir+QO/QoYPP30IvOTkZk8lEamoqJ0+e9HQ5IuJiygPnUh6IiK9SHjiX8kDEf/jnUdEFWg6MJbZ/C7J2Z1OSU0xQpJXoLlEuuULifGoaQZ05cyYzZ850ax3OYA7vhyO0NxTtK3+odUAE2Dq57AqJCjfccAM33HDDeZeZOXMms2bNcmkdIu4S3z+6bsv1q9tycm7+3ukIDAxkxIgRfP3112zdupVOnToRExPjkm0dP36cw4cPYzabGTRokEu24U5RUVF07dqVnTt3smrVKi655BJPlyQiLqQ8cB7lgYj4MuWB8ygPRPyLrpSoB7PFTPPuzWg1tBXNuzdz+4CEPzKbLZhDumCOGFT+3cUDEiJNUcuuEXQc0wJzgKnG+WaLiS7j44jpGObmyvyPv3c6ANq1a0fHjh0xDIOlS5ficDj/ChvDMFi9ejUAXbt2JTIy0unb8IQBAwYQEBBAWloaBw8e9HQ5IuJCygPnUB6IiK9THjiH8kDE/+h/1UVEmoCr3xxMWAsrJkvVgQmTxURE62Cu/Kfvn2niDZpCpwNg2LBhWK1WTp8+zdatW52+/gMHDpCenk5gYCADBgxw+vo9JSQkhN69ewPlVz66osMmIt5BeeAcygMR8XXKA+dQHoj4Hw1KiIg0Ac3ah3L/hglc8EBXgsICAQgKD+SiB7tx3/rxRLUJ8XCF/qGi02Gx+PdVX8HBwQwdOhSAdevWkZub67R1OxwO1q5dC0Dv3r0JDg522rq9QZ8+fbDZbOTk5LBr1y5PlyMiLqI8aDzlgYj4A+VB4ykPRPyTBiVERJqI8JY2pjzVm8mP9wJg8mO9mPR4L8JirB6uzH80lTOhAJKSkoiPj8dut7N8+XKnrXfHjh3k5uYSHBxcedaQPzn77K7169dTWlrq4YpExBWUB42nPBARf6A8aDzlgYh/0qCEiIiIk9jtdqBpdDoARo4cicVi4dixY+zZs6fR6yspKWHDhg1A+f1VAwMDG71Ob9StWzciIiIoLCxky5Ytni5HRFxAedA4ygMR8RfKg8ZRHoj4Lw1KiIiIOElTOhMKICIigoEDBwKQkpJCYWFho9a3ZcsWioqKiIyMpGvXrs4o0SuZzWaSk5MB2Lx5MwUFBR6uSEScTXmgPKgL5YGI/1MeKA/qQnkgTZEGJURERJykqXU6AHr16kVMTAzFxcWsXLmywespKCioPCsoOTkZs9m//0Tp0KEDsbGxlJWVVZ79JSL+Q3mgPKgr5YGIf1MeKA/qSnkgTY1/f6JFRKRSUVEZ77y9mVfmlT8k7NV5a3l3wVaKi8s8XJn/aIqdDrPZzKhRozCZTOzfv5/Dhw83aD3r1q2jrKyMli1bkpiY6OQqvdPgwYMB2LlzJ9nZ2Z4tRkScSnmgPKgP5YGI/1IeKA/qQ3kgTYkGJUREmoCsrCLGjnmH39/2FYdScwBITc3h9psXMXHsu+TmFnu4Qv/QFDsdADExMfTqVf4A9RUrVtT74WzZ2dns3r0b+PkP8aagVatWJCQkYBgGa9as8XQ5TZbd7sAwDE+XIX5GeaA8qA/lgXdQHogrKA+UB/WhPPAOygP30KCEiEgTcM+dX7F9WxoAFdla8X3TxpP84d5vPVSZf2mqnQ6AgQMHEh4eTn5+PmvXrq3Xa9esWYNhGLRv3564uDgXVeidkpOTMZlMpKamcurUKU+X02QUF5cx98U19O7+GtFhz9Kx3csA7Nie5uHKxF8oD5QH9aU88Azlgbia8kB5UF/KA89QHrifBiXqwWF3kLcvjayNh8nbl4bD7nD5Nv/whz9gMpmqfP3ysrWnn36a+Ph4rFYrvXv3ZunSpS6vy1kcDjtljl2U2VdT5tiFw2F3+Ta//vprLrzwQmJjYzGZTCxYsKDaMs8880yt+9SX97s0LUeP5PLZf3Zjt9c80m+3Gyz89w5Oncx3c2X+x24vP4Y1xU5HQEAAI0eOBGDbtm2kpdXtj7eTJ0+SmpqKyWSqfLhbUxIdHU2XLl0AWLVqlYeraRqKisq45OIPeWjWDxxKzQaguKT8szt5/Pt89+0BD1Yn/kJ5oDyoL+WB+ykPxB2UB8qD+lIeuJ/ywDM0KFFHOVuPsvvpr9jz7Dfse/F79jz7Dbuf/orsrUddvu1OnTpx+PDhyq+zHxT0xhtv8Mgjj/Dggw+SkpJCjx49mDp1KseOHXN5XY1V5thAkX0WRWUPU2h/kqKyhymyz6LM4doH+uTn59OrVy+ee+65GufPnz+/1n3qy/tdmp7Vq45R25WHdrvBmtX6/W2spnwmFECbNm1ISkoCYNmyZTgctQ/er169GoAuXboQFRXlyvK81sCBAwkICODUqVOkpqZ6uhy/98Jzq0hZeRTDQbVjo93h4PrrPqWgoH63GBD5JeWB8qAhlAfupTwQd1AeKA8aQnngXsoDz9CgRB3kbjvOwX+sIG/vKQLCrdjaRBEQbiVv7ykO/n25ywcmLBYLbdu2rfxq1apV5bwXX3yRa665hrvvvpv+/fuzYMECbDYb8+bNc2lNjVU+IPECdmMnEImJtkAkdmMnRfYXXDowcfnll/Piiy9y3XXX1Tj/5ZdfrnWf+up+l6ZJ90J0n4pOh8Vi8XAlnjNkyBBsNhuZmZls2rTpvMtWXJIcEBDAwIED3VOgFwoJCam85+6aNWvq1FmThrHbHfzjtQ04HD8fFx2O0xhG2f/+DXm5JXz80Q5PlSh+QnmgPGgI5YH7KA/EXZQHyoOGUB64j/LAczQoUQvDYXDyiy2U5hUS3LYZllArJrMZS6iV4LbNKM0r5MR/N7v0Vk6HDh0iNjaWNm3a8Otf/5q9e/cCUFRUxI4dOxg3blzlshaLhZEjR3r1A3EcDjsl9g8xjBxMtMdsDsNsDsBsDsNEewwjhxL7v91yK6dfKi4urnWf+up+l6ZryNA2mEznX8ZsNpE8ON49Bfmxpn4mFIDNZmPYsGEAbNiwgezs7BqXczgclWdB9erVi5CQEHeV6JX69OmDzWar8lA/cb60U2fIyCio/NnuSKPMsZ+S0p2Vn9/AQDObNur+vdI4ygPlQUMpD9xDeSDuojxQHjSU8sA9lAeeo0GJWpw5mE7B4SyszcMw/eJ/9UwmE9bmYRSkZnLmYIZLtj906FBeeeUV/vvf//LSSy9x+PBhRo8eTXZ2NidPnsRut1e5cgIgNja2zvfq8wQHe3EYBzARi9lc9VfQbDZjIhaHsR8He91eW132qa/ud2m64tuE09oWBue6YMKAtqHhxLYMdWtd/kidjnKdOnWibdu2OBwOli1bVuPVOrt27SInJwebzUafPn08UKV3CQoKon///gCsW7eO0lJdHuwKQdafP5sORwZ2RyoAZnNE5efWMMBqbbpnM4pzKA/KKQ/qT3ngHsoDcRflQTnlQf0pD9xDeeA5GpSoRVleMUZJGWZbYI3zzbZAHCVllOUVuWT7l19+OTNmzGDw4MFceumlfPfdd+Tl5fHWW2+5ZHtuYeRgUALYzrGArXy+kePOqkT81o6VabTODSHECCgfmKj4++9/30ONAGIzbexdf9pTJfoNdTp+NnLkSAICAjh58iS7du2qMq+0tJT169cD0L9/f4KCgjxRotfp3r07ERERFBYWsnXrVk+X45eaNw+mX/84IIsyR/kD68zmWAIDEiqXKStzMGlKJw9VKP5CefAz5UH9KQ9cT3kg7qI8+JnyoP6UB66nPPAcDUrUIiDciikoAEdRzSOSjqJSzEEBBISf6z/YnSsmJoaEhAT27dtHXFwcFouFEydOVFkmLS2N2NhYt9TTIKZITAQB5xrIKSqfb4p0Z1UAddqnPrvfpck6mZpPAGZ62ZvT0R6B7X+HfhtmOpZF0NPenADMnDiY5+FKfZ86HT8LCwtj0KBBQPnD6tL2prH/PwfY/+kB1vywhsLCQiIiIujevbuHK/UeZrOZ5ORkADZt2kRhYaGHK/JP02ckUFK2DwCzKQaL6ecOh8Viok+/lowY2c5T5YmfUB78THlQf8oD91AeiDsoD36mPKg/5YF7KA88Q4MStQhNbEFIu2iKT+dXu7zMMAyKT+cT0r4ZoYkxbqknJyeHI0eO0KpVK2w2G927d2fx4sWV8+12OytWrKg8aHkjM50xmzpgkFbtYT0OhwODNMymjpjp7PbarFZrrfvUV/e7NF3h0VYAzJiINUJoZ5QPnrU3WhNrhGDGVGU5aTh1Oqrq2bMnkbZINv99M08PfpYfb1/Kt7d+xxtXvcGud3bTu2vvarfxa+o6dOhAixYtKCsrqzxbTJzn+PHjGKYD/OayJCzmZgQFdsBsNmGxlB8HO3duxkcfX1Htlp0i9aU8qEp5UH/KA9dSHoi7KA+qUh7Un/LAtZQHnqNPei1MZhNxF/cmMDyYwiOZ2M8UY9gd2M8UU3gkk8CIYFpN7YPZ4ppdeeutt/LVV1+xe/duFi9ezOTJkzGbzcyYMQOAe+65h/fff5+5c+eyceNGrrvuOgoLC7njjjtcUo8zmM0WgixXYTJFYpCKw5GPw1GGw5GPQSomUxRBlisxm11zv7acnBxSUlJISUkB4MCBA6SkpFQ+QPyuu+6qdZ/64n6XpqvvRa0Ii/r58lebKRoA61lXI0XF2ug9Os7ttfmbik6HxaL7TQLYi+wU/b2ErB3ZZJSlk2HP4FDpIcocdkq2l7DnT3uxF9s9XabXGTx4MAA7d+4kJ0e3MnSWU6dO8fXXX2O327l+xhi27XqW+/84jIunJvGbS7sA8M33vyOuVZhnCxW/oDyoSnnQMMoD11AeiDspD6pSHjSM8sA1lAeepaHaOojo2ZrEm0dw4r9bKDiUieP0GcxBAYQntaTV1D5E9Wrjsm0fO3aM6dOnk52dTXR0NIMGDeKnn36idevWANx4442kpaXx5JNPkpGRQdeuXfnss89o08Z1NTlDgLk/Nu6jxP4hDuMABumYCMJi6k6Q5UoCzP1dtu2ffvqJKVOmVP48e/ZsZs+ezWWXXcZHH33EjBkzSE9PP+8+9dX9Lk1TkNXC9P/Xj3l3rz7nMtP/X38CAjVO3RiGYVRe/aUzocrtW7ifsn1ltLG05bDjEHtL9lBqlHfMEi2JnN6SyYHPDtL5St2f82ytW7emXbt2HD58mDVr1jBu3DhPl+TzMjIy+OqrrygrK6NNmzaMHTsWi8XCI3NGAeX3Mf7yyy8JCNBxUBpPeVCd8qBhlAfOpzwQd1IeVKc8aBjlgfMpDzxPR8U6iuzVhoge8Zw5mEFZXhEB4TZCE2NcdoVEhS+++KLWZWbOnMnMmTNdWocrBJj7Y6YPDvaWP9TaFFl+aycXXSFRYfLkydVuxVWhYvrMmTOZNWvWedfjq/tdmqapt3elpMjO249s/PlB14A1xMKNTw5k8k1JnivOT1ScBQXqdFTY894eMEFCQALp9nQKHQUANLc0J8oSDWbY88FedTpqkJyczJEjRzh48CCnTp2iZcuWni7JZ2VmZrJo0SJKSkqIi4tj/PjxOltRXEp5UJ3yoOGUB86jPBB3Ux5UpzxoOOWB8ygPvIOGe+rBbDET3imW6H7tCO8U6/IBiabAbLYQYO5KgGUwAeauLh+QEGmqTCYTl9/fkw+OX8Xl9/cA4Mo/9uKDY1fz6zu7ebg6/6BOR3UFpwrBALPJTFLgzwNf7QMTy//hgIITBR6qzrs1a9aMpKTyfbZ69bmvcpLzy8nJYdGiRRQXFxMbG8vEiRP1+RSXUx5UpzxoOOWBcygPxBOUB9UpDxpOeeAcygPvof9VFxFpQkIjghg4vvw2YwMnxBMSHujhivyHHmJXXWirkMq/NKIsUfS09qSntRdh5vJ7cprMJkJbh3iwQu82cOBALBYLJ0+e5NChQ54ux+fk5eXxxRdfUFhYSPPmzZk0aRJBQUG1v1CkkZQH1SkPGkd50DjKA/EU5UF1yoPGUR40jvLAu2hQQkRExAns9vIHsqnT8bOka5PA8fPPzS0xNLc0r/zZcBgk/Va3DjuX0NBQevXqBZSfDVVxT2Kp3ZkzZ1i0aBFnzpwhKiqKyZMnY7VaPV2WNBHKg+qUB42jPGg45YF4kvKgOuVB4ygPGk554H00KCEiIuIEFWdC6V6UP+t4aQea926OyWKqNs9kMdFiQAsSp7Z3f2E+pG/fvthsNrKzs9mzZ4+ny/EJhYWFLFq0iNzcXCIiIpgyZQrBwcGeLkuaEOVBdcqDxlMe1J/yQDxNeVCd8qDxlAf1pzzwTk1qUOJcDzcWcQf9/ok3cJQ5SN+cAUDGltM47Dqzwll0eXZ1AbYAJi+cSOKvEqt0PEwWEx0v7cDEDydgCVIn7XyCgoLo378/AOvWratyb2Kprri4mEWLFpGdnU1YWBhTpkwhNDTU02VJE6M8qE550HjKg/pRHog3UB5UpzxoPOVB/SgPvFeTODIGBQVht9spKCggLCzM0+VIE1VcXIxhGLpfnXjM/v8cYPUjq9mWsQ1+A2seXcOZ1woY8thgnY3iBOp01CwoIogLXh3N4DmDSFufDiZoOTCW4BY6M6WuunXrxtatW8nLy2PLli2VnRCpqqSkhC+//JLMzExCQkKYMmUK4eHhni5LmiDlQc2UB42nPKgb5YF4C+VBzZQHjac8qBvlgXdrEkfGwMBA1q5dW3mvsNDQUEym6peK1cQwDBwOB2azuc6v8Qdqt3PbbbfbSUtLIyAgQPesE4/Y/+kBfrx9afkPZ/1qF5ws4Iebl3DhPy8g8eL2HqnNX6jTcX4hLUNoPznB02X4JIvFQnJyMt9//z2bN2+mW7duutz4F8rKyvj6669JT0/HZrMxZcoUIiMjPV2WNFHKg/NTHjSc8qB2ygPxJsqD81MeNJzyoHbKA+/XZI6Md911F1u3buXEiRP1/s9mwzCa1H/MV1C7nb/eTp06YTY3qbumiRdwlDlY/cia8y6zevYa2k9OwGRuep95Z1GnQ1ypQ4cObN68mYyMDDZs2MDw4cM9XZLXsNvtfPPNN5w8eZKgoCAmT55MdHS0p8uSJkx5IK6kPDg35YF4G+WBuJLy4NyUB76hyRwZHQ4Hbdq0wWazUVhYWOfXlZWVsXbtWgYNGtSkgkTtdn67Q0NDm9S+FO9xYuVJCtPOf9w7c+wMJ1efotXQODdV5X/U6RBXMplMDBkyhC+++IKdO3fSs2dPnelD+d93ixcv5tixYwQEBDBp0iRiYmI8XZY0ccoDcSXlQc2UB+KNlAfiSsqDmikPfEeTOzIGBQXV657+paWlFBcXExERQWBgoAsr8y5qd9Nqt/i32gYk6ruc1Kyi02Gx6MFs4hqtW7embdu2HDlyhLVr1zJ27FhPl+RRDoeDH374gUOHDmGxWJg4cSItW7b0dFkiygNxOeVBVcoD8VbKA3E15UFVygPfovvIiIj4uZC4ut1bMrRViIsr8W86E0rcYfDgwQAcOHCAtLQ0D1fjOYZhsGzZMg4cOIDZbGb8+PG0bt3a02WJAMoDcQ/lQTnlgXgz5YG4g/KgnPLA92hQQkTEz8UNjSMk7jwDDiYIaxdG7MBY9xXlh+x2O6BOh7hWs2bNSEpKAmD16tUersZzfvrpJ/bs2YPJZGLs2LG0bdvW0yWJVFIeiDsoD8opD8SbKQ/EHZQH5ZQHvkeDEiIifs5sMTP0icFgovzrbP/7eejjQ/SQ60bSmVDiLgMHDsRisXDixAkOHTrk6XLcbtWqVezYsQOACy64gPbt23u2IJFfUB6IuygPlAfi3ZQH4i7KA+WBL9KghIhIE9B+SnsuevNCwuJDq0wPbxfGuLfH0m68ziJoLHU6xF3CwsLo1asXAGvWrMHhcHi4IvdZt24dW7ZsAWDUqFF06tTJwxWJVKc8EHdRHigPxLspD8RdlAfKA1+kQQkRkSai/aQErlxzBYMfLb/n5JAnBnNFyuUakHASdTrEnfr27YvVaiUrK4s9e/Z4uhy32LRpExs2bABg2LBhdO3a1cMVidRMeSDupDxQHoj3Uh6IOykPlAe+RoMSIiJNiMlsolm3aACadW2mWzY5kTod4k5BQUH0798fKD87qOL3z19t27aNNWvWAJCcnEzPnj09XJHIuSkPxJ2UB8oD8V7KA3En5YHywNdoUEJERMQJ1OkQd+vevTvh4eEUFBSwdetWT5fjMrt27WLlypUA9O/fn759+3q2IJFaKA/E3ZQHIt5JeSDupjwQX6JBCRERESew2+2AOh3iPhaLhUGDBgHlly4XFRV5uCLn27dvH8uWLQOgd+/eDBw40MMVidROeSDupjwQ8U7KA3E35YH4Eg1KiIiIOIHOhBJP6NixIzExMZSWllbeT9VfpKamsmTJEqD8rK8hQ4Z4uCKRulEeiCcoD0S8j/JAPEF5IL5CgxIiIiJOUNHpsFgsHq5EmhKTycTgweUPr9+xYwe5ubkersg5jhw5wuLFizEMg6SkJIYPH+7pkkTqTHkgnqA8EPE+ygPxBOWB+AoNSoiIiDiBzoQST4mPj6dNmzY4HA7Wrl3r6XIa7fjx43z77bc4HA46dOjAqFGjMJlMni5LpM6UB+IpygMR76I8EE9RHogv0KCEiIiIE6jTIZ5UcTbU/v37SU9P93A1DXfq1Cm+/vpr7HY77dq148ILL8Rs1p+r4luUB+JJygMR76E8EE9SHoi307soIiLiBOp0iCc1b96cpKQkAFatWuXhahomIyODr776irKyMuLj4xk3bpw6HOKTlAfiScoDEe+hPBBPUh6It9M7KSIi4gTqdIinDRw4EIvFwokTJzh8+LCny6mXrKwsvvzyS0pKSoiLi2PChAm6/7L4LOWBeJryQMQ7KA/E05QH4s00KCEiIuIE6nSIp4WFhdGzZ08AVq9ejWEYHq6obnJycvjiiy8oKiqiRYsWTJw4UZ8j8WnKA/E05YGId1AeiKcpD8SbeWRQYt68ebRv3x6bzcbgwYNZs2bNeZfPzs7m97//Pa1atcJqtZKUlMSXX37ppmpFRMRV/CUP7HZ75b919oZ4Ut++fbFarWRlZbFnzx5Pl1Or/Px8Fi1aRGFhIc2aNWPy5MkEBQV5uizxAOWBiHMpD8RXKQ9EnEt5IN7K7YMSH374Iffffz+zZ89mw4YN9OnThwkTJpCWllbj8iUlJYwbN47U1FQWLlzI7t27+cc//kF8fLybKxcREWfypzyoOAsKdCaUeJbVaqVfv34ArFu3rsrvprcpKCjgiy++ID8/n6ioKKZMmYLVavV0WeIBygMR51MeiC9SHog4n/JAvJXbByWef/55br75ZmbMmEH37t157bXXCAkJ4c0336xx+TfffJPMzEw+/fRThg8fTvv27Rk9ejR9+vRxc+UiIuJM/pQHFX/Ymc1mPXhLPK5Hjx6EhYVx5swZtm3b5ulyalRYWMgXX3xBbm4uERERTJkyheDgYE+XJR6iPBBxDeWB+BrlgYhrKA/EG7l1uLakpIT169czc+bMymlms5mxY8eSkpJS42s+//xzhg4dyu9//3s+++wzWrRowW9/+1sefPDBc14CV1xcTHFxceXPubm5AJSWllJaWlqvmiuWr+/rfJ3arXY3BU213RV/IJeVlTWo7c7YX/6WB4WFhdjtdiwWi9/9PjXVz4mvt7tv374sXbqU9evX07FjR2w2W51e5452FxcXs2jRIjIzMwkJCWH8+PEEBQV5dF/7+vvdUI1tt/JAedAU+Hq7lQf14+vvd0MpD5QH9aHPiW+2W3lQP77+fjeUO/PArYMSGRkZ2O12WrZsWWV6y5Yt2bVrV42vOXDgAD/88APXXnstX375Jfv27eOOO+6gtLSU2bNn1/iap556ikcffbTa9G+//ZaQkJAG1f7dd9816HW+Tu1uWtTupqHieLt69WqysrLq/fqCgoJG1+BveZCbm8vWrVuxWq1ecQ9bV2hqn5MKvtpuwzBITU0lLy+PjIwMunbtWq/Xu6rdpaWlrF+/npycHIKCgkhOTmb58uUu2VZD+Or73VgNbbfyQHnQlPhqu5UHDeOr73djKQ+UB/Whz4lvUR40jK++343ljjzw+hvbORwOYmNj+fvf/47FYmHAgAEcO3aM//u//ztnyMycOZP777+/8ufc3Fzatm3L+PHjiYiIqNf2S0tL+e677xg3bhyBgYGNaosvUbvV7qagqbY7IiKCQ4cOMXjwYIYOHVrv11ecTeRu3pwHaWlp5OfnEx4ezuTJk+vfOC/WVD8n/tDuPn368PXXX2M2mxkxYkSdfudd2e6ysjK++uor2rVrh9VqZcqUKTRr1syp22gof3i/G6Kx7VYeKA+aAn9ot/Kg7vzh/W4I5YHyoD70OfHddisP6s4f3u+GcGceuHVQIiYmBovFwqlTp6pMP3XqFHFxcTW+plWrVgQGBla59K5bt26cPHmSkpKSGp/AbrVaa3wQSmBgYIN/kRrzWl+mdjctanfTUPGgtYCAgAa12xn7yt/ywGQyYbFYsFqtfvu71NQ+JxV8ud2JiYkkJCRw9OhRNm3axEUXXVTn1zq73Xa7nSVLlpCRkUFwcDAXX3wxMTExTlu/s/jy+90YDW238kB50JT4cruVB/Xny+93YygPlAf1oc+J71Ee1J8vv9+N4Y48cOvTdoKCghgwYADff/995TSHw8H3339/zrN1hw8fzr59+3A4HJXT9uzZQ6tWrWoMGBERObfsnzZwaN5/ADg09xNyVm32SB3+lgcVz+moGPAR8RbJyckA7N+/n/T0dI/U4HA4WLx4MUePHiUgIIBJkyZ5ZYdDPEN5IOIeygPxdsoDEfdQHoi3cOugBMD999/PP/7xD95++2127tzJ7bffzpkzZ5gxYwYA06ZNq/Jgo9tvv53MzEzuuece9uzZw6JFi3jyySf5/e9/7+7SRUR8lr24lM03/B8b/vANWbsLAcjcWcj6e79k6y1/xeGBhzf5Ux6o0yHeKiYmhs6dOwPlz5FxN8MwWLJkCYcOHcJisTBx4sRq94oWUR6IuJ7yQHyB8kDE9ZQH4i3cfnS86qqrSE9P55FHHuHkyZP07duXr7/+uvIX8PDhw5jNP4+VtG3blm+++Yb77ruP3r17Ex8fzz333MODDz7o7tJFRHzW7j+/yukdpYAJDNP/ppZ/T99Swt6H/kGXZ+9wa03+lAfqdIg3GzRoEPv37+f48eMcOXKEtm3bumW7hmGwdOlS9u/fj9lsZvz48bRu3dot2xbfojwQcQ/lgXg75YGIeygPxBt45Oh45513cuedd9Y478cff6w2bejQoaxatcrFVYmI+Kfi9ExOrc7j3BfHmTjxUxYdsnIIjI50Z2l+kwfqdIg3CwsLo2fPnmzZsoXVq1fTpk0bTCZT7S9spJ9++ok9e/ZgMpm46KKL3NbZEd+kPBBxPeWB+ALlgYjrKQ/EG7j99k0iIuJep7/8CcPx8+E+taD84XFHCn6+f6TDbub0NyvdXpu/UKdDvF2/fv0ICgoiMzOTvXv3unx7q1atYseOHQBccMEFJCYmunybIt5AeSDeTnkg4h7KA/F2ygPxNA1KiIj4OXtRcZWfHYYBQLGj6nMkHEUlbqvJ31R0OiwWi4crEamZ1WqlX79+AKxdu7byd9YV1q9fz5YtWwAYNWoUnTp1ctm2RLyN8kC8nfJAxD2UB+LtlAfiaRqUEBHxcxF9k+q0XHifzi6uxH/pTCjxBT179iQsLIwzZ86wbds2l2xj8+bNrF+/HoBhw4bRtWtXl2xHxFspD8QXKA9EXE95IL5AeSCepEEJERE/Fzm4N9aQEsA4xxIGtrBiwvt1d2dZfsVutwPqdIh3s1gsDBw4EIBNmzZRVFTk1PVv376d1atXA5CcnEzPnj2dun4RX6A8EF+gPBBxPeWB+ALlgXiSBiVERPxc8ZETREZnYTI7qD4wYWA2O4iMzKLkRJonyvMLOhNKfEXnzp1p1qwZJSUlbNy40Wnr3b17Nz/99BMA/fv3p2/fvk5bt4gvUR6Ir1AeiLiW8kB8hfJAPEWDEiIifq740FECA8uIbZ1OaPgZTKbygQmTySA0/AyxrdMICLRTfOiYhyv1Xep0iK8wmUwMGTIEKD9zKS8vr9Hr3LdvH0uXLgWgV69elWdbiTRFygPxFcoDEddSHoivUB6Ip2hQQkTEz5kCy/8QDgiwE9U8h8joXABimp8hqnkOlgBHleWk/tTpEF/Spk0b4uPjcTgcrF27tlHrSk1NZcmSJQB069aNoUOHOqNEEZ+lPBBfojwQcR3lgfgS5YF4ggYlRET8XEiPLpjO+mO4c1RzADpERldOMwUFEdJdD7puKHU6xNcMHjwYKD+LKSMjo0HrOHr0KIsXL8YwDDp37syIESOcWaKIT1IeiK9RHoi4hvJAfI3yQNytwYMSFQ8qERER72YJCyFq/Egwmc65TPSk0ZiDbW6syr+o0yG+JiYmhk6dOgEN+5vuxIkTfPPNNzgcDjp06MDo0aMxnecYI9JUKA/E1ygPRFxDeSC+Rnkg7tbgQYmhQ4eSlJTEY489xoEDB5xZk4iIOFmL315C+LD+5T+Y//eHgaU8AiJGDybmyos9VJl/sNvtgDod4lsGDRqE2Wzm2LFjHD16tM6vS0tL4+uvv8Zut9OuXTsuvPBCzGZdfCsCygPxTcoDEedTHogvUh6IOzX4N2TBggV07tyZxx57jM6dOzN8+HBee+01MjMznVmfiIg4gSnAQuu7ZpDw5J8IG9AbgPCBvUl46kFa3X4dJovFwxX6toozoSzaj+JDwsPD6dGjBwCrVq3CMIxaX5ORkcGXX35JaWkp8fHxjBs3Th0OkbMoD8QXKQ9EnE95IL5IeSDu1ODfkt/+9rcsWrSI48eP8+KLL2IYBnfccQetW7fmkksuYeHChZSUlDizVhERaSRbh3ZETxgFQPSE0dgS23q4Iv+gy7PFV/Xr14+goCAyMzPZu3fveZfNysriyy+/pKSkhLi4OCZMmKCOtsgvKA/EVykPRJxLeSC+Snkg7tLooauYmBjuvPNOVq5cyd69e3nooYfYtWsXV111FXFxcdxyyy2sWLHCGbWKiIh4JXU6xFfZbDb69u0LwLp16ypvNfBLubm5LFq0iKKiIlq0aMHEiRP1+y5SA+WB+CrlgYhzKQ/EVykPxF2cej1NcHAwISEh2Gw2DMPAZDLx2WefMXr0aAYNGsSOHTucuTkRERGvoE6H+LKePXsSGhpKfn4+27ZtqzY/Pz+fL774goKCApo1a8bkyZMJCgryQKUi3k95IL5MeSDiPMoD8WXKA3GHRg9K5OXlMX/+fMaOHUtCQgKzZs2iffv2LFy4kJMnT3L8+HE+/PBD0tLSmDFjhjNqFhER8SrqdIgvCwgIYNCgQeRkFDFv9meUlpYyvdNC/jT2a757byeff/5f8vPziYqKYsqUKVitVk+XLOK1lAfiy5QHIs6jPBBfpjwQd2jw0fGzzz7j3Xff5YsvvqCoqIhBgwbxt7/9jauvvprmzZtXWfbyyy8nKyuL3//+940uWERExJs4HA4cDgegTof4rpK0CP79+D5KyKPb6HByM2PZtOwI3/zwOR2TQ7jsrv5MmTKF4OBgT5cq4rWUB+IPlAcijac8EH+gPBBXa/DR8Te/+Q1t27blvvvuY9q0aXTp0uW8y/fp04drr722oZsTERHxSmffY1OdDvFFxYVlzLl0CSGlCRRbtnL48GFKjBDS7Lsoo4B9a+wEpHUhNDTU06WKeDXlgfg65YGIcygPxNcpD8QdGnz7ph9++IFDhw7xxBNP1DogAZCcnMz8+fMbujkRERGvVHFpNoDFYvFgJSINs+yjVPIyi7EZzbGaonA4HJwsW0+pkY+ZIFqY+/D1a6kYhuHpUkW8mvJAfJ3yQMQ5lAfi65QH4g4NHpQYM2aME8sQERHxTbpfrPi6HSlpWAJMAERZOvxvqoGJAFqYehNACMf355GXWey5IkV8gPJAfJ3yQMQ5lAfi65QH4g4NHpT4y1/+Qt++fc85v1+/fjz66KMNXb2IiIhPUKdDfJ3JbKr8t9UUQUJCAhZstDD1IdAUVjnPbDHV9HIR+R/lgfg65YGIcygPxNcpD8QdGjwosXDhQiZNmnTO+ZMnT+bDDz9s6OpFRER8gjod4uv6XdQKe9nPl1537dqV+MAhBJnCATCZoUOfaMKirJ4qUcQnKA/E1ykPRJxDeSC+Tnkg7tDgQYnDhw/TsWPHc85PTEzk0KFDDV29iIiIT1CnQ3zd0KntiG0Xes4znQwHXPnHXm6uSsT3KA/E1ykPRJxDeSC+Tnkg7tDgQYmwsLDzDjocPHgQm83W0NWLiIj4BLvdDqjTIb4rINDME4vGERljxXRWv6PiPrJX/7k3Y65K9FB1Ir5DeSC+Tnkg4hzKA/F1ygNxhwYfIceMGcPrr7/ObbfdRnx8fJV5R44c4e9//zsXXHBBowsUERHxZhVnQlksFg9XItJw7bpF8caOS1m8YA+wnw59okno2pyLb+1Kl0Exni5PxCcoD8QfKA9EGk95IP5AeSCu1uBBiccee4zk5GR69OjBjTfeSI8ePQDYtm0bb775JoZh8NhjjzmtUBERaRyjLBfSP8BxYD7QD8eBP2EkXg+xV2OyhNX2cjkHXZ4t/iI0MojJt3Thyy/389ySyQQGBnq6JBGfojwQf6E8EGkc5YH4C+WBuFKDj5BdunRh+fLl3HXXXbzwwgtV5o0aNYqXXnqJbt26NbpAERFpPKMkDWPXb6H4GJSWlE8sOYlx9K+Q8RF0fQ9TYHPPFumj1OkQERFQHoiISDnlgYhI7Rp1hOzduzdLly4lIyODAwcOANChQwdiYnQZj4iINzEOzoLi44Djl3Og6AhG6sOYOr/iidJ8njodIiICygMRESmnPBARqZ1TjpAxMTEaiBAR8VJG0WHIXX6eJeyQ/QNG8XFM1tZuq8tfqNMhIiKgPBARkXLKAxGR2jX6CHn06FE2btxITk4ODscvz8CFadOmNXYTIiLSGGe2VPlx2/5iAA4eL2FYr+D/TTXgzFbQoES9qdMhIiKgPBARkXLKAxGR2jX4CFlUVMT06dP5+OOPcTgcmEwmDMMAwGQyVS6nQQkREQ8zmSv/eTy9rPLfuWd+MZBssrirIr9S0emwWLT/RESaMuWBiIiA8kBEpC7MtS9Ss1mzZvHJJ5/wxBNP8OOPP2IYBm+//TbffvstkyZNok+fPmzevNmZtYqISEOEJwMWCosc/LCuoOZlTIEQPsCtZfkLnQklIiKgPBARkXLKAxGR2jV4UGLhwoXMmDGDBx98kB49egAQHx/P2LFj+eKLL4iKimLevHlOK1RERBrGFBiD0fxXLFlfREFR9dvsgRliLscUEO322vyB3W4H1OkQEWnqlAciIgLKAxGRumjwoERaWhrJyckABAeX35P8zJkzlfMvu+wyPvnkk0aWJyIizrA1+zcczY3HYjYBFbfY+9/3iGGY2v3ZU6X5PJ0JJSIioDwQEZFyygMRkdo1eFCiZcuWnD59GoCQkBCio6PZvXt35fzc3FyKiooaX6GIiDRKWloa6zZsxdTyekb86hkI7lI+I7gLps5/x5T0D0xmm2eL9GHqdIiICCgPRESknPJARKR2DT5CDh48mBUrVvDggw8CMHXqVP7v//6PVq1a4XA4eOGFFxgyZIjTChURkforLi5m8eLFOBwOOnXqTLfki/hxfQGc3Iq5xWWYokZ7ukSfp06HiIiA8kBERMopD0REatfgKyXuvvtuOnToQHFxMQCPPfYYUVFRXHfddUyfPp3IyEheeuklpxUqIiL1t2zZMvLz84mIiGDkyJGeLscvqdMhIiKgPBARkXLKAxGR2jX4CDlixAhGjBhR+XPbtm3ZuXMnW7duxWKx0LVrVx2ARUQ8aMeOHRw8eBCz2cxFF11EUFCQp0vyS+p0iIgIKA9ERKSc8kBEpHYNulKioKCASy+9lHfffbfqysxm+vTpQ8+ePXXwFRHxoNOnT5OSkgKU326vRYsWlfMMh73Kd2kcu718Pyr3RESaNuWBiIiA8kBEpC4aNCgREhLC4sWLKSgocHY9IiLSSKWlpSxevBi73U67du3o1asXAEbeMew/PYVj3SsAONa/in3lsxj5Jz1Zrs+rOBPKYrF4uBIREfEk5YGIiIDyQESkLhr8TIkRI0ZUnoUrIiLe46effiInJ4fQ0FDGjBkDgJF9EPt/b8DY9xUY/7tCwlGGsfcL7P+dgZFz2HMF+zhdni0iIqA8EBGRcsoDEZHaNXhQYu7cuSxfvpy//OUvHD161Jk1iYhIA+3Zs4c9e/ZgMpm46KKLsNlsANhXPAGlBT8PSFQw7FCSj33l0x6o1j+o0yEiIqA8EBGRcsoDEZHaNXhQok+fPhw9epSnnnqKhIQErFYrERERVb4iIyOdWauIiJxHdnY2K1asAGDAgAHExcUBYGTuhYydYDhqfqHhgFObMbJT3VSpf1GnQ0REQHkgIiLllAciIrVr8BHysssuw2QyObMWERFpILvdzuLFiykrK6N169b069evcp6RdaBO6zCyD2KKau+iCv1TRYcD1OkQEWnKlAciIgLKAxGRumrwEfKtt95yYhkiItIYKSkpZGZmYrPZuPDCC6sOGgfYqiw7okskO47A6K5RVVcSYHV9oX7m7E6HHmQnItJ0KQ9ERASUByIiddXg2zeJiIh3OHjwIDt27ADgggsuICQkpMp8U6uBYAmq/LlLq1AmTJhAp7izlgsIxtSyH1I/dnv5MzrMZjNmsyJVRKSpUh6IiAgoD0RE6qrBV0r861//qtNy06ZNa+gmRESkFnl5eSxduhSAvn370rZt22rLmIJCMXW/CmPrO+dcj7nnbzEFBrusTn+l+8WKiAgoD0REpJzyQESkbhp8lLz++uvPOe/s24ZoUEJExDUcDgfff/89JSUltGzZkoEDB55zWXO/m3EU52Ds+RxM/7uM2GQBA0xdL8PU53r3FO1n1OkQERFQHoiISDnlgYhI3TT4KHnw4MFq0+x2O6mpqbzyyiscPnyYt99+u1HFiYjIua1du5a0tDSCgoK46KKLznt5sMlswTLsQYzuV1G29xs4DqYeV2LpPAFTZIIbq/Yv6nSIiAgoD0REpJzyQESkbhp8lExIqPk/sTp06MCFF17IlClTmDt3LvPmzWtwcSIiUrMjR46wefNmAEaPHk1YWFidXmeKao+l7w1w/EssfWZgCgx0ZZl+T50OEREB5YGIiJRTHoiI1I3Lnrpz8cUX8+GHH7pq9SIiTVZBQQFLliwBoEePHiQmJnq4oqZLnQ4REQHlgYiIlFMeiIjUjcsGJfbv309xcbGrVi8i0iQZhsEPP/xAUVERzZs3Z8iQIZ4uqUmz2+2AOh0iIk2d8kBEREB5ICJSVw0+Si5btqzG6dnZ2SxbtoyXXnqJSy65pKGrFxGRGmzcuJHjx48TEBDA2LFjsVgsni6pSas4E0rvg4hI06Y8EBERUB6IiNRVgwclxowZg8lkqjbdMAwsFgtXXHEFL7/8cqOKExGRn504cYL169cDMHLkSCIjIz1ckejybBERAeWBiIiUUx6IiNRNg4+SFfczP5vJZCI6OpqEhAQiIiIaVZiIiPysqKiI77//HsMwSEpKonPnzp4uSVCnQ0REyikPREQElAciInXV4KPk6NGjnVmHiIicx48//khBQQFRUVEMHz7c0+XI/6jTISIioDwQEZFyygMRkbpp8IOuDx48yH//+99zzv/vf/9LampqQ1cvIiL/s2XLFg4fPozFYmHs2LEEBgZ6uiT5H3U6REQElAciIlJOeSAiUjcNHpR44IEHeOmll845f968efz5z38+57z27dtjs9kYPHgwa9asqdM2P/jgA0wmkx6gLSJNRnp6euUxcujQoTRr1szDFTmXr+eBOh0iIs6hPBAREVAeiIg0FQ0elEhJSWHcuHHnnH/RRRexfPnyatM//PBD7r//fmbPns2GDRvo06cPEyZMIC0t7bzbS01N5YEHHmDkyJENLVlExKeUlJSwePFiHA4HiYmJdO/e3dMlOZU/5EFFp8NisXi4EhER36U8EBERUB6IiDQlDR6UyMrKIjw8/Jzzw8LCOH36dLXpzz//PDfffDMzZsyge/fuvPbaa4SEhPDmm2+ec112u51rr72WRx99lA4dOjS0ZBERn7Js2TLy8vIIDw/3y+f4+EMe6EwoEZHGUx6IiAgoD0REmpIGHyXbtWvHTz/9xO23317j/OXLl9OmTZsq00r+f3t3HyVlfd+N/7277AOICJYnH1AixmB8omKxaIxRUVJ722CbxBMbNcakxoc2CamNGhO01mByG485icbG5/Znbq02pr0TiyJCc6uoUaE1FVGjaI0FJRFBiLDsXr8/JrtKQN2Z3Z1hmdfrHM5hrrlm5vNZdq63Xz/XzLVhQx599NGcd9553dsaGxszbdq0LFy48G1f62//9m8zevTonHbaaVv89MXvWr9+fdavX999e/Xq1UmS9vb2tLe3v+vj36pr/3IfN9DpW9/1YGvue8mSJXn66afT2NiYD37wg2loaOizOnvbd1/Usa3kwfr169PR0ZGiKLbK36O+sDW/T/qTvvVdD+SBPCiH94m+64G+5UE55MG22d/b0be+60E186DiocQnPvGJXHzxxZkyZUrOPvvsNDaWPnTR0dGR7373u7n11lvzla98ZZPHrFy5Mh0dHRkzZswm28eMGZMnn3xyi69z33335brrrsvixYt7XNvs2bNz0UUXbbb97rvvzpAhQ3r8PG81d+7cih430Om7vuh767BmzZo8+OCD6ezszF577ZVHHnmkX16n0r7XrVvX69feVvLg0UcfzcqVK1MURZYtW1bR8w0UW9v7pFr0XV/0XR55IA/qib7ri77LIw/kQT3Rd33Rd3nKyYOKhxLnnXde7rvvvnzhC1/IJZdckve9731JkqVLl+aVV17Jhz70oc2GEuVas2ZNTjrppFxzzTUZOXJkWbXNnDmz+/bq1aszbty4HHPMMRk2bFhZNbS3t2fu3Lk5+uij09zcXNZjBzJ967sebI19b9y4MXfccUf22Wef7Lrrrpk+fXoaGhr69DV623fX2UTVtLXmQWdnZ5YvX54jjzxyq/i4eH/YGt8n1aBvfdcDeSAPyuF9ou96oG95UA554H1SD/St73KUkwcVDyVaW1tz991356abbsoPf/jD/OIXv0iSTJkyJX/2Z3+Wk08+ufvTE11GjhyZpqamrFixYpPtK1asyNixYzd7jV/84hdZtmxZjjvuuO5tnZ2dpcIHDcrSpUszYcKELdbW2tq62fbm5uaKf5F689iBTN/1Rd+198ADD+T111/P9ttvn6OPPjotLS399lqV9t0XP6ttKQ+ampoyePDgreZ3qL9sTe+TatJ3fdF3+Y/rLXkw8Hif1Bd91xd5IA/K4X1SX/RdX6qRB7268k5jY2NOPfXUnHrqqT3av6WlJZMnT868efMyY8aMJKXQmDdvXs4+++zN9p84cWIef/zxTbZdcMEFWbNmTb797W9n3LhxvSkfYKvyzDPPZOnSpWloaMiRRx6Ztra2WpfUb7aVPHAhO4DekQcAJPIAoN5UfJT89a9/nRdffDH777//Fu9//PHHs+uuu2bEiBGbbJ85c2ZOOeWUHHTQQZkyZUquuOKKrF27tnuwcfLJJ2eXXXbJ7Nmz09bWln333XeTxw8fPjxJNtsOMJC99tpr3RdmO/DAA7PzzjvXuKL+ty3kgUUHQO/JAwASeQBQTyo+Sn7xi1/M0qVL8+CDD27x/tNPPz177713rrvuuk22n3DCCXnllVfyta99LcuXL8+kSZMyZ86c7osZvfDCC5t97RPAtqyjoyPz5s1Le3t7dtpppxx44IG1LqkqtoU86OjoSGLRAdAb8gCARB4A1JOKj5L33ntvzjjjjLe9/7jjjsvVV1+9xfvOPvvsLX78LkkWLFjwjq9744039rREgAHhoYceysqVK9PW1pYjjzyyzy9svTUb6HnQdSZUU1NTjSsBGNjkAQCJPACoFxWPmF955ZWMHDnybe//vd/7vbz88suVPj1AXVi2bFl+/vOfJ0k+9KEPZbvttqtxRZTDx7MBSOQBACXyAKBnKh5K7LTTTlm0aNHb3v/oo49m1KhRlT49wDbv9ddfz7//+78nSfbff//stttuNa6IcnR2dqazszOJRQdAPZMHACTyAKAcFQ8lZsyYkeuuuy7/+q//utl9//Iv/5Ibbrghxx9/fK+KA9hWdXZ2Zt68eVm/fn1Gjx6dKVOm1LokytR1FlRi0QFQz+QBAIk8AChHxUfJCy+8MPfcc0+OP/74HHDAAdl3332TJD//+c/zH//xH9l7771z0UUX9VmhANuSRx55JCtWrEhLS0uOPPLIrf6CbWzurYsO3xkLUL/kAQCJPAAoR8X/F2yHHXbIgw8+mAsuuCDt7e25/fbbc/vtt6e9vT1f/epX89BDD2X48OF9WCrAtuHFF1/M4sWLkyQf/OAHM2zYsNoWREV8XywAiTwAoEQeAPRcr46U2223XS666CKfiADooXXr1mX+/PlJkr333jt77LFHjSuiUh0dHUksOgDqnTwAIJEHAOXwfSEAVVIURebPn5/f/OY32XHHHTN16tRal0QvOBMKgEQeAFAiDwB6rldHyjfeeCP//M//nMceeyyvvfZaOjs7N7m/oaEh1113Xa8KBNhWLF68OL/85S8zaNCgTJs2zX+sDnAWHQAk8gCAEnkA0HMVHymff/75HHHEEVm2bFmGDx+e1157LTvuuGNWrVqVjo6OjBw5MkOHDu3LWgEGrOXLl+eRRx5Jkhx66KGuubMNsOgAIJEHAJTIA4Ceq/jrm84555y89tprefDBB/PUU0+lKIrceuutef311/ONb3wjgwcPzl133dWXtQIMSOvXr8+9996boiiy55575n3ve1+tS6IPWHQAkMgDAErkAUDPVTyUuPfee3PmmWdmypQpaWwsPU1RFGltbc0555yTo446Kl/4whf6qk6AAWvBggV5/fXXs8MOO+Swww6rdTn0EYsOABJ5AECJPADouYqHEuvWrcv48eOTJMOGDUtDQ0Nee+217vunTp2a++67r9cFAgxkP//5z/P888+nsbExRx11VJqbm2tdEn2ko6MjSdLU1FTjSgCoJXkAQCIPAMpR8VBit912y4svvpikNAXeZZdd8uCDD3bf/8QTT6Stra33FQIMUCtXruw+Lk6dOjUjR46scUX0JWdCAZDIAwBK5AFAz1V8pDzyyCPzL//yL5k1a1aS5FOf+lRmz56dV199NZ2dnfnHf/zHnHzyyX1WKMBA0t7ennvuuSednZ0ZP3589tlnn1qXRB+z6AAgkQcAlMgDgJ6r+Eh57rnn5mc/+1nWr1+f1tbWnH/++XnppZdy++23p6mpKSeeeGIuv/zyvqwVYMD4f//v/2X16tUZOnRoDj/88FqXQz+w6AAgkQcAlMgDgJ6r+Ei52267Zbfdduu+3dbWlmuvvTbXXnttnxQGMFAtXbo0zzzzTBoaGnLUUUeltbW11iXRDyw6AEjkAQAl8gCg5yq+pgQAm3v11Vdz//33J0n+4A/+IGPGjKlxRfQXiw4AEnkAQIk8AOg5QwmAPrJx48bMmzcvGzduzK677poDDjig1iXRjyw6AEjkAQAl8gCg5wwlAPrIwoUL8+tf/zpDhgzJEUcckYaGhlqXRD/qWnQ0NTXVuBIAakkeAJDIA4ByGEoA9IFnn302S5YsSZIcccQRGTx4cI0ror85EwqARB4AUCIPAHrOUAKgl1avXp2f/vSnSZLf//3fzy677FLjiqiGjo6OJBYdAPVOHgCQyAOAchhKAPRCZ2dn5s2blw0bNmTs2LGZPHlyrUuiSpwJBUAiDwAokQcAPWcoAdALDz30UF555ZW0trbmyCOPTGOjw2q9sOgAIJEHAJTIA4Ce83/PACr0wgsv5PHHH0+SfOhDH8rQoUNrXBHVZNEBQCIPACiRBwA9ZygBUIG1a9dm/vz5SZJ99903u+++e40rotosOgBI5AEAJfIAoOcMJQDK1HUdifXr12fkyJE5+OCDa10SNWDRAUAiDwAokQcAPWcoAVCmxx57LMuXL09zc3OmTZuWpqamWpdEDXR0dCSx6ACod/IAgEQeAJTDUAKgDC+99FIee+yxJMlhhx2WYcOG1bgiaqEoiu4zoQylAOqXPAAgkQcA5TKUAOih3/zmN7n33nuTJBMnTsyee+5Z44qola6zoBJnQgHUM3kAQCIPAMplKAHQA0VRZP78+Vm3bl1GjBiRQw45pNYlUUNdZ0ElFh0A9UweAJDIA4ByGUoA9MB//ud/5sUXX8ygQYMybdo0/6FZ57oWHY2NjWloaKhxNQDUijwAIJEHAOUylAB4Fy+//HJ+9rOfJUkOOeSQjBgxosYVUWtdiw7DKYD6Jg8ASOQBQLkMJQDewfr163PPPfeks7MzEyZMyMSJE2tdElsBiw4AEnkAQIk8ACiPoQTAO/jpT3+a119/PcOGDcthhx1W63LYSnRdyM6iA6C+yQMAEnkAUC5DCYC38cQTT+S5555LY2NjjjrqqLS0tNS6JLYSzoQCIJEHAJTIA4DyGEoAbMGvfvWrLFy4MEly8MEHZ9SoUTWuiK2JRQcAiTwAoEQeAJTHUALgd7S3t+eee+5JR0dHdt999+y33361LomtjEUHAIk8AKBEHgCUx9ESqHudxf9kQ8eiJA0Z1PgHuf/+pXnttdey3Xbb5fDDD691eWyFLDoASOQBACXyAKA8jpZA3eosViVJXm//cAbljSTJM0va8/iSCWlt+tMcddSfpK2trYYVsrWy6AAgkQcAlMgDgPL4+iagLhXFG1nXflrXrSTJqlUdWfjAG+ksnsjek27PmDEjalcgW7WOjo4kSVNTU40rAaCW5AEAiTwAKJehBFCX2jvvSGee7r69cWORf7/3jXRsLDJ2p8bsd8B/p73zxzWskK2ZM6EASOQBACXyAKA8hhJAXdrQeVuShu7bP3tofV79dUda2xrywSPa0tDQ9Nt9YHMWHQAk8gCAEnkAUB5DCaAudRYr0vW1TStWbMzSJRuSJB/80OAMGdKYpDNFsaJ2BbJVs+gAIJEHAJTIA4DyOFoCdamxYackq5IkY8YMyh8e2pbfrCuyy65dh8XGNDTsVKvy2MpZdACQyAMASuQBQHl8UgKoSy2NH0/S2X174t4t+f3JrW/Zo/O3+8DmLDoASOQBACXyAKA8hhJAXWpu/EiaGvZ5m3sb09Tw+2luPLaqNTFwdC06mpqaalwJALUkDwBI5AFAuQwlgLrU0NCaIYO+/9tbbz0UNqW5YUa2G3RTGhpaalEaA4AzoQBI5AEAJfIAoDyOlkDdamjYPkkytPmeNA56PElDmhp+P40NI2tbGFs9iw4AEnkAQIk8ACiPoyVQ9xobRqa58Zhal8EA0tHRkcSiA6DeyQMAEnkAUC5f3wQAZXImFACJPACgRB4AlMdQAgDKZNEBQCIPACiRBwDlMZQAgDJZdACQyAMASuQBQHkMJQCgTBYdACTyAIASeQBQHkMJACiTRQcAiTwAoEQeAJTHUAIAytDZ2ZmiKJJYdADUM3kAQCIPACphKAEAZeg6CypJmpqaalgJALUkDwBI5AFAJWoylLjyyiszfvz4tLW15eCDD87DDz/8tvtec801OeywwzJixIiMGDEi06ZNe8f9ARg4BmIedC06GhoaLDoA+og8ACCRBwD1oupDiVtvvTUzZ87MrFmz8thjj+WAAw7I9OnT8/LLL29x/wULFuQTn/hE5s+fn4ULF2bcuHE55phj8stf/rLKlQPQlwZqHvi+WIC+JQ8ASOQBQD2p+lDi8ssvz2c/+9mceuqpef/735+rr746Q4YMyfXXX7/F/W+++eaceeaZmTRpUiZOnJhrr702nZ2dmTdvXpUrB6AvDdQ86Fp0OAsKoG/IAwASeQBQT6o6lNiwYUMeffTRTJs27c0CGhszbdq0LFy4sEfPsW7durS3t2fHHXfsrzIB6GcDOQ+cCQXQd+QBAIk8AKg3VT1irly5Mh0dHRkzZswm28eMGZMnn3yyR8/x5S9/OTvvvPMmQfW71q9fn/Xr13ffXr16dZKkvb097e3tZdXctX+5jxvo9K3veqDvyvrui5/XQM6DN954Ix0dHZts25Z5n+i7HuhbHpRDHmz7vb6VvvVdD+SBPCiH94m+64G++z8PBtQY99JLL80tt9ySBQsWpK2t7W33mz17di666KLNtt99990ZMmRIRa89d+7cih430Om7vui7vlTa97p16/q4kvLVMg/mz5+fxx9/PMOGDct2221X0XMMRN4n9UXf9UUeyINyeJ/UF33XF3kgD8rhfVJf9F1fqpEHVR1KjBw5Mk1NTVmxYsUm21esWJGxY8e+42Mvu+yyXHrppbnnnnuy//77v+O+5513XmbOnNl9e/Xq1d0XPBo2bFhZNbe3t2fu3Lk5+uij09zcXNZjBzJ967se6LuyvrvOJuqNgZwHhxxySNrb2zNmzJgce+yxZT3HQOR9ou96oG95UA554H1SD/St73LIA3lQD/St73pQzTyo6lCipaUlkydPzrx58zJjxowk6b4I0dlnn/22j/vmN7+ZSy65JHfddVcOOuigd32d1tbWtLa2bra9ubm54l+k3jx2INN3fdF3fam07774WQ3kPEhKF7Fra2urq98b75P6ou/6Ig/kQTm8T+qLvuuLPJAH5fA+qS/6ri/VyIOqf33TzJkzc8opp+Sggw7KlClTcsUVV2Tt2rU59dRTkyQnn3xydtlll8yePTtJ8o1vfCNf+9rX8oMf/CDjx4/P8uXLkyRDhw7N0KFDq10+AH1koOaBC9kB9C15AEAiDwDqSdWPmCeccEJeeeWVfO1rX8vy5cszadKkzJkzp/tiRi+88EIaGxu79//e976XDRs25KMf/egmzzNr1qxceOGF1SwdgD40UPPAogOgb8kDABJ5AFBPanLEPPvss9/243cLFizY5PayZcv6vyAAamIg5oFFB0DfkwcAJPIAoF40vvsuAECXzs7OJKXvjQWgfskDABJ5AFAJQwkAKIMzoQBI5AEAJfIAoHyGEgBQBosOABJ5AECJPAAon6EEAJTBogOARB4AUCIPAMpnKAEAZbDoACCRBwCUyAOA8hlKAEAZLDoASOQBACXyAKB8hhIAUIauRUdTU1ONKwGgluQBAIk8AKiEoQQAlMGZUAAk8gCAEnkAUD5DCQAog0UHAIk8AKBEHgCUz1ACAMrQ0dGRxKIDoN7JAwASeQBQCUMJACiDM6EASOQBACXyAKB8hhIAUAaLDgASeQBAiTwAKJ+hBACUwcezAUjkAQAl8gCgfIYSANBDRVFYdAAgDwBIIg8AKmUoAQA91NnZ2f13iw6A+iUPAEjkAUClDCUAoIe6zoJKkqamphpWAkAtyQMAEnkAUClDCQDooa5FR1NTUxoaGmpcDQC1Ig8ASOQBQKUMJQCgh3xfLACJPACgRB4AVMZQAgB6qOs7Yy06AOqbPAAgkQcAlTKUAIAeeuvHswGoX/IAgEQeAFTKUAIAesjHswFI5AEAJfIAoDKGEgDQQxYdACTyAIASeQBQGUMJAOgh3xkLQCIPACiRBwCVMZQAgB6y6AAgkQcAlMgDgMoYSgBAD/l4NgCJPACgRB4AVMZQAgB6yKIDgEQeAFAiDwAqYygBAD1k0QFAIg8AKJEHAJUxlACAHvKdsQAk8gCAEnkAUBlDCQDooa5FR1NTU40rAaCW5AEAiTwAqJShBAD0kI9nA5DIAwBK5AFAZQwlAKCHLDoASOQBACXyAKAyhhIA0EMWHQAk8gCAEnkAUBlDCQDoIYsOABJ5AECJPACojKEEAPRQ14XsLDoA6ps8ACCRBwCVMpQAgB7qOhOqqampxpUAUEvyAIBEHgBUylACAHrIx7MBSOQBACXyAKAyhhIA0EMWHQAk8gCAEnkAUBlDCQDoId8ZC0AiDwAokQcAlTGUAIAesugAIJEHAJTIA4DKGEoAQA/5eDYAiTwAoEQeAFTGUAIAeqCzszNFUSSx6ACoZ/IAgEQeAPSGoQQA9MDGjRu7/27RAVC/5AEAiTwA6A1DCQDoga5FR0NDQxobxSdAvZIHACTyAKA3HDUBoAe6Fh3OggKob/IAgEQeAPSGoQQA9EDXoqOpqanGlQBQS/IAgEQeAPSGoQQA9IAzoQBI5AEAJfIAoHKGEgDQA86EAiCRBwCUyAOAyhlKAEAPOBMKgEQeAFAiDwAqZygBAD1g0QFAIg8AKJEHAJUzlACAHrDoACCRBwCUyAOAyhlKAEAPdHR0JLHoAKh38gCARB4A9IahBAD0gDOhAEjkAQAl8gCgcoYSANADFh0AJPIAgBJ5AFA5QwkA6AEfzwYgkQcAlMgDgMoZSgBADzgTCoBEHgBQIg8AKmcoAQA9YNEBQCIPACiRBwCVM5QAgHfQvn5dfv2PX8iK730zSbLqxkvyxjNza1wVANUmDwBI5AFAX6jJUOLKK6/M+PHj09bWloMPPjgPP/zwO+5/2223ZeLEiWlra8t+++2XO++8s0qVAtCftvY8eOOpu/PC6Z/PKz/ZmLW/ai1te2Fonr/gR3nl8k/362sD1BN5AEAiDwDqRdWHErfeemtmzpyZWbNm5bHHHssBBxyQ6dOn5+WXX97i/g888EA+8YlP5LTTTsuiRYsyY8aMzJgxIz//+c+rXDkAfWlrz4P29vb88uu3ZeMbLUmSjUWRJGlqbEqS/PrhwXn15pn98toA9UQeAJDIA4B6UvWhxOWXX57PfvazOfXUU/P+978/V199dYYMGZLrr79+i/t/+9vfzoc//OGcc8452XvvvXPxxRfnwAMPzHe/+90qVw5AX9ra8+D1f/pyNr7RmqQhSdLR+dtFR0ND97ZV81/rl9cGqCfyAIBEHgDUk6pejWfDhg159NFHc95553Vva2xszLRp07Jw4cItPmbhwoWZOXPTSfP06dPzox/96G1fZ/369Vm/fn337dWrVycpTbXb29vLqrlr/3IfN9DpW9/1QN+V9d0XP6+BkAerF69Kx6Ch6VpgtP92jN/Q3JyOQaWzoX7zxtCs/Z8n0jLyve/4XAOZ94m+64G+5cE7kQcl3if6rgf6lgfvRB6UeJ/oux7ou//zoKpDiZUrV6ajoyNjxozZZPuYMWPy5JNPbvExy5cv3+L+y5cvf9vXmT17di666KLNtt99990ZMmRIBZUnc+fW50WL9F1f9F1fKu173bp1vX7tAZEHH/jIJjdfeuCBZM2a/PLYQ/LG7/1e9/anH346ydPv/FzbAO+T+qLv+iIP5EE5vE/qi77rizyQB+XwPqkv+q4v1ciDqg4lquW8887bZFq+evXqjBs3Lsccc0yGDRtW1nO1t7dn7ty5Ofroo9Pc3NzXpW619K3veqDvyvruOptoIOhNHqz436dn9RNvngk1YuOGPH30Qdl//uIMKUrbGpo6svtVX09zy+B+66HWvE/0XQ/0LQ/eiTwo8T7Rdz3Qtzx4J/KgxPtE3/VA3/2fB1UdSowcOTJNTU1ZsWLFJttXrFiRsWPHbvExY8eOLWv/JGltbU1ra+tm25ubmyv+RerNYwcyfdcXfdeXSvvui5/VQMiDUX/+2aw99/9LiiRpyMhBLXl15MgMKRrStLEjSZHtd1+dIduVN+weqLxP6ou+64s8kAfl8D6pL/quL/JAHpTD+6S+6Lu+VCMPqnqh65aWlkyePDnz5s3r3tbZ2Zl58+Zl6tSpW3zM1KlTN9k/KX2E5O32B2DrNxDyoHX3qfm9Qzb89lbxlnuKJEUGDV6fkV/a/KPfAPScPAAgkQcA9aaqQ4kkmTlzZq655prcdNNNWbJkSc4444ysXbs2p556apLk5JNP3uTCRp///OczZ86cfOtb38qTTz6ZCy+8MI888kjOPvvsapcOQB8aCHkw8i+vyZg/3T4tw36TroVHY1Nnhu25OuO+dX5adtyj314boF7IAwASeQBQT6p+TYkTTjghr7zySr72ta9l+fLlmTRpUubMmdN9caIXXnghjY1vzkoOOeSQ/OAHP8gFF1yQ888/P+9973vzox/9KPvuu2+1SwegDw2UPBj+8Usz/OPJul89n6fvW5zdvvfNd78IHgA9Jg8ASOQBQD2pyYWuzz777LedXC9YsGCzbR/72MfysY99rJ+rAqDaBlIeNA/bOcniuvw+SYD+Jg8ASOQBQL2o+tc3AQAAAAAA9clQAgAAAAAAqApDCQAAAAAAoCoMJQAAAAAAgKowlAAAAAAAAKrCUAIAAAAAAKgKQwkAAAAAAKAqDCUAAAAAAICqMJQAAAAAAACqYlCtC6iGoiiSJKtXry77se3t7Vm3bl1Wr16d5ubmvi5tq6VvfdcDfVfWd9extOvYOpDIg/LpW9/1QN/yoBx+X/RdD/St73LIA78v9UDf+q4H1cyDuhhKrFmzJkkybty4GlcCsO1Ys2ZNdthhh1qXURZ5AND35AEAiTwAoKQnedBQDMRRdpk6Ozvz0ksvZfvtt09DQ0NZj129enXGjRuX//7v/86wYcP6qcKtj771XQ/0XVnfRVFkzZo12XnnndPYOLC+BVAelE/f+q4H+pYH5fD7ou96oG99l0Me+H2pB/rWdz2oZh7UxSclGhsbs+uuu/bqOYYNG1ZXv4Rd9F1f9F1fetP3QDsDqos8qJy+64u+64s8qIzfl/qi7/qi7/LJA78v9ULf9UXf5etpHgysETYAAAAAADBgGUoAAAAAAABVYSjxLlpbWzNr1qy0trbWupSq0re+64G+66vv3qrXn5u+9V0P9F1fffdWvf7c9K3veqDv+uq7t+r156ZvfdcDffd/33VxoWsAAAAAAKD2fFICAAAAAACoCkMJAAAAAACgKgwlAAAAAACAqjCUAAAAAAAAqsJQIsmVV16Z8ePHp62tLQcffHAefvjhd9z/tttuy8SJE9PW1pb99tsvd955Z5Uq7Vvl9H3NNdfksMMOy4gRIzJixIhMmzbtXX9OW6ty/7273HLLLWloaMiMGTP6t8B+Um7fq1atyllnnZWddtopra2t2WuvvQbk73q5fV9xxRV53/vel8GDB2fcuHH54he/mDfeeKNK1faNn/70pznuuOOy8847p6GhIT/60Y/e9TELFizIgQcemNbW1uy555658cYb+73OrZE8kAc9IQ/kwUAhDyonD+RBT8gDeTBQyIPKyQN50BPyQB4MFFtVHhR17pZbbilaWlqK66+/vviv//qv4rOf/WwxfPjwYsWKFVvc//777y+ampqKb37zm8UTTzxRXHDBBUVzc3Px+OOPV7ny3im37xNPPLG48sori0WLFhVLliwpPvWpTxU77LBD8eKLL1a58t4pt+8uzz33XLHLLrsUhx12WPGRj3ykOsX2oXL7Xr9+fXHQQQcVxx57bHHfffcVzz33XLFgwYJi8eLFVa68d8rt++abby5aW1uLm2++uXjuueeKu+66q9hpp52KL37xi1WuvHfuvPPO4itf+Urxwx/+sEhS3HHHHe+4/7PPPlsMGTKkmDlzZvHEE08U3/nOd4qmpqZizpw51Sl4KyEP5IE82Jw8kAfyQB7Igy2TB/JgIJEHlZEH8kAebE4eyIO+yoO6H0pMmTKlOOuss7pvd3R0FDvvvHMxe/bsLe7/8Y9/vPjjP/7jTbYdfPDBxemnn96vdfa1cvv+XRs3biy233774qabbuqvEvtFJX1v3LixOOSQQ4prr722OOWUUwZkyJTb9/e+971ijz32KDZs2FCtEvtFuX2fddZZxZFHHrnJtpkzZxaHHnpov9bZn3oSMn/zN39T7LPPPptsO+GEE4rp06f3Y2VbH3lQIg/kwVvJgzfJg/ohD0rkgTx4K3nwJnlQP+RBiTyQB28lD94kD3qnrr++acOGDXn00Uczbdq07m2NjY2ZNm1aFi5cuMXHLFy4cJP9k2T69Olvu//WqJK+f9e6devS3t6eHXfcsb/K7HOV9v23f/u3GT16dE477bRqlNnnKun7X//1XzN16tScddZZGTNmTPbdd998/etfT0dHR7XK7rVK+j7kkEPy6KOPdn9k79lnn82dd96ZY489tio118q2cFzrLXkgD+SBPHgrefCmgXZc6y15IA/kgTx4K3nwpoF2XOsteSAP5IE8eCt58Ka+Oq4N6vUzDGArV65MR0dHxowZs8n2MWPG5Mknn9ziY5YvX77F/ZcvX95vdfa1Svr+XV/+8pez8847b/aLuTWrpO/77rsv1113XRYvXlyFCvtHJX0/++yzuffee/Pnf/7nufPOO/PMM8/kzDPPTHt7e2bNmlWNsnutkr5PPPHErFy5Mh/4wAdSFEU2btyYz33uczn//POrUXLNvN1xbfXq1fnNb36TwYMH16iy6pEH8iCRB1siD+SBPCiRB+9OHgwc8kAevBt5IA/kQYk82Jw8kAd9lQd1/UkJKnPppZfmlltuyR133JG2trZal9Nv1qxZk5NOOinXXHNNRo4cWetyqqqzszOjR4/O97///UyePDknnHBCvvKVr+Tqq6+udWn9asGCBfn617+eq666Ko899lh++MMf5ic/+UkuvvjiWpcGWyV5sO2TB/IAekIebPvkgTyAnpAH2z55IA/6Sl1/UmLkyJFpamrKihUrNtm+YsWKjB07douPGTt2bFn7b40q6bvLZZddlksvvTT33HNP9t9///4ss8+V2/cvfvGLLFu2LMcdd1z3ts7OziTJoEGDsnTp0kyYMKF/i+4Dlfx777TTTmlubk5TU1P3tr333jvLly/Phg0b0tLS0q8194VK+v7qV7+ak046KZ/5zGeSJPvtt1/Wrl2bv/iLv8hXvvKVNDZum3PctzuuDRs2rC7OgkrkgTwokQebkwfyQB6UyIO3Jw/kgTzY9sgDeSAPSuTB5uSBPOirPNg2f2I91NLSksmTJ2fevHnd2zo7OzNv3rxMnTp1i4+ZOnXqJvsnydy5c992/61RJX0nyTe/+c1cfPHFmTNnTg466KBqlNqnyu174sSJefzxx7N48eLuP3/yJ3+SI444IosXL864ceOqWX7FKvn3PvTQQ/PMM890h2qSPPXUU9lpp50GRMAklfW9bt26zYKkK2hL1wDaNm0Lx7XekgfyQB7Ig7eSB28aaMe13pIH8kAeyIO3kgdvGmjHtd6SB/JAHsiDt5IHb+qz41qvL5U9wN1yyy1Fa2trceONNxZPPPFE8Rd/8RfF8OHDi+XLlxdFURQnnXRSce6553bvf//99xeDBg0qLrvssmLJkiXFrFmziubm5uLxxx+vVQsVKbfvSy+9tGhpaSluv/324n/+53+6/6xZs6ZWLVSk3L5/1ymnnFJ85CMfqVK1fafcvl944YVi++23L84+++xi6dKlxY9//ONi9OjRxd/93d/VqoWKlNv3rFmziu233774P//n/xTPPvtscffddxcTJkwoPv7xj9eqhYqsWbOmWLRoUbFo0aIiSXH55ZcXixYtKp5//vmiKIri3HPPLU466aTu/Z999tliyJAhxTnnnFMsWbKkuPLKK4umpqZizpw5tWqhJuSBPJAH8qCLPJAH8kAeyAN5UBTyQB7IA3kgD+RBiTzo+zyo+6FEURTFd77znWK33XYrWlpaiilTphQPPvhg932HH354ccopp2yy/z/90z8Ve+21V9HS0lLss88+xU9+8pMqV9w3yul79913L5Js9mfWrFnVL7yXyv33fquBGjJFUX7fDzzwQHHwwQcXra2txR577FFccsklxcaNG6tcde+V03d7e3tx4YUXFhMmTCja2tqKcePGFWeeeWbx6quvVr/wXpg/f/4W369dvZ5yyinF4YcfvtljJk2aVLS0tBR77LFHccMNN1S97q2BPJAHXeTBm+SBPKhH8kAedJEHb5IH8qAeyQN50EUevEkeyIO+0FAU2/BnTAAAAAAAgK1GXV9TAgAAAAAAqB5DCQAAAAAAoCoMJQAAAAAAgKowlAAAAAAAAKrCUAIAAAAAAKgKQwkAAAAAAKAqDCUAAAAAAICqMJQAAAAAAACqwlACAAAAAACoCkMJAAAAAACgKgwlAAAAAACAqjCUAAAAAAAAqsJQAgAAAAAAqApDCQAAAAAAoCoMJQAAAAAAgKowlAAAAAAAAKrCUAIAAAAAAKgKQwkAAAAAAKAqDCUAAAAAAICqMJQAAAAAAACqwlACAAAAAACoCkMJAAAAAACgKgwlAAAAAACAqjCUAAAAAAAAqsJQAgAAAAAAqApDCQAAAAAAoCoMJQAAAAAAgKowlAAAgAFg/Pjx+V//63/VugwAAIBeMZQAAAC2aN26dbnwwguzYMGCWpcCAABsIwwlAACALVq3bl0uuugiQwkAAKDPGEoAAECNrF27ttYlAAAAVJWhBAAAVMGFF16YhoaGPPHEEznxxBMzYsSIfOADH8jGjRtz8cUXZ8KECWltbc348eNz/vnnZ/369Vt8nrvvvjuTJk1KW1tb3v/+9+eHP/zhFl/nd914441paGjIsmXLurc98sgjmT59ekaOHJnBgwfnPe95Tz796U8nSZYtW5ZRo0YlSS666KI0NDSkoaEhF154YZLkU5/6VIYOHZpf/vKXmTFjRoYOHZpRo0blr//6r9PR0bHJa3d2duaKK67IPvvsk7a2towZMyann356Xn311U32e6d6utxyyy2ZPHlytt9++wwbNiz77bdfvv3tb7/7PwAAALBVGFTrAgAAoJ587GMfy3vf+958/etfT1EU+cxnPpObbropH/3oR/OlL30pDz30UGbPnp0lS5bkjjvu2OSxTz/9dE444YR87nOfyymnnJIbbrghH/vYxzJnzpwcffTRZdXx8ssv55hjjsmoUaNy7rnnZvjw4Vm2bFn3kGPUqFH53ve+lzPOOCPHH398/vRP/zRJsv/++3c/R0dHR6ZPn56DDz44l112We65555861vfyoQJE3LGGWd073f66afnxhtvzKmnnpq/+qu/ynPPPZfvfve7WbRoUe6///40Nze/az1JMnfu3HziE5/IUUcdlW984xtJkiVLluT+++/P5z//+fL+IQAAgJowlAAAgCo64IAD8oMf/CBJ8h//8R8566yz8pnPfCbXXHNNkuTMM8/M6NGjc9lll2X+/Pk54ogjuh/71FNP5Z//+Z+7BwSnnXZaJk6cmC9/+ctlDyUeeOCBvPrqq7n77rtz0EEHdW//u7/7uyTJdtttl49+9KM544wzsv/+++eTn/zkZs/xxhtv5IQTTshXv/rVJMnnPve5HHjggbnuuuu6hxL33Xdfrr322tx888058cQTux97xBFH5MMf/nBuu+22nHjiie9aT5L85Cc/ybBhw3LXXXelqamprH4BAICtg69vAgCAKvrc5z7X/fc777wzSTJz5sxN9vnSl76UpPQ/4d9q5513zvHHH999e9iwYTn55JOzaNGiLF++vKw6hg8fniT58Y9/nPb29rIe+1Zv7SdJDjvssDz77LPdt2+77bbssMMOOfroo7Ny5cruP5MnT87QoUMzf/78HtczfPjwrF27NnPnzq24XgAAoLYMJQAAoIre8573dP/9+eefT2NjY/bcc89N9hk7dmyGDx+e559/fpPte+6552bXi9hrr72SZJNrRfTE4Ycfnj/7sz/LRRddlJEjR+YjH/lIbrjhhre9lsWWtLW1dV93osuIESM2uVbE008/nddeey2jR4/OqFGjNvnz+uuv5+WXX+5xPWeeeWb22muv/NEf/VF23XXXfPrTn86cOXPK6hsAAKgtX98EAABVNHjw4M22benC1JV6u+f63YtPNzQ05Pbbb8+DDz6Y//t//2/uuuuufPrTn863vvWtPPjggxk6dOi7vlZPvkKps7Mzo0ePzs0337zF+7uGGj2pZ/To0Vm8eHHuuuuu/Nu//Vv+7d/+LTfccENOPvnk3HTTTe9aCwAAUHs+KQEAADWy++67p7OzM08//fQm21esWJFVq1Zl991332T7M888k6IoNtn21FNPJUnGjx+fpPRJhSRZtWrVJvv97qcuuvzhH/5hLrnkkjzyyCO5+eab81//9V+55ZZbkvTNsGTChAn51a9+lUMPPTTTpk3b7M8BBxzQ43qSpKWlJccdd1yuuuqq/OIXv8jpp5+ef/iHf8gzzzzT61oBAID+ZygBAAA1cuyxxyZJrrjiik22X3755UmSP/7jP95k+0svvZQ77rij+/bq1avzD//wD5k0aVLGjh2bpDQESJKf/vSn3futXbt2s08SvPrqq5sNOCZNmpQk3V+ZNGTIkCSbDzjK8fGPfzwdHR25+OKLN7tv48aN3c/dk3p+9atfbXJ/Y2Nj9t9//032AQAAtm6+vgkAAGrkgAMOyCmnnJLvf//7WbVqVQ4//PA8/PDDuemmmzJjxowcccQRm+y/11575bTTTsvPfvazjBkzJtdff31WrFiRG264oXufY445JrvttltOO+20nHPOOWlqasr111+fUaNG5YUXXuje76abbspVV12V448/PhMmTMiaNWtyzTXXZNiwYd3DksGDB+f9739/br311uy1117Zcccds++++2bfffftcY+HH354Tj/99MyePTuLFy/OMccck+bm5jz99NO57bbb8u1vfzsf/ehHe1TPZz7zmfz617/OkUcemV133TXPP/98vvOd72TSpEnZe++9e/NPAQAAVImhBAAA1NC1116bPfbYIzfeeGPuuOOOjB07Nuedd15mzZq12b7vfe97853vfCfnnHNOli5dmve85z259dZbM3369O59mpubc8cdd+TMM8/MV7/61YwdOzZf+MIXMmLEiJx66qnd+3UNQG655ZasWLEiO+ywQ6ZMmZKbb755k4txX3vttfnLv/zLfPGLX8yGDRsya9assoYSSXL11Vdn8uTJ+fu///ucf/75GTRoUMaPH59PfvKTOfTQQ3tczyc/+cl8//vfz1VXXZVVq1Zl7NixOeGEE3LhhRemsdGHwAEAYCBoKH73M9IAAAAAAAD9wOlEAAAAAABAVRhKAAAAAAAAVWEoAQAAAAAAVIWhBAAAAAAAUBWGEgAAAAAAQFUYSgAAAAAAAFVhKAEAAAAAAFSFoQQAAAAAAFAVhhIAAAAAAEBVGEoAAAAAAABVYSgBAAAAAABUhaEEAAAAAABQFYYSAAAAAABAVfz/9GExPDc0k0IAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_changes(r_ecec, param_name='accuracy_importance', height=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Surprisingly, ECEC demonstrates very subtle sensitivity to the only tunable fitting huperparameter *accuracy_importance* which results in a flattend values in case only aerliness is considered, and downsliding dependency for other values which may be an evidence of overestimation of parameters. \n", + "\n", + "Moreover, for last subplots, the first 5 estimation points are set to (0, 0) since no objects passed over the thresholds." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API launch" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from fedot.core.pipelines.pipeline_builder import PipelineBuilder\n", + "from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels\n", + "from copy import deepcopy\n", + "from tqdm.autonotebook import tqdm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's prepare some configuration dictionaries" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [], + "source": [ + "series_length = train_data.features.shape[-1]\n", + "\n", + "interval_percentage = 5\n", + "consecutive_predictions = 2\n", + "transform_score = True\n", + "prediction_mode = 'all'\n", + "accuracy_importance = 0.5\n", + "common_dict = {\n", + " 'prediction_mode': prediction_mode,\n", + " 'interval_percentage': interval_percentage,\n", + " 'transform_score': transform_score,\n", + " 'accuracy_importance': accuracy_importance,\n", + "}\n", + "models = {\n", + " 'economy_k': {\n", + " 'lambda': 100000,\n", + " },\n", + " 'ecec': {},\n", + " 'teaser': {},\n", + " 'proba_threshold_etc': {\n", + " 'probability_threshold': 0.8,\n", + " },\n", + "}\n", + "for model in models:\n", + " models[model] |= common_dict\n", + "prediction_idx = (np.linspace(0, 1, 21) * series_length).astype(int)\n", + "earliness = 1 - prediction_idx / series_length\n", + "\n", + "results = {model: [None] * len(prediction_idx) for model in models}" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "metadata": {}, + "outputs": [], + "source": [ + "with IndustrialModels():\n", + " repo = IndustrialModels().setup_repository()\n", + " for model, params in models.items():\n", + " pipeline = PipelineBuilder().add_node(model, params=params).build()\n", + " pipeline.fit(train_data)\n", + " prediction = pipeline.predict(test_data).predict\n", + " prediction, scores = prediction\n", + " prediction = prediction.argmax(-1)\n", + " scores = scores[..., 0]\n", + " results[model] = ETSCPareto(\n", + " yte, prediction, scores, reduce=False, metric_list=('accuracy',).metric()\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "def plot_bicrit_metric(metrics: dict, select=None):\n", + " plt.figure(figsize=(10, 10))\n", + " for model, metric in metrics.items():\n", + " selection = metric.iloc[select, :]\n", + " sizes = ((np.arange(selection.shape[0]) * 2)[::-1]) ** 1.5 + 10\n", + " plt.plot(selection.robustness, selection.accuracy, alpha=0.3)\n", + " plt.scatter(x=selection.robustness, \n", + " y=selection.accuracy,\n", + " s=sizes, \n", + " label=model)\n", + " plt.legend(loc=\"upper right\", bbox_to_anchor=(1.5, 1))\n", + " plt.xlabel('Robustness')\n", + " plt.ylabel('Accuracy')\n", + " plt.xlim((-0.05, 1.05))\n", + " plt.ylim((-0.05, 1.05))\n", + " plt.xticks(np.linspace(0, 1, 11))\n", + " plt.yticks(np.linspace(0, 1, 11))\n", + " plt.grid(True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABMoAAANBCAYAAAARI1KsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3zdddn/8df3e/Y52aMZbZrRSVu66KBslSWKN06GCBaZ3ihYFUVliuACEcUbBRkqDgR+yH0LyEZZLZ1QumiSNkmbveeZ398fhxwamqRpe5KT8X4+Hn2E853X+ZDk5Fzn87kuw7IsCxERERERERERkQnOTHQAIiIiIiIiIiIio4ESZSIiIiIiIiIiIihRJiIiIiIiIiIiAihRJiIiIiIiIiIiAihRJiIiIiIiIiIiAihRJiIiIiIiIiIiAihRJiIiIiIiIiIiAihRJiIiIiIiIiIiAoA90QGMtEgkwt69e0lOTsYwjESHIyIiIiIiCWJZFu3t7eTn52OamkMgIiITMFG2d+9eCgoKEh2GiIiIiIiMEpWVlUyZMiXRYYiIyCgw4RJlycnJQPTFMCUlJcHRQDAY5Nlnn+XUU0/F4XAkOpxRQ+PSP43LwDQ2/dO4DExj0z+Ny8A0Nv3TuAxMY9O/0TQubW1tFBQUxN4jiIiITLhEWe9yy5SUlFGTKPN6vaSkpCT8D4XRROPSP43LwDQ2/dO4DExj0z+Ny8A0Nv3TuAxMY9O/0TguKskiIiK9tBBfREREREREREQEJcpEREREREREREQAJcpERERERERERESACVijTERERERERGQisyyLUChEOBxOdCgiI8Jms2G324dUk1KJMhEREREREZEJIhAIUF1dTVdXV6JDERlRXq+XvLw8nE7noMcpUSYiIiIiIiIyAUQiEcrLy7HZbOTn5+N0OtX1VcY9y7IIBALU19dTXl7OjBkzMM2BK5EpUSYiIiIiIiIyAQQCASKRCAUFBXi93kSHIzJiPB4PDoeD3bt3EwgEcLvdAx6rYv4iIiIiIiIiE8hgs2lExquhft/rp0NERERERERERAQlykRERERERERERAAlykRERERERERExqRdu3ZhGAYbN25MdCjjhor5i4iIiIiIiMhBCYYjrN3VTGt3kFSPgyVF6ThsmosjY58SZSIiIiIiIiIyJMFwhHteLuWB13fR1BmIbc/wOVl5TBGXnzRNCTMZ0/TdKyIiIiIiIiIHFAxHuPihtdzx3I4+STKAps4Adzy3g0v+sJZgODIs949EItx2220UFxfj8XhYsGABjz76aGz/u+++yyc/+UlSUlJITk7m+OOPp7S0NHbuzTffzJQpU3C5XCxcuJBnnnkmdm7vEsbHH3+cj3zkI3i9XhYsWMAbb7zRJ4bHHnuMuXPn4nK5KCoq4vbbb++zv6ioiFtuuYULLriApKQkCgsLefLJJ6mvr+e//uu/SEpKYv78+axduxaAzs5OUlJS+jwPgCeeeAKfz0d7e/tBjVE4HOaiiy5i9uzZVFRUHNS5EqVEmYiIiIiIiIgc0D0vl/Lv9+qxBthvAa9sr+e3r5QOy/1vu+02/vCHP3DPPffw7rvv8o1vfIPzzz+fV155hT179nDCCSfgcrl48cUXWbduHRdddBGhUAiAX/7yl9x+++38/Oc/5+233+a0007jU5/6FO+9916fe3z/+9/nW9/6Fhs3bmTmzJmce+65sWusW7eOL3zhC5xzzjm888473HjjjVx33XU8+OCDfa7xi1/8gmOPPZYNGzbwiU98gi996UtccMEFnH/++axfv55p06ZxwQUXYFkWPp+Pc845hwceeKDPNR544AE+97nPkZycPOTx8fv9fP7zn2fjxo385z//YerUqYcwyqKllyIiIiIiIiIyqGA4wgOv78IaKEv2Pgt44LVdXHZifJdg+v1+br31Vp5//nlWrFgBQElJCa+++iq//e1vKSoqIjU1lb/+9a84HA4AZs6cGTv/5z//Od/5znc455xzAPjJT37CSy+9xJ133sndd98dO+5b3/oWn/jEJwC46aabmDt3Ljt37mT27NnccccdfOxjH+O6666LXX/Lli387Gc/48tf/nLsGmeccQaXXXYZANdffz3/8z//w9KlS/n85z8PwHe+8x1WrFhBbW0tubm5XHzxxRxzzDFUV1eTl5dHXV0dTz31FM8///yQx6ejo4NPfOIT+P1+XnrpJVJTUw92iOV9mlEmIiIiIiIiIoNau6t5v+WWA2nsDLBud3Nc779z5066uro45ZRTSEpKiv37wx/+QGlpKRs3buT444+PJcn21dbWxt69ezn22GP7bD/22GPZunVrn23z58+P/XdeXh4AdXV1AGzdurXfa7z33nuEw+F+r5GTkwPAkUceud+23usuW7aMuXPn8tBDDwHwpz/9icLCQk444YShDA0A5557Lp2dnTz77LNKkh0mJcpEREREREREZFCt3cGDOr6l6+COP5COjg4A/vnPf7Jx48bYvy1btvDoo4/i8Xjicp99E22GYQDR+maHe40DXffiiy+OLeF84IEHWLlyZey4oTjjjDN4++2396upJgdPiTIRERERERERGVSqZ/+ZWoNJ8x7c8QcyZ84cXC4XFRUVTJ8+vc+/goIC5s+fz3/+8x+Cwf0TdCkpKeTn5/Paa6/12f7aa68xZ86cIcdwxBFH9HuNmTNnYrPZDu2Jve/8889n9+7d3HXXXWzZsoULL7zwoM6/4oor+PGPf8ynPvUpXnnllcOKZaJTjTIRERERERERGdSSonQyfM4hLb/M9Dk5qjA9rvdPTk7mW9/6Ft/4xjeIRCIcd9xxtLa28tprr5GSksKVV17Jr371K8455xyuvfZaUlNTefPNN1m2bBmzZs3i29/+NjfccAPTpk1j4cKFPPDAA2zcuJGHH354yDF885vfZOnSpfzwhz/k7LPP5o033uDXv/41v/nNbw77+aWnp/OZz3yGb3/725x66qlMmTLloK/xta99jXA4zCc/+UmefvppjjvuuMOOayJSokxEREREREREBuWwmaw8pog7ntsxYNdLAANYeWxRXAv59/rhD39IdnY2t912G2VlZaSlpbF48WK+973vkZmZyYsvvsi3v/1tTjzxRGw2GwsXLozVFPv6179Oa2sr3/zmN6mrq2POnDk8+eSTzJgxY8j3X7x4MY888gjXX389P/zhD8nLy+Pmm2/uU8j/cHzlK1/hz3/+MxdddNEhX+Pqq68mEolwxhln8Mwzz3DMMcfEJbaJRIkyERERERERETmgy0+axrqKZl7ZXt9vsswATpyVzWUnThuW+xuGwVVXXcVVV13V7/758+fzr3/9q999pmlyww03cMMNN/S7v6ioCOtDLT3T0tL22/bZz36Wz372swPGuGvXrv22ffga/d0LYM+ePWRmZvJf//VfA15/KHGvWrWKVatWDfka0pdqlImIiIiIiIjIATlsJvdesIRVp8wk0+fssy/T5+Sbp87k3guWDMtssvGsq6uL0tJSfvzjH3PZZZfhdDoPfJIMG80oExEREREREZEhcdhMvvaxGdHZZbubaekKkuZ1cFRhuhJkh+inP/0pP/rRjzjhhBO49tpr++y79dZbufXWW/s97/jjj+fpp58eiRAnFCXKREREREREROSgOGwmR5dkJjqMceHGG2/kxhtv7Hff5Zdfzhe+8IV+93k8nmGMauJSokxEREREREREZBTKyMggIyMj0WFMKJoXKSIiIiIiIiIiQoITZf/+978588wzyc/PxzAMnnjiiQOe8/LLL7N48WJcLhfTp0/nwQcfHPY4RURERERERERk/Etooqyzs5MFCxZw9913D+n48vJyPvGJT/CRj3yEjRs3cvXVV3PxxRcP2P5VRERERERERERkqBJao+zjH/84H//4x4d8/D333ENxcTG33347AEcccQSvvvoqv/jFLzjttNOGK0wREREREREREZkAxlSNsjfeeIOTTz65z7bTTjuNN954I0ERiYiIiIiIiIjIeDGmul7W1NSQk5PTZ1tOTg5tbW10d3f32xrV7/fj9/tjj9va2gAIBoMEg8HhDXgIemMYDbGMJhqX/mlcBqax6Z/GZWAam/5pXAamsemfxmVgGpv+jaZxGQ0xiIjI6DKmEmWH4rbbbuOmm27ab/uzzz6L1+tNQET9e+655xIdwqikcemfxmVgGpv+aVwGprHpn8ZlYBqb/mlcBqax6d9oGJeurq5EhyAydoWDUPEm9LSAOw2mHg02R6KjEjlsYypRlpubS21tbZ9ttbW1pKSk9DubDODaa69l1apVscdtbW0UFBRw6qmnkpKSMqzxDkUwGOS5557jlFNOweHQL5VeGpf+aVwGprHpn8ZlYBqb/mlcBqax6Z/GZWAam/6NpnHpXW0iIgchHIRX74TV90BXwwfbvVmw/HI47molzGRMG1OJshUrVvDUU0/12fbcc8+xYsWKAc9xuVy4XK79tjscjoS/MO9rtMUzWmhc+qdxGZjGpn8al4FpbPqncRmYxqZ/GpeBaWz6NxrGJdH3FxlzwkH4yzmw8wXA6ruvqwFe+hFUroZz/zIsybJIJMJPfvITfve731FTU8PMmTO57rrr+NznPgfAu+++y3e+8x3+/e9/Y1kWCxcu5MEHH2TatGkA3Hfffdx+++2Ul5dTVFTE17/+db761a/Grl9VVcW3v/1t/vWvf+H3+zniiCO4++67Wb58edyfi4xeCU2UdXR0sHPnztjj8vJyNm7cSEZGBlOnTuXaa69lz549/OEPfwDg8ssv59e//jXXXHMNF110ES+++CKPPPII//znPxP1FEREREREREQmhlfvhNJ+kmQxFux8Hl67E074dtxvf9ttt/GnP/2Je+65hxkzZvDvf/+b888/n+zsbKZPn84JJ5zASSedxIsvvkhKSgqvvfYaoVAIgIcffpjrr7+eX//61yxatIgNGzZwySWX4PP5uPDCC+no6ODEE09k8uTJPPnkk+Tm5rJ+/XoikUjcn4eMbglNlK1du5aPfOQjsce9SyQvvPBCHnzwQaqrq6moqIjtLy4u5p///Cff+MY3+OUvf8mUKVO47777OO2000Y8dhEREREREZEJIxyMLre0BkqS9bJg9W/h2KvjOqvM7/dz66238vzzz8dWlZWUlPDqq6/y29/+lqKiIlJTU/nrX/8amy06c+bM2Pk33HADt99+O5/5zGeAaH5hy5Yt/Pa3v+XCCy/kz3/+M/X19bz11ltkZGQAMH369LjFL2NHQhNlJ510EtYgP2QPPvhgv+ds2LBhGKMSERERERERkT4q3uxbk2wwnfXRJZhFx8Xt9jt37qSrq4tTTjmlz/ZAIMCiRYtoaWnh+OOP73dJdWdnJ6WlpXzlK1/hkksuiW0PhUKkpqYCsHHjRhYtWhRLksnENaZqlImIiIiIiIhIAvS0HNzx3c1xvX1HRwcA//znP5k8eXKffS6Xi6uvvvqA595777371Ruz2WwAAzYIlIlHiTIRERERERERGZw77eCO96TH9fZz5szB5XJRUVHBiSeeuN/++fPn89BDDxEMBvebVZaTk0N+fj5lZWV88Ytf7Pf68+fP57777qOpqUmzyiY4JcpEREREREREZHBTjwZv1tCWX/qyoSC+nSKTk5P51re+xTe+8Q0ikQjHHXccra2tvPbaa6SkpHDllVfyq1/9inPOOYdrr72W1NRU3nzzTZYtW8asWbO46aab+PrXv05qaiqnn346fr+ftWvX0tzczKpVqzj33HO59dZbOeuss7jtttvIy8tjw4YN5Ofnx2qiycRgJjoAERERERERERnlbA5YfjlgHOBAA5ZfFtdC/r1++MMfct1113HbbbdxxBFHcPrpp/PPf/6T4uJiMjMzefHFF2PdK4866ijuvffe2Oyyiy++mPvuu48HHniAI488khNPPJEHH3yQ4uJiAJxOJ88++yyTJk3ijDPO4Mgjj+THP/5xbGmmTByaUSYiIiIiIiIiB3bc1dEi/TufB/przGfA9JOjHS+HgWEYXHXVVVx11VX97p8/fz7/+te/Bjz/vPPO47zzzhtwf2FhIY8++uhhxyljm2aUiYiIiIiIiMiB2Rxw7l/gI9+PLsPcly8bPvr96P5hmE0mMlI0o0xEREREREREhsbmgBO//cHssu7maOH+guVKkMm4oESZiIiIiIiIiBwcmwOKjkt0FCJxp6WXIiIiIiIiIiIiKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIgcpGAnyVs1bvLD7Bd6qeYtgJDis9zvppJO4+uqrh/UeIgD2RAcgIiIiIiIiImNDMBLk/nfu5+GtD9Psb45tT3el88UjvshFR16Ew3QkMMLECQQCOJ3ORIchh0kzykRERERERETkgIKRIF9/8evcvfHuPkkygGZ/M3dvvJurXrwq7rPLvvzlL/PKK6/wy1/+EsMwMAyDXbt2sXnzZj7+8Y+TlJRETk4OX/rSl2hoaIid98wzz3DccceRlpZGZmYmn/zkJyktLY3tDwQCXHnlleTl5eF2uyksLOS2226L7W9paeHiiy8mOzublJQUPvrRj7Jp06bY/htvvJGFCxdy3333UVxcjNvtjuvzlsRQokxEREREREREDuj+d+7ntT2vYWH1u9/C4tU9r/LA5gfiet9f/vKXrFixgksuuYTq6mqqq6tJTk7mox/9KIsWLWLt2rU888wz1NbW8oUvfCF2XmdnJ6tWrWLt2rW88MILmKbJpz/9aSKRCAB33XUXTz75JI888gjbt2/n4YcfpqioKHb+5z//eerq6nj66adZt24dixcv5mMf+xhNTU2xY3bu3Mljjz3G448/zsaNG+P6vCUxtPRSRERERERERAYVjAR5eOvDAybJellYPLz1YVbOWxm3JZipqak4nU68Xi+5ubkA3HLLLSxatIhbb701dtz9999PQUEBO3bsYObMmXz2s5/tc53777+f7OxstmzZwrx586ioqGDGjBkcd9xxGIZBYWFh7NhXX32VNWvWUFdXh8vlAuDnP/85TzzxBI8++iiXXnopEJ2V9oc//IHs7Oy4PFdJPM0oExEREREREZFBbazbuN9yy4E09TSxsW7jsMazadMmXnrpJZKSkmL/Zs+eDRBbXvnee+9x7rnnUlJSQkpKSmy2WEVFBRBd0rlx40ZmzZrF17/+dZ599tk+1+/o6CAzM7PPPcrLy/ss3ywsLFSSbJzRjDIRERERERERGVSbv21Yjz9YHR0dnHnmmfzkJz/Zb19eXh4AZ555JoWFhdx7773k5+cTiUSYN28egUAAgMWLF1NeXs7TTz/N888/zxe+8AVOPvlkHn30UTo6OsjLy+Pll1/e7/ppaWmx//b5fMPy/CRxlCgTERERERERkUGluFKG9fgDcTqdhMPh2OPFixfz2GOPUVRUhN2+f2qjsbGR7du3c++993L88ccD0eWU+8WZksLZZ5/N2Wefzec+9zlOP/10mpqaWLx4MTU1Ndjt9j51y2T809JLERERERERERnUwkkLSXelD+nYDHcGCyctjOv9i4qKWL16Nbt27aKhoYH//u//pqmpiXPPPZe33nqL0tJS/vWvf7Fy5UrC4TDp6elkZmbyu9/9jp07d/Liiy+yatWqPte84447+Mtf/sK2bdvYsWMHf//738nNzSUtLY2TTz6ZFStWcNZZZ/Hss8+ya9cuXn/9db7//e+zdu3auD43GV2UKBMRERERERGRQTlMB1884osYGIMeZ2DwxSO+GLdC/r2+9a1vYbPZmDNnDtnZ2QQCAV577TXC4TCnnnoqRx55JFdffTVpaWmYpolpmvz1r39l3bp1zJs3j2984xv87Gc/63PN5ORkfvrTn7JkyRKWLl3Krl27eOqppzBNE8MweOqppzjhhBNYuXIlM2fO5JxzzmH37t3k5OTE9bnJ6KKllyIiIiIiIiJyQBcdeRGb6jfx6p5X++1+aWBw3OTjWDlvZdzvPXPmTN544439tj/++OMDnnPyySezZcuWPtss64O4L7nkEi655JIBz09OTuauu+7irrvu6nf/jTfeyI033niAyGWs0YwyERERERERETkgh+nglx/9Jf+98L/3W4aZ4c7gykVX8suP/jLus8lERpJmlImIiIiIiIjIkDhMB5ctuIyLjryIjXUbafO3keJKYeGkhUqQybigRJmIiIiIiIiIHBSH6WBp7tJEhyESd1p6KSIiIiIiIiIighJlIiIiIiIiIiIigBJlIiIiIiIiIiIigBJlIiIiIiIiIiIigBJlIiIiIiIiIiIigBJlIiIiIiIiIiIigBJlIiIiIiIiIjJBFRUVceedd06Y+xqGwRNPPHFY1zjppJO4+uqrBz0mUc8vHpQoExEREREREZGDYgWDdK5eQ9tzz9G5eg1WMJjokEalBx98kLS0tESHMabdeOONLFy4cMTuZx+xO4mIiIiIiIjImGYFgzTedx9Nf/wT4aam2HZbRgYZXzqfzIsvxnA4EhhhVCAQwOl0JjqMuAoGgzhGwdiOd5pRJiIiIiIiIiIHZAWDVH71v6m/61d9kmQA4aYm6u/6FZX/feWwzC476aSTuPLKK7nyyitJTU0lKyuL6667DsuygOhSvx/+8IdccMEFpKSkcOmllwLw2GOPMXfuXFwuF0VFRdx+++37Xbu9vZ1zzz0Xn8/H5MmTufvuu/vsv+OOOzjyyCPx+XwUFBTw1a9+lY6OjgPG/PLLL7Ny5UpaW1sxDAPDMLjxxhtj+7u6urjoootITk5m6tSp/O53v4vt27VrF4Zh8Le//Y0TTzwRt9vNww8/DMB9993HEUccgdvtZvbs2fzmN7+JnRcIBLjyyivJy8vD7XZTWFjIbbfd1ieuhoYGPv3pT+P1epkxYwZPPvlkn/2vvPIKy5Ytw+VykZeXx3e/+11CodCAz7Ouro4zzzwTj8dDcXFxLM6hamlp4eKLLyY7O5uUlBQ++tGPsmnTJiA6I++mm25i06ZNsTF88MEHY+dddtll5OTk4Ha7mTdvHv/3f/93UPfujxJlIiIiIiIiInJAjffdR+err8L7yan9WBad//kPjb///bDc/6GHHsJut7NmzRp++ctfcscdd3DffffF9v/85z9nwYIFbNiwgeuuu45169bxhS98gXPOOYd33nmHG2+8keuuuy6WaOn1s5/9LHbed7/7Xa666iqee+652H7TNLnrrrt49913eeihh3jxxRe55pprDhjvMcccw5133klKSgrV1dVUV1fzrW99K7b/9ttvZ8mSJWzYsIGvfvWrXHHFFWzfvr3PNXrj2bp1K6eddhoPP/ww119/PT/60Y/YunUrt956K9dddx0PPfQQAHfddRdPPvkkjzzyCNu3b+fhhx+mqKiozzVvuukmvvCFL/D2229zxhln8MUvfpGm9xOfe/bs4YwzzmDp0qVs2rSJ//mf/+H3v/89t9xyy4DP88tf/jKVlZW89NJLPProo/zmN7+hrq7ugOPT6/Of/zx1dXU8/fTTrFu3jsWLF/Oxj32MpqYmzj77bL75zW8yd+7c2BieffbZRCIRPv7xj/Paa6/xpz/9iS1btvDjH/8Ym8025PsOREsvRURERERERGRQVjBI0x//NHCSLHagRdMf/0jmV74S9yWYBQUF/OIXv8AwDGbNmsU777zDL37xCy655BIAPvrRj/LNb34zdvwXv/hFPvaxj3HdddcBMHPmTLZs2cLPfvYzvvzlL8eOO/bYY/nud78bO+a1117jF7/4BaeccgpAn8L1RUVF3HLLLVx++eV9ZnL1x+l0kpqaimEY5Obm7rf/jDPO4Ktf/SoA3/nOd/jFL37BSy+9xKxZs2LHXH311XzmM5+JPb7hhhu4/fbbY9uKi4vZsmULv/3tb7nwwgupqKhgxowZHHfccRiGQWFh4X73/fKXv8y5554LwK233spdd93FmjVrOP300/nNb35DQUEBv/71rzEMg9mzZ7N3716+853vcP3112Oafedb7dixg6effpo1a9awdOlSAH7/+99zxBFHDDo2vV599VXWrFlDXV0dLpcLiCY8n3jiCR599FEuvfRSkpKSsNvtfcbw2WefZc2aNWzdupWZM2cCUFJSMqR7HohmlImIiIiIiIjIoLrWb9hvueVAwo1NdG3YEPcYjj76aAzDiD1esWIF7733HuFwGIAlS5b0OX7r1q0ce+yxfbYde+yxfc7pvc6+VqxYwdatW2OPn3/+eT72sY8xefJkkpOT+dKXvkRjYyNdXV2H9Xzmz58f++/eZNqHZ2Lt+5w6OzspLS3lK1/5CklJSbF/t9xyC6WlpUA0CbZx40ZmzZrF17/+dZ599tlB7+vz+UhJSYndd+vWraxYsaLPOB977LF0dHRQVVW137W2bt2K3W7nqKOOim2bPXv2kBsYbNq0iY6ODjIzM/s8p/Ly8thz6s/GjRuZMmVKLEkWT5pRJiIiIiIiIiKDCre1HtzxrQd3fDz4fL64X3PXrl188pOf5IorruBHP/oRGRkZvPrqq3zlK18hEAjg9XoP+dofLsxvGAaRSKTPtn2fU29dtHvvvZfly5f3Oa53yeHixYspLy/n6aef5vnnn+cLX/gCJ598Mo8++uhB3XekdHR0kJeXx8svv7zfvsGSbR6PZ9hiUqJMRERERERERAZlS0k9uONTD+74oVi9enWfx2+++SYzZswYsC7VEUccwWuvvdZn22uvvcbMmTP7nPPmm2/ud93epYPr1q0jEolw++23x5YdPvLII0OO2el09pm9djhycnLIz8+nrKyML37xiwMel5KSwtlnn83ZZ5/N5z73OU4//XSamprIyMg44D2OOOIIHnvsMSzLis0qe+2110hOTmbKlCn7HT979mxCoRDr1q2LLb3cvn07LS0tQ3pOixcvpqamBrvdvl8ttV79jeH8+fOpqqpix44dcZ9VpqWXIiIiIiIiIjIo7+JF2IaQaAGwZWbgXbQo7jFUVFSwatUqtm/fzl/+8hd+9atfcdVVVw14/De/+U1eeOEFfvjDH7Jjxw4eeughfv3rX/cpqA/RRNBPf/pTduzYwd13383f//732HWnT59OMBjkV7/6FWVlZfzxj3/knnvuGXLMRUVFdHR08MILL9DQ0HDYyzVvuukmbrvtNu666y527NjBO++8wwMPPMAdd9wBRDt0/uUvf2Hbtm3s2LGDv//97+Tm5g55KeRXv/pVKisr+drXvsa2bdv4xz/+wQ033MCqVav2q08GMGvWLE4//XQuu+wyVq9ezbp167j44ouHPOPr5JNPZsWKFZx11lk8++yz7Nq1i9dff53vf//7rF27FoiOYXl5ORs3bqShoQG/38+JJ57ICSecwGc/+1mee+652Cy6Z555ZmgDOQglykRERERERERkUIbDQcaXzod9alf1f6BBxpe+FPdC/gAXXHAB3d3dLFu2jP/+7//mqquu4tJLLx3w+MWLF/PII4/w17/+lXnz5nH99ddz88039ynkD9GE2tq1a1m0aBG33HILd9xxB6eddhoACxYs4I477uAnP/kJ8+bN4+GHH+a2224bcszHHHMMl19+OWeffTbZ2dn89Kc/PaTn3uviiy/mvvvu44EHHuDII4/kxBNP5MEHH6S4uBiA5ORkfvrTn7JkyRKWLl3Krl27eOqpp/pNcvVn8uTJPPXUU6xZs4YFCxZw+eWX85WvfIUf/OAHA57zwAMPkJ+fz4knnshnPvMZLr30UiZNmjSk+xmGwVNPPcUJJ5zAypUrmTlzJueccw67d+8mJycHgM9+9rOcfvrpfOQjHyE7O5u//OUvADz22GMsXbqUc889lzlz5nDNNdfEZfaeYVkHalkxvrS1tZGamkprayspKSmJDodgMMhTTz3FGWecsd864YlM49I/jcvANDb907gMTGPTP43LwDQ2/dO4DExj07/RNC6j7b2ByHDr6emhvLyc4uJi3G73QZ9vBYNU/veVdP7nP/13vzQMfMcfT8Hdv457ouykk05i4cKF3HnnnXG9rkwcQ/3+14wyERERERERETkgw+Gg4O5fk/31r+23DNOWmUH2VV8fliSZyEhSMX8RERERERERGRLD4SDriivIvPhiujZsINzaii01Fe+iRRMyQfbxj3+c//znP/3u+973vsf3vve9EY5odHn44Ye57LLL+t1XWFjIu+++O8IRHZgSZSIiIiIiIiJyUAyHA9+yZSN2v5dffnnE7nUw7rvvPrq7u/vdN5Quk+Pdpz71KZYvX97vvkQvvx+IEmUiIiIiIiIiIodg8uTJiQ5hVEtOTiY5OTnRYRwU1SgTERERERERmUAmWE8/EWDo3/dKlImIiIiIiIhMAL1L3bq6uhIcicjI6/2+P9CSTy29FBEREREREZkAbDYbaWlp1NXVAeD1ejEMI8FRiQwvy7Lo6uqirq6OtLQ0bDbboMcrUSYiIiIiIiIyQeTm5gLEkmUiE0VaWlrs+38wSpSJiIiIiIiITBCGYZCXl8ekSZMIBoOJDkdkRDgcjgPOJOulRJmIiIiIiIjIBGOz2YacOBCZSFTMX0REREREREREBCXKREREREREREREgFGQKLv77rspKirC7XazfPly1qxZM+CxwWCQm2++mWnTpuF2u1mwYAHPPPPMCEYrIiIiIiIiIiLjVUITZX/7299YtWoVN9xwA+vXr2fBggWcdtppA3bf+MEPfsBvf/tbfvWrX7FlyxYuv/xyPv3pT7Nhw4YRjlxERERERERERMabhCbK7rjjDi655BJWrlzJnDlzuOeee/B6vdx///39Hv/HP/6R733ve5xxxhmUlJRwxRVXcMYZZ3D77bePcOQiIiIiIiIiIjLeJCxRFggEWLduHSeffPIHwZgmJ598Mm+88Ua/5/j9ftxud59tHo+HV199dVhjFRERERERERGR8c+eqBs3NDQQDofJycnpsz0nJ4dt27b1e85pp53GHXfcwQknnMC0adN44YUXePzxxwmHwwPex+/34/f7Y4/b2tqAaL2zYDAYh2dyeHpjGA2xjCYal/5pXAamsemfxmVgGpv+aVwGprHpn8ZlYBqb/o2mcRkNMYiIyOhiWJZlJeLGe/fuZfLkybz++uusWLEitv2aa67hlVdeYfXq1fudU19fzyWXXML//u//YhgG06ZN4+STT+b++++nu7u73/vceOON3HTTTftt//Of/4zX643fExIRERERkTGlq6uL8847j9bWVlJSUhIdjoiIjAIJm1GWlZWFzWajtra2z/ba2lpyc3P7PSc7O5snnniCnp4eGhsbyc/P57vf/S4lJSUD3ufaa69l1apVscdtbW0UFBRw6qmnjooXw2AwyHPPPccpp5yCw+FIdDijhsalfxqXgWls+qdxGZjGpn8al4FpbPqncRmYxqZ/o2lcelebiIiI9EpYoszpdHLUUUfxwgsvcNZZZwEQiUR44YUXuPLKKwc91+12M3nyZILBII899hhf+MIXBjzW5XLhcrn22+5wOBL+wryv0RbPaKFx6Z/GZWAam/5pXAamsemfxmVgGpv+aVwGprHp32gYl0TfX0RERp+EJcoAVq1axYUXXsiSJUtYtmwZd955J52dnaxcuRKACy64gMmTJ3PbbbcBsHr1avbs2cPChQvZs2cPN954I5FIhGuuuSaRT0NERERERERERMaBhCbKzj77bOrr67n++uupqalh4cKFPPPMM7EC/xUVFZjmB405e3p6+MEPfkBZWRlJSUmcccYZ/PGPfyQtLS1Bz0BERERERERERMaLhCbKAK688soBl1q+/PLLfR6feOKJbNmyZQSiEhERERERERGRicY88CEiIiIiIiIiIiLjnxJlIiIiIiIiIiIiKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICKFEmIiIiIiIiIiICgD3RAYiIiIgMlb+0lK6167ClppB00kmYbneiQxIRERGRcUSJMhkzKtoq+NlbP2N1zWqyPFlcfOTFfGbGZxIdloiIjAArHKb6hhtoffSx2DYzJYWCe/4H7+LFCYxMRERERMYTLb2UMaEr2MUFT1/Af/b8h+5QN5Xtldzw+g38b+n/Jjo0EREZAS2PPNInSQYQ6eig8oqvEvH7ExSViIiIiIw3SpTJmPBCxQs09jQStsKxbQYGD737UAKjEhGRkdL890fBMPpujESItLbS8coriQlKRERERMYdJcpkTGjqacKg7xskC4umnqYERSQiIiMp0toCltX/vra2kQ1GRERERMYtJcpkTDg2/1gs+r5BMjE5qeCkxAQkIiIjyrviGLDZ+t3nmT9/hKMRERERkfFKiTIZE6anT+fzMz/fZ9uU5ClcuejKBEUkIiIjKeuySzF9vv2SZb7jjyfU2Ei4ozNBkYmIiIjIeKKulzImNHY3csKUE5ibOZe6rjosLI7JO4YMd0aiQxMRkRHgLCig+PHHaPzdvXS+/jqG24132TJ8xx5DpLuH7g3rcc+diz1DrwsiIiIicuiUKJNRz7IsylrLAFg4aSG5vlzeqnmLtmAb4UgYm9n/UhwRERlfnFOmkHfzTQBEurvpXL0aIhaYBlYwRPemTbhnzcKRn5/gSEVERERkrNLSSxn1ajpr6Ax2YjftTE2Zis/hw2lzErEitAZaEx2eiIgkgOnx4Jw8+f3/9mLPmQQW9Gzbjr+0FGuAwv8iIiIiIoNRokxGtXAkTHlbOQCFKYU4TAcAme5MAJq61fVSRGSichYWYthtRDo7sWdl4SwqAiCwu4Keze9ihcOJDVBERERExhwtvZSEilgR/l31b/6y7S+8Xf82ESvCzPSZnD3jbACqOqoIhAO47W4mJ02OnZfuTqe6s5pmf3OiQhcRkQQznE6cU6fiLysnUFaGd/lyTK+Hnm3bCNXX072hB/f8+ZhOZ6JDFREREZExQokySZhAOMC3XvkWL1W+hM2wEbain/y/3fA2W+u3cm3qtWxv3o7P5aMktQTT+GACZLo7HQODzmAnPaEe3HZ3op5G3FmWxbradZS2lJLlyeL4KcfjtOlNnohIfxwFBQT27CHS3UNw716cU6ZguFz0bN5MuK2d7rVrcc9fgC3Jl+hQRURERGQM0NJLSZhb3ryFlytfBoglySA6ywygIdLAA5sfIMmRxCTvpD7nOkwHKa4UAJp7xs+ssqr2Kj7z5GdY+a+V3LL6Fq5++Wo+8shHeH3P64kOTURkVDJsNly9Sy7Ld2GFQtjT0/EedRSm10Okx0/3+nWEmrRUX0REREQOTIkySYiazhqe2PkEFv0XW7awaIm0UNpaSk+op99j0l3pADT1jI83P6FIiEufu5Ty1vI+29sD7Vz54pVUtFUkKDIRkdHNnp+P6fViBYMEKqK/K02vF+/ixdjSUrFCYbo3bSK4d2+CIxURERGR0U6JMkmIJ0ufxDCMAff3JtBs2Hiu4rl+j8nwZADRRFnvLLSx7N9V/6ayvbLP7DqIjkXEivDX7X9NUGQiIqObYRi4ppUAEKysJOL3R7c7nXgWLuzbEXPnTnXEFBEREZEBKVEmCbG3Yy/mIN9+vYkyC4s9HXv6PSbZkYzDdBC2wrQH2oclzpG0pXELdqP/soFhK8zmhs0jHJGIyNhhz86Ozh4LRwjs2hXbbpgmnrlzcRYXARCoqFRHTBEREREZkBJlkhBDLb5vYOCxefrfZxiku6PLLxt7GuMWW6KkOFMGnBlnGiZprrSRDUhEZIxxlbw/q2zvXiKdnX33FRfjnnMEmMb7HTE3EAkEEhGmiIiIiIxiSpRJQhybfywhK3TA4yJEmJM5Z8AEUoY7uvxyPBT0P734dBhgNWrEivDJkk+ObEAiImOMLS0Ne3YWWOAvK9tvvyM3F8+CBRgOe6wjZrijs58riYiIiMhEpUSZJMSxk48l35ePafT/LWhg4DJcOEwHxanFrKlZQ11X3X7H9c4oaw+0EwiP7ZkBk7yT+PaSbwPstyz12Pxj+djUjyUiLBGRMcVZXAIGhOobCLe07LdfHTFFREREZDBKlElCmIbJz0/8OQ7DgdHPNCoTkyJbEdcsuYYUVwo9oR62NG5hfe16Wv2tseNcNhc+hw8YH7PKzp9zPt9f/n2OyDyCbE82R2QcwXmzz+O82ecRYew3LBARGW62JB+OvDyg/1llMEBHzD3918MUERERkYlFiTJJmDmZc7hm2TWUpJXst29m2kwMw+BT0z/F8tzlFKUUYRombYE2NtRt4N2Gd+kKdgGQ6c4Eot0vx7pwJEx+Uj6XL7icf5z1D/72yb9xcuHJRIiwu3V3osMTERkTnMXFGDaTcEsrofr6fo/p7YjpyM2JdsTcvkMdMUVERESE/lvsiYyAyvZKJnkn8Z2l36E71M1bNW/htDk5pfAUZqXO4qmnngLAZtooSi0iPymf8tZyajprqO+up7GnkXxfPqmuVCA6o8yyLAxjgEJfY0Czv5mIFcFlc5HsTAZgWto03q5/mz0de8h2ZSc4QhGR0c90uXAUFBDYtRt/aRm2rKx+XxsM08Q9Zw6Gx0OgfBeBikoi3d3RbTZbAiIXERERkUTTjDJJiJ5QDxXtFQDkJ+VjN+0sz1vOhXMvZF7WvH7PcdqczMqYxZLcJWS4M4hYEao6qtjatBWAQCRAR7BjxJ7DcKjvis58yPZ8kBDLcGeQ6c7EwqK0tTRRoYmIjCnOggIMh51IVxeh6upBj3UVF+OeO+f9jpgN0Y6Yfv8IRSoiIiIio4kSZZIQZa1lRKwIqa5UOoIdRKwI6e70WBfLwfgcPuZnz2d+9nx8Dl+fjpjbmrYNZ9jDKmJFaOxpBCDb23fm2LS0aZiGSVNPEx2RsZ0MFBEZCYbDgbOoCAB/eTlWODzo8Y6cHLwLF2I4HNGOmOvWEe7Q71sRERGRiUaJMhlxrf7WWAfLHG9ObBbVtNRpB3WdDHcGS3KWMDtjdmxbZ7Bzv4L/Y0WLv4VQJITDdJDiTOmzz+vwku/LB6AuXNcnOSgiIv1zTJ6M6XZh+QMEq6oOeLwtLQ3vUYv36Yi5nlBj4whEKiIiIiKjhRJlMqIsy+K95vcAyPPlUdtVC0CuL5ckZ9JBX88wDHJ9uSzNXRrb1l/B/7GgN2GY5em/lk5haiEO00GAANWdgy8jEhGRaA0yZ0m0YUygogIrEDjgOR90xEyLdsR8+211xBQRERGZQJQokxFV01lDR7ADm2Ej2ZlMq78V0zApTi0+rOv6HD48dg8ATtOJgUF9dz1ra9eys3knwXAwHuEPG8uyaOyOzlrI8mT1e4zDdFCUWgTArrZdo/45iYiMBvacHMwkH1YwRGD30LoHRztiLlBHTBEREZEJSF0vZdj4w36e3fUsT5U/RUtPCzm+HI7IOIIZ6TMoSS2hsr0SgCnJU3DZXId9vwx3Bns69pDpyWRK8hRKW0pp6mmiqqOKmq4aCpMLmZw8GdMYffnhtkAbgUgAm2Ej3Z0+4HF53jxchotQJMSutl3MSJ8xglGKiIw9hmHgmj6d7o2bCOzZg2PKFEyP58Dn9dMRk/Z2iGjpu4iIiMh4pkSZDIuq9iouefYSqjqqMA2TiBVhS+MWXqh4gRlpM7j1+FvpDnXjMB1MTZ4al3v2JsqaepqYlTGL+dnzaepporSllM5gJ6Wtpezp3ENJagmTvJPics94aehuACDTkzloIs8wDCaZ0dj3duwlPykfn8M3IjGKiIxV9owMbOlphJtbCJSX454zZ8jnuoqLMb1eerZuJdzQgGfXrmhHTIdjGCMWERERkUQZfVNrZMwLhANc8uwlsTpavYXnI0S/7mzZyU2v3wRAUWoRdjM++do0VxqmYeIP+2O1yfYt+O+0OekJ9bClccuoK/hf3x2tT5btyT7AkeAzfWR5srCw2Nmyc7hDExEZF1zTpwMQrKk96G6WvR0xcTgwe3roWb9BHTFFRERExiklyiTunt/9PFUdVYStcL/7LSw2N26mqaeJPF9e3O5rM22kulIBaOz5oEtZb8H/5bnLKUopwjTMUVXwvyPQQU+oB9MwB112ua+S1BJMw6S5pzk2G01ERAZmS07GnhOdkRvYefAfMtjS0vAsWkTE6cTy96gjpoiIiMg4pUSZxN0zu57BPMC3lonJzuadca8XluHKAKC5p3m/fTbTRlFqEUfnHU2eL2/UFPzvnU2W4c4Y8uw6j93D5KTJAJS2lMZm7YmIyMBcJSVgGoSamgk17/86cSCm10t3cTHmPh0xA1XqiCkiIiIynihRJnHX6m+NLbMciIFByArF/d4ZnmiirMXfQjjS/4w2p83JrIxZLMldQoY7g4gVoaqjitU1q6lsqxzxpFPvjLCBul0OpDClEKfppDvUzZ4OvVETETkQ0+PBkZ8PRGeVHVIXS5sN9/z5OPJywQL/jh3433tPHTFFRERExgklyiTu8pPysRm2QY+JEBlSPa6D5XP4cNqcRKwIrYHBa5D5HD7mZ89nfvZ8fA4foUiI0tZS1tSsoa6rLu6x9acr2EVnsBMDg0x35kGdazftFKcWA7C7bXfCZsSJiIwlrqIiDLuNcHsHobpD+11vmCbuI47AVRL9HRyorKJn82ascP8f0IiIiIjI2KFEmcTdp6d/esD6ZPvK9eVS1lpGMBLfBE9vwqmpu2lIxyey4H9jd7S+TaorFYft4Duo5fpyY0m+8rbyeIcnIjLuGE4nzqnRbsuBsjKsyKHPInYWFeGeOye6nLO+ge7166MdMUVERERkzFKiTOJuae5Sjsk/ZtD6Yx8v/jjJzmQq2ipYU72Gyvb4LXnsLYjf7B96/ZlEFfyPdbv0HtrsOsMwmJE2A4Dqjmo6g51xi01EZLxyFBRgOJ1EunsI7t17eNd6vyOm4XAQbu+ga+1adcQUERERGcOUKJO4MwyDX5z0C06Zekr0MQY2w4aBgYnJRfMu4rbjb2Ne1jy8di/BSJDSllJWV6+mprPmsOu8pLvTMTDoDHbSE+o5qHNHsuC/P+ynLdAGHHx9sn2ludPI9mRjYbGz5eA7uYmITDSGzYaruAiAQPkurNDh1cy0paXhPWoxpteL5Q+oI6aIiIjIGDa0FnsiB8nr8PLzk37O19q+xqM7HmV3224y3ZlctuAycn25QDQ5lOnOpKazhvK2cvxhP9uatlHZXslU39RDvrfDdJDiSqHV30pzTzN5SXkHfY3egv9TkqdQ2lJKU08TVR1V1HTVUJhcyOTkyYfdsbO3iH+KMwWXzXVY1ypJK6Gxp5HmnmYauhsOK/EmIjIR2PPzMSsriXR1E6ioiHbEPAym14v3qMV0b95MuLmF7rffxjVjJs4pk+MUsYiIiIiMBM0ok2E1JWkKK/JX8JkZn+GieRfFkmS9DMMgLymP5bnLKUktwW7a6Qx28k7DO1SEKmIzrg5Wuiu6/LKpZ2h1ygYynAX/D7XbZX88dg9TkqYAUNpSOuKdO0VExhrDMHC+nxwLVlbGpbaY4XDgWbBAHTFFRERExjAlymRYVXVUEQgHcNvdTE4e+FN1m2ljaspUluctpyC5ANMw6bK6DrlGWIYnA4gmyuKRNIp3wf9gJEhLTwsQn0QZwNSUqThNJ92hbva074nLNUVExjPHpEnYUlOwwhECu3bF5Zr9dsR8553DXt4pIiIiIiNDiTIZNoFwgIq2CgCKU4qHtFTRYTqYljaNpTlLSTVSMYxojbC3at5ie9N2/OGhfeKf7EjGYToIW2HaA+2H9Tx6xbPgf2N3IxYWPocPr8Mbl/jspp3i1Ogbs11tuwiEA3G5rojIeOaaNg2A4N69RDrj1xAl2hFzbrQjZkMj3Rs2qCOmiIiIyBigRJkMm11tuwhbYZKdyUzyTjqoc912N3n2PI6adBRZniwsLKo7q1ldvZqy1jKCkcGL6huGEet+2dgT34LK8Sj4H89ll/vK9eWS5EgibIUpby2P67VFRMYjW1oa9qzM6FLJsvj+3nTkTMK7aJE6YoqIiIiMIUqUybDoCnZR3VENwLTUaRiGcUjX8Tl8zMuax6JJi0h1pRKxIlS0VbC6ejWV7ZWDLqvMcEeXXzb3NB/SvQ+kt+D/ktwlZLgziFgRqjqqWF2zmsq2gWMLR8Kx2mnZnuy4xmQYBjPSZwBQ01lDR0BvyEREDsRZMg0MCNXXE249+OX0g7GlpuJdctQHHTHXrSPU0BDXe4iIiIhI/ChRJsOirLUMC4ssTxZp7rTDvl6qK5VFkxYxL2seXrs3WlS/pZTV1aup6azpt1By74yy9kD7sC5DPNiC/71109x2N0nOpLjHk+pKJdubjYXFzpadcb++iMh4Y0vy4ciLdkj2l5bG/fqmx4P3qMXY0tOwwhG633mHQFVV3O8jIiIiIodPiTKJu5aeFhq6GzAwKEktieu1szxZLM1dyqz0WThtTvxhP9uatrG2di2N3X2XWLpsLnwOHzB8s8r2NdSC//Xd9dHn4o7vsst9laSWYBomLf6W2DJPEREZmLO4GEyDcEvrsMz4inXEzM97vyPme/Ts2KGOmCIiIiKjjBJlEleWZVHaGv00PteXG7dC9fsyDIO8pDyW5y6nJLUEu2mnM9jJOw3vsLFuY5+kVKY7EyC21HG4Hajgf2ew84Nll974Lrvcl8fuYUryFAB2tuyMS+dPEZHxzHS5cBYUAOAvLRuWBJZhmrhnz8Y1LfohUrBqjzpiioiIiIwySpRJXNV11dEeaMdm2GIdGIeLzbQxNWUqy/OWU5BcEJtBtW8Xyn3rlI3kp/YDFfx/q+YtQpEQpmGS4kwZ1himJk+NzWyratcSHxGRA3FOnYrhsBPp7CRUXT189yks7NMRs2v9enXEFBERERkllCiTuIlYEcrboh3DpqZEkzQjwWE6mJY2jWW5y/ZLStV01QAQiAToCI58YfsPF/zvFbEiVLVXDetML7tpjy193d22e1jrtImIjAeGw4GzqAgAf3k5Vjg8bPeKdcR0Ool0dEY7Yra3D9v9RERERGRolCiTuNnTsYeeUA9Om5MpSVNG/P5uuzuWlMryZGFhUdNZE9tf31U/4jH18jl8HJl1ZJ9tgxX8j5ccbw7JzmTCVpjy1vJhu4+IyHjhmDwZ0+3C8gcIDnPBfVtqKt6jFn/QEXP9enXEFBEREUkwJcokLoKRILvbdgNQnFKMzbQlLBafw8e8rHksmrSIVFdqbHtFewWV7ZUJq9fVFmgDwGbYmJk+c9CC//FiGAbT06YDUN1ZTXtAsxVERAZjmCbOkuhs3EBFBVZgeGfjqiOmiIiIyOiiRJnExe7W3YQiIXwOH7m+3ESHA0CqK5VFkxYxLW1abFtpSymrq1dT01kz4p3Geme0ZXoyyU/KH7Dgf1ewK673TXWlMsk7CYg+fxERGZw9JwczyYcVDBHYvXvY76eOmCIiIiKjR8ITZXfffTdFRUW43W6WL1/OmjVrBj3+zjvvZNasWXg8HgoKCvjGN75BT0/PCEUr/ekOdbO3cy8AJaklGIaR4Ij6KkguwG13xx77w362NW1jbe1aGrsbRyyOhp7ocppsT7Tb5UAF/9fWrmVn806C4WDc7l2SWhJrdpDIJagiImOBYRi4pkU/ZAns2UNkBP7O6Lcj5ttvqyOmiIiIyAhLaKLsb3/7G6tWreKGG25g/fr1LFiwgNNOO426uv5rNv35z3/mu9/9LjfccANbt27l97//PX/729/43ve+N8KRC0AoEqKyvZK3694mYkVId6eT6clMdFj9ynRH48rx5lCSWoLdtNMZ7OSdhnfYWLdxWJY97qsj0EFPqAfTMPsU9Yf9C/5HrAhVHVWsrllNZVt8loq67W4KkguAaG20RC0/FREZK+yZmdjS0yBiESgrG7H7OgsLcc97vyNmY5M6YoqIiIiMsIQmyu644w4uueQSVq5cyZw5c7jnnnvwer3cf//9/R7/+uuvc+yxx3LeeedRVFTEqaeeyrnnnnvAWWgSfzubd3LG42dwxuNncPFzF/P4e49TklKS6LAG1JucavG3MDVlKsvzljM1eWpsltVwLXvsVd9dH4tjoPptPoeP+dnzmZ89H5/DRygSimvB/6nJU2N10SrbKw/7eiIi413vrLJgTS3hjpHrnOyYpI6YIiIiIomSsERZIBBg3bp1nHzyyR8EY5qcfPLJvPHGG/2ec8wxx7Bu3bpYYqysrIynnnqKM844Y0RilqiIFeHKF6+ktqs2tu2lypd4vuL5BEY1uDRXGqZh4g/76Qp24TAdlKSVsCx3WZ9lj2/VvMX2pu34w/H99L6hO7rsMsuTdcBjM9wZLMlZwuyM2XEt+G8zbZSkRpOZu9t2x/05ioiMN7aUFOyTojUeA6UjW+Mx1hHT5/ugI2a9ls6LiIiIDDd7om7c0NBAOBwmJyenz/acnBy2bdvW7znnnXceDQ0NHHfccViWRSgU4vLLLx906aXf78e/z5KFtrZo58FgMEgwGL8aUIeqN4bREMtQNfY00tDRgB07EaJL+Bw42FizkU8WfTIu9xiOcfHZfDT3NFPbUcuUpCkA2LBRklxCjjuHXW27aOhuoLK1kr1te5mcNJkpyVNwmI7Dum93qJvW7lYMwyDFnjLk55TpzCQtM42qjioq2ytp6mqivr2ePaE9tHW3kULKQceS4cjAa3ppC7TxXuN7zEqfddDXGK3G4s/SSNC4DExj0z+NS19mwRTCNdWE6+oIZUU/7BixsbHbcRw5D/+WLYSam+nYuBHntOk4CqaMzP2HSN8zA9PY9G80jctoiEFEREYXw0pQS6W9e/cyefJkXn/9dVasWBHbfs011/DKK6+wevXq/c55+eWXOeecc7jllltYvnw5O3fu5KqrruKSSy7huuuu6/c+N954IzfddNN+2//85z/j9Xrj94QmoB6rh9pwLV7DS7YtO9HhHFBjuJH6SD0+w0eBvaDfY7oiXdRH6um2ugEwMckys0gzozPShuu+BxKyQjREGmiJtABgYJBmppFpZmI3Di7f3R3pZnc42sWtyF6E23Af4AwRkYnNWVODo6mJiNtNd3ExjHTTmkgEZ20tjuZmAILpGQRyc0Y+DpFxqKuri/POO4/W1lZSUg7+Q0gRERl/EpYoCwQCeL1eHn30Uc4666zY9gsvvJCWlhb+8Y9/7HfO8ccfz9FHH83Pfvaz2LY//elPXHrppXR0dGCa+ycy+ptRVlBQQENDw6h4MQwGgzz33HOccsopOByHN3NpJD347oP8dtNvsRt2IlaEVHcqD57+IJO8k+Jy/eEYl85gJ2tr12IzbKzIWzFgrTCILpXc1baLzmAnEC2GX5RSxCTPpIPu6rmhbgNtgTZmpM0gPyn/sJ5DS1cLf3v+b8xYMAObzYbdtDM1eSqTkyYfVCJva9NW6rrqSHWlsjB74WHFNFqM1Z+l4aZxGZjGpn8al/1ZgQBdq1cT8gd4s6Gej372swkZm2BlZWwJqC0jA9ecORj2hC0OiNH3zMA0Nv0bTePS1tZGVlaWEmUiIhKTsL+unE4nRx11FC+88EIsURaJRHjhhRe48sor+z2nq6trv2SYzRZNdgyU73O5XLhcrv22OxyOhL8w72u0xXMglyy8hOL0YtbUrCHZmczZs86OW5JsX/EclzRHGl6Xl0A4QJfVRYYjY8Bj8xx55CbnUtNZQ3lbOYFwgJ1tO6nurqY4tXhItcaAaE20SBd2u53clFwctsN7LmneNArsBSzKWcTuzt10Bjup6Kygzl9HSWrJkP8fzMqaRUt1C53hTpqDzcPy/y5RxtrP0kjRuAxMY9M/jcs+HA6s4mK6d+7EWVeH3WZLyNg4SkpwJCXh37oVq7WN4Dvv4Jk/H9M9OmYG63tmYBqb/o2GcUn0/UVEZPRJ6MeQq1at4sILL2TJkiUsW7aMO++8k87OTlauXAnABRdcwOTJk7ntttsAOPPMM7njjjtYtGhRbOnlddddx5lnnhlLmMnIObnwZE4uPPnAB44ime5MqjuraepuinXCHIhhGOQl5THJO4m9HXvZ3R5NTG1u2EyaK43i1GJSXamDXqO3iH+KMwWXbf+E7aFKd6eTnZRNbVctZa1lsYL/Ve1VTEubdsC4XDYXU5OnsqttF6UtpWS6MwedYSciMtE5CwroqajADAQI7d2Ls7g4IXE4Jk3CdLvpfvudaEfMdevwzJ+PLTk5IfGIiIiIjDcJTZSdffbZ1NfXc/3111NTU8PChQt55plnYgX+Kyoq+swg+8EPfoBhGPzgBz9gz549ZGdnc+aZZ/KjH/0oUU9Bxph0d3o0UdbTNORzbKaNgpQCcpNyqWyrpKqjihZ/CxvqNpDlyaIktQSvo/96d/Vd0Q5l2Z7413AzDINcXy7Znmwq2yupaK+gLdDGhroNZHuyKU4tHjAugILkAqo7q/GH/VR1VFGYUhj3GEVExgvDbsdRVASvv05g9248BQUJW/ZoS0nBu+Qouje9TaSzk+7163HPmYM9e/TXCxUREREZ7RJe2OLKK68ccKnlyy+/3Oex3W7nhhtu4IYbbhiByGQ8SnenY2DQFeqiJ9SD2z705SoO00FJWgn5SfnsbttNTWcNDd0NNHY3kuvLpSi1qM+ssWA4SKu/FYBMT2bcn0svm2mjKLWI/KR8ylvLqemsob67nsaeRvJ9+RSmFPa75NNm2ihJLWFr01Z2t+0m15cb11lvIiLjjT03l4jTCcEggYpKXCWJmVUGYLrdeBcvoufddwk1NdP9zmZcM6bjLDi0pjEiIiIiEnVobfxExiiH6SDFFS3U2tzTfEjXcNvdzMqYxZLcJWR5srCwqO6sZnX1aspaywhGom3GG3sasbDwOXyDzuyKF6fNGYsrw51BxIpQ1VHF6prVVLZVErEi+52T48shxZlCxIpQ1lI27DGKiIxlhmkSmBSt6RisrCASCCQ2HocD94IFOCZHG8X439tJz/btA9ZtFREREZEDU6JMJpx0VzrAQS2/7I/P4WNe1jwWTVpEqiuViBWhoq2C1dXRxFRtVy3AkAv/x4vP4WN+9nzmZ8/H5/ARioQobS1lTc0a6rrq9jt+evp0AGq7amkLtI1orCIiY004JQUzJQUrHCFQXp7ocDAMA/esWbimTwMguGcvPZs2YYVCCY5MREREZGxSokwmnAxPtIh/U09Tv7OsDlaqK5VFkxYxL2ten8RU74y1kU6U9cpwZ7AkZwmzM2bjtDljBf/X166PLQmFaKOBHG+0LuDO5p0JiVVEZCxxlpQAENy7l0hXV4KjiXJOnYrnyHkYNpNQUzNd69cT6elJdFgiIiIiY44SZTLhJDuScZgOwlaY9kB73K6b5cliSc4SZqXP6rN9W9O2WPfLkdZb8H957nKKUoowDTNW8P/dhnfpCkbf4JWklcT21XbWJiRWEZGxwpaWhj0rEyzwl46eZev27Gw8ixZhOJ3Rjphr1xFu00xhERERkYOhRJlMOIZhkO6OLr9s7GmM+7XzkvL6zCLrDHayuWEzG+s29pnJNZJ6C/4fnXc0eb48DAzqu+tZW7uWnc07MTFjXS9LW0sJR8IJiVNEZKxwlkwDA0L19YRbE/O7vT+9HTHNJB9WIED3hg2E6usTHZaIiIjImKFEmUxIGe7o8stDLeg/mIgViV13XtY8piZPxTRMWvwtbKjbwOaGzbGZXCNtsIL/AE7TSSAcoLK9MiHxiYiMFbYkH47cXGB0zSqD3o6Yi7FnZmCFI3S/s5lARUWiwxIREREZE5Qokwmpd0ZZe6CdQDi+Xcuae5oJW2GcppNMdyYlaSUsz1sem8nV0N3AWzVvsb1pO/6wP673Hqr+Cv6Xt5YTiETHoqK9gp6QatuIiAzGWVwMpkG4pYVQQ/yW2FuWRes//kHNzTdTf/fdhFtaDvoaht2Oe/58HJMnA+DfWRrtiBk5/NqcIiIiIuOZEmUyIblsLnwOHxD/WWW99ciyvFkYhhG7X+9MrixPFhYW1Z3VrK5eTVlLGcFIMK4xDNWHC/73ilgRNtZvTEhMIiJjhel24ywoAKKzyizList1a2+9lb3f+S7Nf3uEhrt/Q/nnPn9IyzujHTFn4poR7W4c3LOXnrffxgom5jVHREREZCxQokwmrEx3JhDtfhkvlmXR2B2te9Zft0ufw8e8rHksmrSIVFcqEStCRXsFq6tXU9lWGZcunAfrwwX/e/WEenh97+sJWyYqIjIWOKdOxXDYiXR2EqqpOezrBWtraf7jn6IPwmGIRAju2UPLo48deowFBR/qiLlBHTFFREREBqBEmUxY+9Ypi9csgLZAG4FIALtpJ82VNuBxqa5UFk1axLysebGlj6WtpayuXk1NZ03c4jkYvQX/j8k/JrYtEA7wVs1b7GzeSTCsGQgiIh9mOBw4C6PNUPxlZVjhw2uG0u8yS8MgUFV5WK8N9uxsPIsXY7icRDrVEVNERERkIEqUyYSV4krBZtgIRAJ0BDvics36rmhnsUx3JqZx4B+vLE9Wn6WP/rCfbU3bWFu7NraEc6Q5bU5W5K+IPbawYgX/EzXrTURkNHNMmYLpdmH5AwSrqg7rWs6pU7GlpcH7S/cBiESwpafTtXrNYXWwtCUn4z2qb0fMYF3dYcUrIiIiMt4oUSYTlmmYsaL+8Vp+Wd8dfQOT7c0e8jn7Ln2cljoNu2mnM9jJ5obNbKjbQKv/4OvSHC6XzUVxanHsscfuic16W1OzhrouvbESEellmCbOkhIAAhUVh1UDzPR4KPjtPdjS02PbMlauxLtoMZGuLrrf2UzXunWEmg+tvuaHO2L2bH5XHTFFRERE9mFPdAAiiZTuTqehu4HmnmYKUwoP61rtgXb8YX80AedKP/AJH2IzbRSkFJCblEtlWyVVHVW0+lvZULeBLE8WJaklOHAcVowHoyC5gOrOanpCPWR7s/HavZS1ltET6mFL4xaq2quYljaNVFfqiMUkIjJa2XNyMCsqiHR0Eti9G9f06Yd8Lc+CBcx4+SWCNTXY0tOxJSdjBYMEKisJVlYSbm2je8NG7FmZOEtKsCUlHdT1ezti+ne8R3DPHvw7S4l0d+OaMQPD1GeoIiIiMrHpryGZ0HrrlLX6WwlFQod1rd7ZZJnuTGym7ZCv4zAdlKSVsDxvOXm+PAwMGrobeKvmLXY07yBojUytMNMwmZY6DYCq9irSXGmxgv+mYdIWaGND3QbebXhXBf9FZMIzDAPXtOjvzEBV1WEXyzeczugyzOTk6GOHA1dJCd6jj8YxOR8MCDU00rXmLXq2bDno+6kjpoiIiEj/lCiTCc1j9+Cxe7CwaPG3HNa1ertdZnoy4xBZdPnjrIxZLMldQpYnCwuL6s5qykJllLWWEYwM/5uZbG82aa40IlaEstayWMH/o/OOjiXx6rvrWVu7VgX/RWTCs2dmYktPg4hFoKxsWO5huly4Z83Ct3w59kmTAAjW1NL55pv433sPKxA4qOv12xGzu3s4QhcREREZE5Qokwmvd1ZZb6LrUHQFu+gMdmJgxC1R1svn8DEvax6LJi0i1ZWKhUVleyWrq0emuP60tOgMibquuli9NKfNGUviZbgziFgRFfwXEYHYrLJgTS3hjvg0iumP6fXimTcX75KjPkjOVVbR+eabBHbtOqjum/t1xFy3Xh0xRUREZMJSokwmvN5E2eEU9O/tUJnmTsNhDk8dsVRXKguzFzLFNgWfwxcrrr+6ejU1nTVYljUs9012JpPnywPgveb3+tzH5/AxP3s+87Pn94lJBf9FZKKypaTEZnoFSktH5H7eRYvwLFwQ7WYZCuMvK6fzjTcJ7tmDFRnaBxe25GS8S5bEOmJ2rV+vjpgiIiIyISlRJhNemisN0zDxh/2HXGsr1u3SM/Rul4cqyUziqElHMTtjNk6bE3/Yz7ambaytXRtL2MVbcWoxNsNGR7CD2q7a/fZnuDNYkrMkFlNvwf/1tesT0rVTRCSRXCXF0RpijU2H3J3yYNkzMvAuXYp77hxMjxsrEKBn+w661qwhWDu0hJfpckU7YmZlQsSKdsTcvXuYIxcREREZXZQokwnPZtpinRsbew5++aU/7Kc90A5AlicrrrENxDAMcn25LM9dzrTUadhNO53BTjY3bGZD3Ya4J6ecNmesK2hZa1m/jQ/2jUkF/0VkIjO9Xhz5k4GRmVXWyzAMHDk5eJcvxzVzBobDQaSrm55336Vr7dohJe0Mux33kUfimBKN319aRs+2bUOemSYiIiIy1ilRJgJkuKLLL5t7Dv6T//qu6GyyVFcqTpszrnEdiM20UZBSwPK85UxNnoppmLT6W9lQt4HNDZvjmpyakjwFj91DIBygor1i0JhU8F9EJjpncRGG3Ua4rX3IM7rixTBNnFOm4FtxNM6iIgybSbitne4NG+neuPGAtdMMw8A9cyaumTPAgODeanXEFBERkQlDiTIRIMMTTZS1+FsIR4ZeABk+qE82UrPJ+uMwHZSklbA8b3ksOdXQ3cBbNW+xvWk7/rD/sO9hGmassH9VexXdocG7oqngv4hMZKbTiaOgAIBAeVlCZmQZdjuukmK8K1ZEZ4iZRrSz5Zq36H733QN2t3ROmYLnyCPVEVNEREQmFCXKRIgWpXfanESsCK2BoS9bDIaDsWWOiUyU9XLZXMzKmMXS3KVkebKwsKjurGZ19WrKWsoIRg5vNkCWJ4s0VxoRK0JZa9mQzlHBfxGZqJwFBRhOJ5GuboJ7qxMWh+l04p45E9+yZdhzoo0GQrV1dK5eTc+OHUQCgQHPtWdl7d8Rs1W1J0VERGT8UqJM5H2Z7kwAmrqH3v2ysacRCwufw4fH7hmu0A6a1+FlXtY8Fk1aRKorlYgVoaK9gtXVhz+ba3ra9Ohyyq56WnpahnyeCv6LyERj2O04i4oACJSXY4X2r+84kkyvF8/cuXiXLsGekQ4Ri2DVHrrefBP/IPHt1xFzwwZ1xBQREZFxS4kykfelu9MBaOoZeqKstz7ZSHS7PBSprlQWTVrEvKx5fWZzra5eTU1nDZZlHfQ1k5xJ5PpyAdjZsvOgrqGC/yIy0Tjy8zC9HqxgkEBFZaLDAaKJL8/ChXgWLcSWnIQVChMo30Xnm6sJVFX1u0xUHTFFRERkolCiTOR96e50DAy6Ql30hHoOeHwoEqLZHy3+n+0dnYmyXlmerD6zufxhP9uatrG2dm2sxtrBKE4txmbY6Ah2UNNZc9Dnq+C/iEwUhmniLCkBIFhZMegyx5FmT0/Hs2QJ7rlzo8m8QAD/jvfoWr2aYG3tfh+EqCOmiIiITARKlIm8z2E6SHGlAEPrftnU00TEiuCxe/A5fMMd3mHbdzbXtNRp2E07ncFONjdsZkPdhoNa/ui0OSlKKQKgvLWcUOTQlhOp4L+ITASOSZOwpSRjhSMEynclOpw+DMPAkTMJ77JluGbOjNZU6+6h590tdK9dS6ipab/j9++I+Q6ED64RjoiIiMhopUSZyD7SXUNfftk7E2u0LrsciM20UZBSwPK85UxNnoppmLT6W9lQt4HNDZuHvPxxcvJkPHYPgUiAiraKw4pJBf9FZLxzTot2DQ7u3UOka/QtMzdME+eUyfiOXo6rpBjDbiPc3kH3xk10bdhAuK2tz/Gxjph2G5GWZjy7dqkjpoiIiIwLSpSJ7CPDkwF8MFtsIBErQmN3IwCZnswRiS3eHKaDkrQSluctjy1/bOhu4K2at9jetB1/2D/o+aZhMi0t+savqqOK7tDhv0FSwX8RGa/s6enR+l5WdMniaNXbgMB39NE4C6aAaRBubqFr7Tq6N7/bJ8kX64jpdGH6/XSvV0dMERERGfuUKBPZR7IjGYfpIGyFaQ+0D3hcc08zYSuM0+YkxZkyghHGn8vmYlbGLJbmLiXLk4WFRXVnNaurV1PWUkYwMnC9sCxPFunudCJWhLKW+LzxU8F/ERmvemuVherrR31CyXA6cc2YgW/5chy5OQCE6uroXL2anu07YrXWbElJuI9aTMTthmAw2hGzVjOBRUREZOxSokxkH4ZhxLpfNvY0Dnhc77LLLE8WhmGMSGzDzevwMi9rHosmLSLVlUrEilDRXsHq6sHrhU1Pmx4rxt/S0xK3eFTwX0TGG1tSEo68aNfg0TyrbF+mx4N7zhy8y5Ziz8wAC4J79tD1xhv4y8qwQiFMl4vuwkJsme93xHz3XQK7diU6dBEREZFDokSZyIdkuKPLLwcq6G9Z1pitTzYUqa5UFk1axLyseX3qha2uXk1NZ81+XdB8Dh95SXkAvNfy3n77D5cK/ovIeOIsLo4uZ2xpIdQ48Acyo40tKQnPggV4Fi3ElpoSbUywazedb7xJsLIKDAPXvHnR5ZqAv6ycnq1b1RFTRERExhwlykQ+pHdGWXugnUA4sN/+Vn8rwUgQu2kn1ZU60uGNmCxPVp96Yf6wn21N21hbuzaWKOxVnFIc66JZ01kzLPGo4L+IjAem241zyvvJpJ2lcf9wYbjZ09PxHnUUniPnYXq9WMEggdKdeMrKCNfW4pw+HdfMmdGOmNU1dG/ahBXUDGAREREZO5QoE/kQl82Fz+ED+p9Vtu+yS9MY3z9C+9YLm5Y6LZYM29ywmQ11G2IF9h02B4UphQCUtQ5e1+xwDVTwf0PdBroiql8mIqOfs7AQw2En0tlJqGZ4PlwYbvbsbLzLl+GePStazD8QwL9tG11vvYXpceOZPz/aObO5ha5169URU0RERMaM8f0uX+QQZbqjnSybepr221ffXQ9EE2UThc20UZBSwPK85UxNnoppmLT6W9lQt4HNDZvpCnYxOWkyXruXhu4GfvbWz7ji+Su46sWreGzHY3EvwD9Qwf+KcAVbGreo4L+IjGqGw4GzMPrhgr+sDCscTnBEh8YwDBz5+XiWLyMwaRLY7UQ6Oune9DaB3RW4pk/HcDmJdHXRtXYd4ZaWRIcsIiIickD2RAcgMhpluDOoaK+guacZy7JiBfvbA+34w35MwyTdlZ7gKEeew3RQklbC5OTJ7GrdRU1nDQ3dDTR2N5Lry2VPxx5ufvNmwpEwFhYGBi9Wvsg9m+7h/tPupyClIK7x9Bb8z0/K573G94BoIrO1tpV8Xz6FKYU4bI643lNEJB4cU6YQrKoi0uMnuGcPzqlTEx3SITNsNoJZWXiXL8fau5dAVRXhlhbCLS3YUlOIhMNYwSBdGzfiPuIIHDk5iQ5ZREREZECaUSbSjxRXCjbDRiASoCPYEdveO5ss052JzbQlKryEc9lczMqYxdLcpWR5srCweLfhXW5+42ZCkRAW0Zo7vV/ru+v56gtfHbbC+06bk5npMym2F6vgv4iMCYZpRgv7A4Hdu8dFHS/D4cA1fTq+o4+Odvc0INzahhV6f8ZcxKLn3S3qiCkiIiKjmhJlIv0wDTNW1H/f5ZcNXR/UJxPwOrzMy5rHokmLWFu7lrDV//KhsBVmV9su3tj7xrDG4zJcHJl1pAr+i8iYYM/NxUzyYQVDBHbvTnQ4cWO63biPOALv0mXYs/d/vVRHTBERERnNlCgTGUBvoqy3oH9XsIuuUBemYZLhyUhkaKNOqiuVyvbK2Ayy/tgNO+tq141IPAMV/F9fuz7WgEBEJNEMw8A1bRoAgaoqIj09CY4ovmxJPjxHHol38SJsaX27RAera2h/6SWqb/4h5Z/9HHuu+Q49O3YkKFIRERGRD6hGmcgAMtzRZFirv5VQJBRbdpnmSsNhqu7Vhx1oKaqFhd0cuV85vQX/sz3ZVLZXUtFeQVugjQ11G8j2ZFOcWozX4R2xeERE+mPPzMSWlka4pYVAeTnuI45IdEhxZ0tLw7t4MaGGBvylZUQ6O4l0dlDzw1uIdHRAJELPtm20/+tfFD/6d1wzZiQ6ZBEREZnANKNMZAAeuweP3YOFRYu/hYbu6LLLbE92giMbnU6YcgIGxoD7w1aYYycfO4IRRfUW/D8672jyfHkYGNR317O2di07m3cSDI/9ukAiMra5pkdnlQWrawh3dBzg6LHLnpWFd9lS3EfMpmvtOiJtbdC7/DIcxgqFaHzgwYTGKCIiIqJEmcggemeV7WnfQ3ugHYBMT2YiQxq1Pj390yQ7kzGN/X+t2AwbC7MXMj9rfgIii3LanMzKmMWS3CUq+C8io4otJQX7pEkABEpLExzN8DIMA0deHpgG2D40EzkcJrB7V0LiEhEREemlRJnIIN5rfo+frPkJ5z99Ptf+51peqnip30SQRGu6/e7U35HuitZ2sxt2bEb0TdCczDnc9dG7MIyBZ5yNFJ/Dx/zs+Sr4LyKjiqukGAwINTYRam5OdDjDzrNgAYQ/1ADGNPEuXpyYgERERETepxplIgN4fc/rfOc/34k97gh28PjOxzEMgxuPuTFxgY1iczPn8uznnuX53c+zqX4TDtPBiQUnsiRnyahIku0rw51Bek46tV21lLWWxQr+V7VXMS1tGqmu1ANfREQkTkyvF0f+ZIJ79hAoLcW+ZEmiQxpWqWecQcvfH6V73Tqw2yEcxpGfT8bKlYkOTURERCY4JcpEBvCbTb/BxCRC3yV5j7/3OJcvuJxcX26CIhvdnDYnZ5ScwRklZyQ6lANSwX8RGU2cxUWEaqoJt7UTrK3DkTMp0SENG8PppPCB+2l79jl63nkHZ1EhKWd+CluSL9GhiYiIyASnRJnIALY2bt0vSQbR7o3bm7YrUTaO9Bb8z0/Kp7y1nJrOGuq762nsaSTfl09hSiEOmzqdisjwMp1OHAVTaXnsUWp/8hOCe/ZgS0sj7XOfJfPiS8ZdEslwOkn95CdI/eQnEh2KiIiISIyKLYkMYJJ34E/ylSQbn1TwX0QSre2Zp2n6/f0EKyogFCLc0EDj7+5l9wUXEOnuTnR4IiIiIuOeEmUiAzh/zvn7bbMZNuZnzWdWxqwERCQjRQX/RSQRgnV1NPz67ugDy/pgRySCf+tWWh57PDGBiYiIiEwgSpSJDODc2eeyct5K7MYHK5TnZ8/nzo/cmbigZERluDNYkrOE2RmzcdqcsYL/62vX0+pvTXR4IjLOdLzwQt8E2Ye0/fOfIxiNiIiIyMSkGmUiAzANk1VHrWLl3JW81/weWd4sSlJLEh2WjDAV/BeRkRLp7gHD6D9ZZllEOjtHPigRERGRCUaJMpEDSHensyxvWaLDkARTwX8RGW7eoxZDZIBaiDYb3qOXj2xAIiIiIhOQll6KiBwEFfwXkeHinj8f77JlYLP13WEYGA4HGefvXztTREREROJLiTIRkUOggv8iEm+GYTDl7l+T9JGPRJdgvs+WmUnBfffhnDo1gdGJxMf2mnZ+8/JOAN7Zo3qfIiIy+mjppYjIYchwZ5Cek05tVy1lrWWxgv9V7VVMS5tGqis10SGKyBhiS06m4Ne/IlC1h54d2wlVVmGfMhlXcVGiQxM5bHe/tJPNz/2B8xz/pmnxN3js9z/l/y0+n5vPOhJjn+SwiIhIImlGmYjIYeot+L88dzlFKUWYhhkr+P9uw7t0BbsSHaKIjDHOKZNJ+ehH8R59NIZhEqqtTXRIIodl855Wtj33AP/j/CXLzK0A3Oh4COfa3/Ly9voERyciIvIBJcpEROKkt+D/0XlHk+fLw8CgvruetbVr2dm8k2A4mOgQRWSMceRMAiDU0IAVDic4GpFD939vV7PS/gwRC94JF9K95238ERtfsT/F/27am+jwREREYpQoExGJMxX8F5F4MVNTMd0urFCYUGNjosMROWSBUAQnQUwDDCxMK0wHHpyE8If1uigiIqOHEmUiIsNEBf9F5HAZhoE9JweAUK1+b8jY9ZHZ2fxfeAWW9cG2MAZPho/ho7MmJS4wERGRD1GiTERkmGW4M1iSs4TZGbNx2pyxgv/ra9fT6lfHLxEZXCxR1tiAFQolOBqZcBp2whNXwB1z4bcnwroHIXLwM8COm55F5REX86vwWfhxAPBkeAXPT7mCMxfkxzloERGRQ6dEmYjICFDBfxE5VLakJEyvFyIWoYaGRIcjE0nDTvjdifD236CtCqo3wf9eBU9fc9CXMgyDu85bQuHnbuWVtM8A4Prod3nw4hNw2vWWRERERo9R8ap09913U1RUhNvtZvny5axZs2bAY0866SQMw9jv3yc+8YkRjFhE5NCo4L+IHAp7b1F/db+UkfTvn0GoGyK9jSTeXzf51n3QVH7Ql7OZBv+1cDKnzs0F4Ix5uUqSiYjIqJPwV6a//e1vrFq1ihtuuIH169ezYMECTjvtNOrq+q/D8fjjj1NdXR37t3nzZmw2G5///OdHOHIRkUOngv8icjAcvcsvm5qwAoEERyMTRvkr+yTJ9mVBxZsjHo6IiMhISHii7I477uCSSy5h5cqVzJkzh3vuuQev18v999/f7/EZGRnk5ubG/j333HN4vV4lykRkTFLBfxEZCtPrxZacBBYE6+oTHY5MFJ50wOh/nyt1REMREREZKQlNlAUCAdatW8fJJ58c22aaJieffDJvvPHGkK7x+9//nnPOOQefzzdcYYqIDDsV/BeRA4kV9a/T8ksZIUetJLbcMsYEVwo43NDdkoCgREREhpc9kTdvaGggHA6T8/4ffr1ycnLYtm3bAc9fs2YNmzdv5ve///2Ax/j9fvx+f+xxW1sbAMFgkGAw8bWAemMYDbGMJhqX/mlcBjZexibTmUlaZhpVHVVUtlfS1NVEU1cT2Z5silOL8dg9B3W98TIuw0Fj0z+Ny8ASPTZWejrhcIhwYyO29nZMtzshcXxYosdlNBvzY7PwAtj1Osa2/wMMLMMETxrGcd8AfyeUv4qVOR3SS8AYYOZZPyLvd80MhkMJH5tE319EREYfw7KsD39MNGL27t3L5MmTef3111mxYkVs+zXXXMMrr7zC6tWrBz3/sssu44033uDtt98e8Jgbb7yRm266ab/tf/7zn/F6vYcevIjIMAtZIRoiDbREWgAwMEgz08g0M7EbCf2cQ0QSxL1rF7auLgI5OQQzMxMdjox3VoTs9ncxrRAt3mL8jjQADCtESncl7mALAEGbl1ZPEWGba0iX9VdtACCUPQ+fyzEckQ9ZV1cX5513Hq2traSkpCQ0FhERGR0S+k4rKysLm81G7Yc6ONXW1pKbmzvouZ2dnfz1r3/l5ptvHvS4a6+9llWrVsUet7W1UVBQwKmnnjoqXgyDwSDPPfccp5xyCg5HYv9QGE00Lv3TuAxsPI9NZ7CTstYymnqaALCbdqYmT2Vy0mRMY/AV9ON5XA6XxqZ/GpeBjYaxCe7ZQ+C99zCTkvAsWZKQGD5sNIzLaDXmx6atGqPGDnY3VvEJ8OHXnLa9GHVbIBIC046VPRtSpxzwshufb6W8rIzjTzie3MyMYQp+aHpXm4iIiPRKaKLM6XRy1FFH8cILL3DWWWcB0anYL7zwAldeeeWg5/7973/H7/dz/vnnD3qcy+XC5dr/0y2HwzGq/mAZbfGMFhqX/mlcBjYexybNkcZi72KaepoobSmlM9hJRWcFdf46SlJLmOSddMBrjMdxiReNTf80LgNL5NjY8/MJl5dDdw+2YBBzFM2O1/fMwMbs2HTuBbsdMovA2c9sscxCSJkE1W9DdxM0bgN/E+QcCXbngJc1zWjCzWGzJ3xcEn1/EREZfRLe9XLVqlXce++9PPTQQ2zdupUrrriCzs5OVq5cCcAFF1zAtddeu995v//97znrrLPI1LIDEZkgVPBfRAynE3tGdAZOsFZF/WUY9bRBdzNgQGrBwMc5PFCwDLJmRmecddTBrv9Ah7qziojI2JTwIjdnn3029fX1XH/99dTU1LBw4UKeeeaZWIH/ioqK2KdOvbZv386rr77Ks88+m4iQRUQSxjAMcn25ZHuyqWyvpKK9grZAGxvqNsQK/nsdo2eGiYjEnz0nh1BjE6HaOlzFxYkOR8ar1sro16RJ0Q6XgzEMyJwGvmyo3gSBDtizFtKmQvZsMG3DH6+IiEicJDxRBnDllVcOuNTy5Zdf3m/brFmzSGAPAhGRhLOZNopSi8hPyqe8tZyazhrqu+tp7Gkk35dPYUohDpuWk4iMR/asLDANIl1dhNvbsSUnJzokGW/CIWjbG/3vtMKhn+dOgcJjoGEHNO+ClgrobIC8BeBJG45IRURE4i7hSy9FROTQOW1OZmXMYknuEjLcGUSsCFUdVayuWU1lWyURK5LoEEUkzgy7HXtmFgAhLb+U4dC2J1qg3+kD30GWOTFtMOkImLIU7C4IdkHFm9CwE/RBt4iIjAFKlImIjAM+h4/52fOZnz0fn8NHKBKitLWUt2rfoi2ijl4i4409J9rEI1RXp1n2En8tFdGvaVMP/Rq+LCg6HpJzAQsa3yNU/h/W3P9tOl+/D4CNf/8RPV0dhx+viIhIHClRJiIyjvRX8H9veC8b6jao4L/IOGLPzMSw24j0+Im06mdb4qirKVpjzLBByuTDu5bNAfmLIHc+mHZ2PvFjluz+HT6rE4Aj655ky93nxCFoERGR+FGiTERknOkt+L88dzlFKUUYGLGC/+82vEtXsCvRIYrIYTJstmitMiBYW5fgaGRcadkd/ZqSH010xUPqZDoy5jG99Q1M44PNNiwWd/6H6t3b43MfERGROFCiTERknLKZNgpTCplmn0aeLw8Dg/ruetbWrmVn806C4WCiQxSRw2B/v0N4qL5eyy8lPoI90P5+3bvDWXbZj46uTuzGB3UzI4aJi+jrUHtjdVzvJSIicjiUKBMRGefshp2Z6TNV8F9knLGlp2M4HFiBAOHm5kSHI+NBaxVggSc92sEyjiblF7PHyCFsGTSSSoVrNnbCtOFl6hFL43ovERGRw6FEmYjIBDFQwf81NWuo69LSLZGxxjBN7JPeL+qv7pdyuCwLWuNQxH8Aps1G66m/xI+TCCZB000YGztX/BS3xxf3+4mIiBwqJcpERCaY/gr+b2ncwvra9Sr4LzLGOHq7X9bXY0U0O1QOQ0cdhPzRumRJucNyizkrPk7ga29TWvh5AFq//AKLT/vSsNxLRETkUClRJiIyAX244L9pmCr4LzIGmampGC4nVihMuLEx0eHIWNby/myy1KlgDt9bhLSsXKYtOwOAzJz4z1wTERE5XPZEByAy1lmBAO0vvEDP9u1YPX5sqSn4TjgBz9y5iQ5N5IBspo2i1CLyk/Ipby2nprOG+u56GnsayfflU5hSiCNeXc9EJO4Mw8CRk0OgopJgbR327OxEhyRjkb8Duhqi/51WkNhYREREEkyJMhk+kTDsfAHKX4FAB7jTYPYnYcoSMIwDnj7aRbq6aLj3Xpr//Bcira1gf//HybKo/+VduOfNJfPSS0k59dTEBioyBE6bk1kZs5iSPIXSllKaepqo6qiipquGwuRCJidPxjQ0CVlkNLK/nygLNdRjhUIYdv15JweptTL61TcJHJ7ExiIiIpJg+ktKhsc7j8Jz10PbHjD3+TZ77U6YNBc+cTsUrkhYeIcr1NxMxUVfwb99O/TWhAmF+hzTs2Ure75+Ff6vfpXsr38tAVGKHLzegv9NPU2UtpTSGeyktLWUPZ17KEktYZJ3UqJDFJEPsSUnY3o9RLq6CTU04Mg9tPpSEb+fcEsLps+HLSkpzlHKqBUJQ+ue6H8PQxF/ERGRsUbTAyT+3vwfeOwr0SQZQCT0wT+A+q3w0CfhvecSF+NhsAIBKi+/Av+OHR8kyfrz/r6G3/yGpj/8cYSiE4kPFfwXGVvsk3KAQ+t+GaisZO8PrmPH0mXsPPEkdixdRsXFF9O5ek28w5TRqG0vRILRmWS+rERHIyIiknBKlEl8VbwJz3x38GOsSPTTy799CdprRiauOGp7+ml6Nm2CcHjI59TdcQfhjo5hjEok/lTwX2TsiHW/bGrCCgSGfF7Ptm2Uf/oztP6///fBeZZF5xtvUvHlL9Py+P8bjnBlNGnZHf2aNnVclMYQERE5XEqUSXy9cXffpZYDsiDsh/V/GPaQ4q3pj3866G5Qlt9P65NPDlNEIsOrt+D/0XlHk+fLw8CgvruetbVr2dm8k2A4mOgQRSY80+fDlpwEFoTq64d0jhWJUPX1q4h0d+//4U84DJZF9Q9+QKBqzzBELKNCdzP428EwIWVKoqMREREZFZQok/jpqIdt//fBEssDsSLw1n1gWcMbVxz5y8rp2bx58CWXA2j5+6PDEJHIyOkt+L8kdwkZ7gwiVoSqjipW16ymsq2SiHXwPxciEj/2nOjyy2Bt3ZCO73zjDYIVFQecId3yyCOHHZuMUi0V0a/JeWB3JjYWERGRUUKJMomf5vJo8utgdNTCGFq+FaqpPrQTLYtQ9SGeKzLK9Bb8n589H5/DRygSorS1lDU1a6jrGtobdBGJP/uk6PLLcEsLEb//gMd3rV0LNtvgB0UidL75RjzCk9EmFPigBIaK+IuIiMQoUSbxc6izScbQLBQrcuiz36wxNHNOZChU8F9kdDHdbmxpqQCE6oaQtI5YQ6tJdQizqGUMaK2M/g3mSgFPWqKjERERGTWUKJP4SZsKHGQRWHcaOMdOC3p7dnZCzhUZrVTwX2R0OZjul54j50HoAOUSbDY8ixbHIzQZTSwrmigDzSYTERH5ECXKJH5S8mHaR8E4wDKOXoYNjrpwTHVYcs2cgXNaycHHbBikffqsYYlJZDRQwX+R0cE+KRsMCLe1E+kaPFGddNJJ0Q9xBntNC4dJP+fsOEcpCdfZAMFuMB3Rv99GSCAU4R8b9/DAa7sAeGJ9Fd2BoXcRFxERGQlKlEl8Hf1VsA7iD56jVg5fLMPAMAwyvvSlg29AYLOR+pnPDE9QIqOICv6LJJbpdGJPTwcOvPzSsNvJv/3n0TplH06Wvf84e9UqXNOmDUuskkC9RfxTJ4M5xA84D1N7T5DP/s/rXPXXjWyoaAbg5n9u5eO//Dd1bT0jEoOIiMhQKFEm8TXjZDj2qiEcaMB/3Q0ZxcMeUrylfupTOIuKDlwAed9zzjoLW1rasMUkMtqo4L9I4hxM90v3nLlM+ta3cM2e3We7s6SY/J//nKxLLxmWGCWBAl3Q+f73xgguu/z5v7azZW+0fuW+H5lUNnfzg39sHrE4REREDsSe6ABkHDr5JvBmwcu3vd/R0gAsonnZCPiy4RO3w5z/Smych8j0epn6+/vYdf6Xop/WhwefQec9+miSPnISPZvfxT13Doap/LRMHBnuDNJz0qntqqWstSxW8L+qvYppadNIdaUmOkSRcceelQWmQaSzk3BHB7akgWuB+t/bgXPqVPJuvgkrHKF7/Toc+ZNJPu1UjDFUGkEOQm9tMm8WOH0jcstAKMLf1lYS7mdCfjhi8dyWWho6/GQluUYkHhERkcHoHbvEn2HAsV+Hb70HH/k+FCyDvAUw83T4/EOwauuYTZL1ckyeTPGjfyf1U2eC3R59zjYbmGb0MWCfNImc732P/J/8BMNmI1RfT8/bb2MdqHCyyDijgv8iI8twOLBnZgKDF/UP1dcTbm4B08A1fTr2rCycRcU4Jk9Wkmy8ikQSUsS/vSdIT3DgpfeWBXVt/hGLR0REZDCaUSbDx5UER5wJuUdGH2fPgoySxMYUR/bMTPJvu41J11xD6xP/wL9jB5a/BzM5haQTTyDpxBMx3l+eaTjs9LzzDqGmZro3bcIzfz6Gw5HgZyAysnoL/ucn5VPeWk5NZw313fU09jSS78unMKUQh00/FyLxYJ+UQ6i+gVBdXb81xqxIBP/OnQA4p07F9HhGOkRJhPZqCAfB7oakSSN221SPg2SXnXZ//x8W2k2DyWn6HhQRkdFBiTIZXhNgpog9PZ3MlV8e/JiMDDwLF9L99tuEW9voWr8Bz8IFmC4tMZCJp7fg/5TkKZS2lNLU00RVRxU1XTUUJhcyOXkypqEJzyKHw56ViWG3EenuIdzaii217zLnYGUlke4eDJcT59SRm1kkCdZbxD9t6oh2HbfbTL60opB7Xikl8qHllzbD4L8W5pPq1QclIiIyOuidiAyvYHeiIxg1bKmpeBYtxnA5iXR20r1+PZFujY9MXCr4LzJ8DJstWqsMCH5o+WXE7yewaxcArmnTMOz63HRC6GmFnhYwTEidMuK3v+rkGZw4KxuIJsd6zZ+Syg2fmjvi8YiIiAxEiTIZXkG1+96XLcmHd/FiTI+bSHcPXevXE+7oTHRYIgmV4c5gSc4SZmfMxmlzxgr+r69dT6u/NdHhiYxZvd0vQ3X1WNYH03gCZWVY4Qi21JTYMTIB9M4mS8oB+8jPaHfZbdx/4VL+eunRnDQzmsT91bkLeeyKY0hxazaZiIiMHkqUyfAJByESTHQUo47p8eBZvBjT58PyB+jesJ5wq5IBMrGp4L9I/NnS0zEcDqxAgHBzMwDhtjaC1TUAuKZPV9H+iSIchLbq6H+PYBH/DzMMg6NLMvn04uiMtmOnZ2Oa+h4UEZHRRXPtZfho2eWATJcL7+JFsZpl3Rs34j7ySOwZGYkOTSShVPBfJH4M0yTc1Ejz3x6hZ8cOsNnwzJ1L0gkn4Fu+bL+6ZTKOtVaBFQZXMnj1t4aIiMhgNKNMho8SZYMyHA48CxZgz0jHCkfofvttgnWqyyQCHxT8X5K7hAx3BhErQlVHFatrVlPZVknEiiQ6RJFRr+Xx/8fe73yX7o0bsTo6sFpb6XrzTep+8hP8772X6PBkJO1bxF9EREQGpUSZDB8tlTogw27HPX8+9uxsiFj0vPsuwb17Ex2WyKihgv8ihyZYU0P1D34QfRDZJ7EciYBlUf39HxBqakpMcDKyOhujf5OZdkjOT3Q0IiIio54SZTJ8QirkPxSGaeKeNxdHfh5Y0LNtO4HKykSHJTKqqOC/yMFp+fujg+63QiFa/98TIxOMJFbL7ujXlHywqeqKiIjIgejVUoaPZpQNmWEYuGfPxrDbCVRU4n9vJ1YwiKukJNGhiYwavQX/sz3ZVLZXUtFeESv4n+3Jpji1GK/Dm+gwRUYFf2kp7NPpcj+miX/nzpELSEaOZcHO52Hdg9C8O1qXbMapUHRcoiMTEREZE5Qok+ETfH9GmelQ98shck2fjmG34y8rJ7BrN1YwhGvmDHUlE9mHCv6LHJjp84JpQjg8yDG+EYxIRsxz18HrvwLDFi3gb5hQ8TrYXbDiq4mOTkREZNTT0ksZPr3F/B2exMYxxjiLinDNnAlAcM8eerZswYqocLnIh6ngv8jAUk7/+KBJMsJhUs74+MgFJCOj4s1okgyiSTKA3t+F//oeNO9KSFgiIiJjiRJlMjzCoQ9mkTnciY1lDHJOmYx77hwwIFRbR8/mzViDveERmcBU8F9kf75jj8Fz1FHRWWUfZpr4jj8Oz6JFIx+YDK+Nf44W7e+PYcLbj4xsPCIiImOQEmUyPHrrk5mOgf9gk0E5cnLwHHkkmAahhka6N23CCmoJq8hAVPBf5AOGaVLw23tIPu1U2Hf5vmmScuYnmXLXXVrWPx51NkAk1P8+w4TO+pGNR0REZAxSBkOGR2/HSy27PCz2rCy8CxfS/fbbhFtaCW3ahBEa4A9gEVHBf5F92JKSmPKLXxD89l661q0Hw8C7dCmOnEmJDk2GQygASdlEPwfvZ+l5JAiT5ox0VCIiImOOEmUyPHpnlClRdthsaWl4Fi2ie+Mmwh0duHftItLTAw4VKxcZyFAK/otMFI78fFLz8xMdhgyXSARadkNjKUxeAhv+FN22L8MG7hQ48nOJiVFERGQM0dJLGR5BzSiLJ1tyMt6jFmO43JiBAD0bNhLp7Ex0WCKj3qAF/9tV8F9ExriOetj9KtRvi84YyyiGz94HzqTo/t7yF94MOP9xcCUnLlYREZExQjPKZHjsO6MsHEhsLOOE6fXiXrSQyNq3sPw9dK3fgGfhAmzJ+qNX5EB6C/439TRR2lJKZ7CTstYyykPl1HfVk5+q2TYiMob4O6LJsd6aYzYHZM2C1CnRmnQzToUtT0LbHsgogdmfALsrsTGLiIiMEUqUyfAIdke/2j2AimjHi+l2011YiJmUhNXdQ/eGDbiPPBJ7enqiQxMZEzLcGaTnpFPbVct7je8RJMiWpi3U9NQwLW0aqa7URIcoIjKwcBAad0LzbsCKFuhPK4TMadFkWS+nDxaem7AwRURExjIlymR4hN5PlGnpZfzZ7bgXLiS0dRvhlha6N23CM28e9qysREcmMib0FvxPs6exw9yBzbCp4L+IjG6WBa2V0LAjmiwD8E2CSbOjSTERERGJG9Uok/gLhz74I06JsmFh2O14FszHnpUJEYvud94hWFub6LBExhSbaSPLlsWy3GXk+fIwMKjvrmdt7Vp2Nu8k2Pt7TEQkkbqaYPdrUPtu9O8rpw+mLIUpRylJJiIiMgw0o0zir3c2menouwxA4sqw2XDPm0fP1q2EauvoeXcLVjCEc8rkRIcmMqb0FvyfkjyF0pZSmnqaqOqooqarhsLkQiYnT8Y09LmSiIywQFe0DlnH+x+EmQ7Img6pU8HU7yQREZHhokSZxF9vfTKHO7FxTACGaeKeMwe/3UFwzx78O3ZAKIizqCjRoYmMOf0V/C9tLWVP5x5KUkuY5J2U6BBFZCIIh6CpDJrLwYoABqQVQOYMsDsTHZ2IiMi4p0SZxF9Q9clGkmEYuGfNxHA4COzahb+sHCsUwjV9eqJDExmT9i34X9ZaRk+ohy2NW6hqr1LBfxEZPpYFbXuhYTuE/NFt3kyYdAS4xkeH69d2NvDAa7tYXVrPOcVwxZ/W8qVjpnHKnBwMw0h0eCIiIoASZTIcYokyFcMeSa6SYgyHHf97OwlUVEaTZbNm6Q9PkUPQW/A/25NNVUcVu9t2q+C/iAyf7mao2wo973cKd3ghezYk5yQ2rjj66TPb+M3LpdhMA4MIAGt3NfOf0nV87qgp/PSz8zFN/c0iIiKJp0SZxF9vjTK7ll6ONGdBAYbNRs/27QT3VmMFQ7jnzsFQLRORQ2IzbRSmFJLny6O8tZyazhrqu+tp7Gkk35dPYUohDtViFJFDFeyJ1iFrr44+Nu2QUQLpxeOqDtn/vb2X37xcCkA4YmF//6mFrejXR9dVcUReCl85rjhBEYqIiHxg/LwCy+ihGWUJ5cjPxz13LpgGofp6et5+GysUSnRYImNab8H/JblLyHBnELEiVHVUsbpmNZVtlUSsSKJDFJGxJBKGhp1Q/u8PkmSpU6D4BMicNq6SZAC/faWMA00Wu/c/ZYQj1sgEJCIiMojx9Soso0OwK/pVNcoSxjFpEp758zFsJqGmZro3bcIKBhMdlsiY11vwf372fHwOH6H/z959x0d+V/f+f33bdGnUu1Ztm7evbLCNnXDpSSiBNFoCGC7JJfhCfoaQQC6h5AYSkgDJDUluIMBNaM4lgZALIdcxFwzYuG31rrepd43aSJr6bb8/vhqVXW3RrqQZac7Tj3loNTvl6OvZ0cx7zud8HIuueBdPjD7BeHI83+UJIbaC2REvIJu8CK4NwXJoeR7UHQTdn+/q1t10IsvpoTjXy8BG42kujc9vTlFCCCHENUhQJtaXY4O9EMhIUJZXekUFwSNHUAwdOz5L8thxnEwm32UJsS1UBCq4o/YO9lbsxaf5Fgf+Hxs7RjwTz3d5QohClI5D/09g5ARYaW9ERf0R2HEXBLbvJiEZ68Y7bjOWvYGVCCGEEDdGgjKxvnLLLlUDZG5P3mnRKMGjnSh+H04iQerYMZxUKt9lCbEt5Ab+31l3J23RNlRFXRz4f2biDMlcd60Qoqipjgljz0Dfo97QfkWDyl3eMsvS+nyXt+EqIz5K/Ncfi6yrCs3lMrZDCCFE/klQJtbX4nwyGeRfKLRImFBnJ2owgJNKkzx2DHs+ke+yhNg2cgP/76q/i/pwPQoKsVSMp8ae4tL0JUxblj0LUZQcB6Z7qJo7ixIf9M4rqYe2n4KqnaBq+a1vkxiayuvv3IF2jV24NVXh5w7WUx72bWJlQgghxOokKBPrS+aTFSQ1GCTY2YkaDuNmsqSOH8OOy/IwIdaTDPwXQiyaH4feH6LEzqPgeEsrm++EhiNF+Rrpvzy/g/qyANoqE/01VaEkoPPbL9uTh8qEEEKIK0lQJtaXlfa+6sX3IrDQqX4/oc6jaNFSXNMideIE1tRUvssSYtuRgf9CFLHMHAw8CUNPex8e6n7iwR24zXdBqCLf1eVNRdjHP7/jefynPdVcHpXd0VLON3/zHporZNmlEEKIwnD9gQFCrIV0lBU0xTAIHj5M+plnvN0wT50isG8fRk1NvksTYtupCFRQXlvOWHKM7nj34sD/wblBOso6iPq37vBua3qa+D//MzNf/yesWAw0jeCBA5S/8Q1Env98FK04lpQJscg2YeIizPQDLigqlLfilu4gfe7/wjWWHRaLmtIAf/fm5zAwleTvH+2GWDf/8NbncudOeQ0ihBCisEhQJtaXudBRZsingoVK0XUChw6RPnMWKxYjfeYMWBZGQ0O+SxNi28kN/K8OVjM4P0jfbN/iwP/qYDVt0TZCW+z5Mv6v/4eRD3wA17LAdRfPT/zkJyR+/GN87e3s+OzfYjQ25rFKITaJ68JMH0xcAmdhHmGkBqpvA18ITJlReLnmihC3t5TzdAxaKrfW858QQojikPell5/5zGdobW0lEAhw55138sQTT1zz8jMzM7zzne+kvr4ev9/P7t27+c53vrNJ1YrrWuwok2H+hUxRVQIH9mM01IML6XPnyQ4M5LssIbat7TLwP/7tbzP827+Na5orQjIAbBuAbF8fvW94o9dpJsR2lpiA3h/B+LNeSOYvgabnQuPtXkgmhBBCiC0pr0HZgw8+yAMPPMCHPvQhjh07xuHDh3nZy17G+PjqM1yy2SwveclL6O3t5etf/zrnz5/ns5/9LI3yqXVhcGyws96ft1iHRDFSFIXA3r34djQDkLl4iUx3d56rEmJ728oD/51EgpH/9sHrLyGzbayJCcY/9elNqUuITZdNeDPIBp+E7DxoBtTuh5Z7IFyZ7+qEEEIIcYvyGpR98pOf5O1vfzv33Xcf+/bt42/+5m8IhUJ8/vOfX/Xyn//855mamuKb3/wm99xzD62trTz/+c/n8OHDm1y5WJWZ8r6quveiUWwJ/p078be3AZDt7SN9/gLu5Z0iQoh1tRUH/sf/9f/gplJXdpKtxraZ/dd/ld11xfZiWzB+zusimx8HFChvhbbnQ9kOmUMmhBBCbBN5C8qy2SxPP/00L37xi5eKUVVe/OIX89hjj616nW9961vcfffdvPOd76S2tpYDBw7wsY99DHthuYfIs9yOlzLIf8vxtbbi370bAHNoiPTZs7hO4Xa2CLFdVAQquKP2DvZW7MWn+RYH/h8bO0Y8U1ghU/xb31pTEOCaJnMPf28DKxJik7guzAxAzw9gugdcB8LV0Hov1NwmHw4KIYQQ20zehvlPTExg2za1tbUrzq+treXcuXOrXqe7u5vvfe97vPGNb+Q73/kOly5d4jd/8zcxTZMPfehDq14nk8mQyWQWv5+dnQXANE3MAhiwmquhEGq5ZalZFMsCv+HNr8mxLBTL8gY/3+DPua2OyzrayOOi1Nag45I5dw57eBgrk8G/b9+W2b1OHjOrk+NydYV0bCp9lZRVljE0P0T/XD9TySmmklOLA/+D+uZ9AHG145KJx7F9vpUXdl3IheqKshSkKQpoGumpyYI4vuulkB4zG8GyTGzbAtta08+4rY9Lahol9iykvdeP+MK41Xu9oAyu+7pmWx+bW2Bb3ofcprW2x9pGyPf9CyGEKDyKm6c1VsPDwzQ2NvLoo49y9913L57/vve9jx/84Ac8/vjjV1xn9+7dpNNpenp60BbevH/yk5/kT/7kTxgZGVn1fj784Q/zkY985Irzv/KVrxAKyRyt9RRJDxHOjJP0VTMXbFo8P5rsJWBOMxdoIOmvvcYtiEKgzc0RGBwC18EOhUg3N8MWCcuE2A4s12LCmWDGmQFAQaFMLaNSrURXCmizasch2N2Nms2uONvVdKyyKGZZGa7fn6fixM3Qp6bwj45ilZaSaWq6/hW2MdXJUpIeImDOAOAqGvP+OpK+KlDyvhfWlvfMlBeqt5S4lOS5IS+ZTPKGN7yBeDxOaWlpfosRQghREPL2iruqqgpN0xgbG1tx/tjYGHV1datep76+HsMwFkMygNtuu43R0VGy2Sy+yz/pBt7//vfzwAMPLH4/OztLc3MzL33pSwvil6Fpmjz00EO85CUvwTC2eOv+yEmUuRHc6j1Q3rZ0/ugplNnhK8+/hm11XNbRZh0XOx4nfeoU2DZqJELg0CGUVf59FRJ5zKxOjsvVFfqxSZgJuuPdTKWnANBVnR0lO2iMNKJu4Bv1qx2XsT/6I2a+8c3F3S1xHK+jLNdJlvvcbdnnb3Uf/CDh592NVl29ZbpTr6XQHzO3yhwaInvxIlp1NYH9+2/8etvpuDg2THejTPWA2wGKglvaBFW7QFv778FtdWzWkf7MCCdOnOD5z38+dWXhvNaSW20ihBBC5OQtKPP5fNx+++08/PDDvPrVrwbAcRwefvhh7r///lWvc8899/CVr3wFx3FQVe9NwoULF6ivr181JAPw+/34V/lE2zCMgnrBUmj13BTXBF2HYCks/1l0fem0xp9xWxyXDbDRx8WoqsJ4znNInTiJm0pjnj5N8MgR1EBgw+5zvchjZnVyXK6uUI9NmVFGZ6iTqfQUXTNdJMwE/Yl+xjPjtEfbqQnVbOj9X35cqn7ltcx9+Ss3fH29qQlfdRXWxUvYvb3otbUYDY1okfy+KV4PhfqYuVWubmBrOrqm39TPt+WPy+wwxM6BlQFNhWCVN4MscOsfrG75Y7PONN0Lzg395h5r6ynf9y+EEKLw5LV3/IEHHuCzn/0s/+t//S+effZZ3vGOd5BIJLjvvvsAeNOb3sT73//+xcu/4x3vYGpqine/+91cuHCBb3/723zsYx/jne98Z75+BLGctbDrpV74YYq4Pq2khNDtnagBP04yRerYMZxEIt9lCVF0CmXgf2DPbiIveiGoN/bSofY9D+Bva0MN+HFNC3NwiOQTT5B8+mnMkRFc2YhHFIrUDPQ9BiMnvZDMCELDUdhx57qEZEIIIYTYWvI67OS1r30tsViM3//932d0dJQjR47w3e9+d3HAf39//2LnGEBzczP//u//zv/3//1/HDp0iMbGRt797nfzO7/zO/n6EUSO43gvLgEMmf22XaihEMHOTlInTuIkkySPHSd45DBaSUm+SxOiqCiKQl24jupgNYPzg/TN9jGbneX4+PHFgf+hTXjubfzEJ+h705tJnz27YonlIlUFx6Hmd3+H0p/9WcDbVdeensYcGsaaiGHHZ7HjsygXLy50mTXIc4rIDzMNExdgdsj7XtGgssMbE3GDgbAQQgghtp+8TwW+//77r7rU8vvf//4V591999385Cc/2eCqxJrluskUDfTCnmUl1kYNBAh1HiV18iT23Dyp48cJHDyIXl6e79KEKDqaqtFS2kJ9uJ6eeA+jiVFiqRiT6Ukawg20lLZgaBu3jEgJBql697uY/T//h/lHfogzM7Pi70OdnVT+xq8T+amfWrqOoqBXVKBXVOBks1gjI5gjIzjJFObQMObQMFpJBKOhAb22FkXP+0sTsd05Dkz3wGQXuAudjaWNULUbDOmKF0IIIYqdvBoV68NcCMqMYH7rEBtC8fkIHj1K6tRp7JkZUidPEjxwAL2qKt+lCVGUfJqPPRV7aCppomumi6n0FIPzg4wmR2kpaaGxZPWB/zPpGX40/CNmM7OU+Eq4t/FeygM3Hnpn+/rAson+/M9T/9GPkjp2DHNsHMVnELjtNvzt7de8vurz4WtpwdixA3tmBnN4GCsWw56bxz5/AeXSJa/LrL4eLRpd83ER4rrmRr05ZLnXLYEybw5ZsCyfVQkhhBCigEhQJtaHBGXbnqLrBA8fIn3mDNbEJKnTpwns24exsFRaCLH5wkaYQ9WHVgz874p3MZQYWjHwfyo9xaee+hTf7vk2pmOioODioqs6P9v6szxwxwNUBa8dfDuplBeUAf6du1ADAcLPe95N1a0oCnp5OXp5OW42izk2hjk0jJNMYg6PYA6PoEbCGA0NGLW1KDJsW9yq9CyMPwspbxdZdD9U74XShvzWJYQQQoiCI0GZWB8SlBUFRdMIHDhA+tlnscbGSZ85i2ta+Joa812a2AhWFs78Mzz1eZjqBl8E9r8G7ngrlDXnuzqxTEWggvLacsaSY3THuxcH/g/ODRL1R/nN//hNxpJj2AvLzFy8+WKWY/Gdnu/wxOgTfOnnvkRduO6q95G5eBEcF628DKN2/XbcVHw+fM3N+JqbF7vMzPFxnPkEmQsXyVy6hFFT480yKytbt/sVRcLKenPI4oOAC4oKFe3eSdXyXZ0QQgghCpAEZWJ9mEnvqwRl256iqgT27SOjG5hDQ2QuXADLxNfamu/SCs50eppvXPoGPx76MQkzQdQf5UU7XsTL219O2Ajnu7xrS8/CP7wGhp7y3li6DiRi8OM/h5/8Nbzha9D+n/JdpVjmagP//+Anf8BoYhQHZ9Xr2a7NRGqC9/7gvXzp57606mWsWAxrYhJUhcDu3Rv2M2hlZWhlZfh37fK6zIaHceYTmKNjmKNjqKEQRuNCl5lP5mGKa3AcmOnz5pA5pndeSZ3XRSavVYQQQghxDRKUifVhpb2vurz4LAaKohDYsxvFMMj29pLp7sG1LPw7d+a7tILguA5/ffKv+dzpz2E79mL3joLCo8OP8idP/gnvveO9vHbva/Nc6TV8634YPu792V0WsLi29+/9K6+Ddx2H0vr81CeuavnA/+8PfJ9LM5euex3btTkZO8nZybPsKt214u9c2/a6yQBfczNqeONDXsUw8DU14Wtqwp6d9WaZjY3hJJNkLl4i09WFXlWN0eh1mSmKsuE1iS1kPgbjZ5c+xPOXenPIQhX5rUsIIYQQW4IEZWJ9SEdZUfK3t6EYOpmLl8j2D3hh2Z49Rf2m1XVd/ujxP+Kr57965d8tBGZpO81/f/y/k7bTvHn/mze7xOub7oOz34KFeq/kgp2Fp78IL3j/JhYmlnNdFxcXx3VwcVd+v/Dnp8eeRkW9ajfZcpqi8a2ub/Geo+9ZcX62rw8nnUEN+PPSOaqVlqKVluLu3Im10GVmz81jjY9jjY+jhoIY9fXo9fWo0mVW3DLz3qD+RMz7XvN5O1lGm6CIfy8JIYQQYm0kKBO3znHAynh/lqCs6Piam1F0nfS5c5jDI7imRWD/PhT1yh33isEPh364aki2mj996k+5u+Fudpdv3FK2m3Lh30Hh6jkZeJ1lZ/9lSwVluQDJwQEXMnYG0zVJWSlMzKXAKRc6LVw2Fzq57lVCqVW+x2Xxz85CR94NXXfZfV3tMrnbuhHnps4tBrQ3cnzGk+Mrz0smyfb3A+DbuRNFy99MJ0XXMRobMRobsefmMIdHsMZGcZIpMl3dZLq7vS6zhnq0ioqiDuyLjm3C5CUv5M/NIStrgcqdoMlL3UKSNm2+fWqErz/VxwEffOPEML/2vHYifvn/JIQQonDIbyVx66yFQf6K5u0iJYqOUV+PouukzpzBisVInzpF4MABFL34nmK+dPZLaIq2ODT9WjRF48FzD/LBuz+4CZXdADMFqWmY7sF1vV0RHby8zAVcBRyUxdjFSU3i9j2K44+AvwTHF8bV9JXB0PKQKRc6XR4CXSWYupHQadXvXa4IuFYLiyzLosvq4onRJ9C3yWNVVdTFrwoKfs2/uMPl9SiKgk9b2ZGVuXABHBe9ohyjZv0G+N8qraQEbU8J7s4OrPFxr8ssPuvNUovFUAN+9Pp6jIYGVL/8Xtq2XBfiA96wfnthDlmkxptD5ivwOZBFqDs2zxs/9zgj8TQ+1eXALvjUQxf4m0d6+fu3PpfDzWX5LlEIIYQAJCgT62Fxx8tAfusQeaVXVxM8fJj0qVNYU9OkTp4keOgQimHku7RNE0vGeGzksRu+vO3afPPSN3n7obejKdoNBT+5bqirditdFjqZpkmf1cfx8eOomrrs7x2cbAI3M4ebmcXNzuEsdIY65hQErxcuKFBaCZf/vJoPjBD4QqAvfNUK8zGgKAoKCrqqo6s6Kioo3iy5XNCUu4yqqIt/VhQFlYXvl//9ssvnzkNh8bK521cVdfE8YPX7yn2//L5y932V+1qtgyqWuvHHpOM63F57++L3ViyGNTUNqoJ/Awf43wpF0zDq6zHq67HnE5jDQ94ss3SGbE8v2d5e9MpKjPp6tKoq6TLbThKTEHsWMnPe976IN4csXJXfusSqTNvh1/7uCcbnFn7PLPu7ubTJmz7/OD/8nRdSGijM3xdCCCGKiwRl4tYtBmWh/NYh8k4vLyd45AipU6ew47Mkjx0neORw0XR0jCXH1nydrJPlydEnKfGVbEBFXtdUyk0xm5lBdzKQnYdMwvt6Rdeb4i2f3nE3dP2Hd7kVf+uieJdCxUW57ZVQ1opqplDMJKqVBUA1M6hWFoUZQEHVfShGCao/Ar4Iqr8UxfDfcOh0RTC0EBbBUsh0ze+XhVTLb8+yLOZPznNPwz0Y2zTQ/ZnWn+GPnvgjEmbiupcN6kFe3vZy7xvHIXupCxXw7diBGir853ctEkbbvRu3owNrYgJzaBh7ZgZrYhJrYhLF71sM1dSgjAnYsrJJLyCbX1gmrBpQtdNbailBaMF66OwYQzOpVf/OcWE2ZfHPTw/ylnvaNrkyIYQQ4koSlIlblwvKdOkoE6BFowSPdpI6eQInkSB17BjBI0eK4o2pptzc/KaoP0qJr+Sa4dCqQdAq3UqL17VNlPQctjlFOp3iwNwUhqahoKMaZSi+chTVQAlEUYMVKMEylGA5iqovhEtB+OdfRyUXjC2jqND2n+Ce3wV12c9sm153R3oG0rOQmYXssoDGBlLz3knzQSDq7UYXiHhffYUfxmw1AT3Ae+94Lx957CPXvewDtz9AyAhhmia+2ARuJIIaDuNradmEStePomkYtbUYtbU4iQTmyAjmyChuJku2t49sbx96RTl6QwN6VVXRzlPccmwLprpgundhJ14FynZ4c8h02cSh0D3WNYmuKljO1ZeBP9Y9KUGZEEKIgiBBmbh1uRllMshfLNAiYUKdnaROnMBJpUkeO0bw8BG0yPaeGdNc0oyhGpiOecPXqQpWcW/jvYtB2E3LJrz5YrnTQkBlWRZVDlTpYfRAGILlSyd/6dU7MA691uvU+O7vwvyYF465jheMHX4D/NyfrAzJwFtiGarwTjm25QVmmdml8Cwz7+2amYgt7U4H3v0FShfCs1IvSDNC0iVyi35p9y+RttJ84slPoCjKio0ANEXDcR0euP0BXrf3dYA3wN+YmoK2Vvy7duV1gP+tUsNh/Dt34mtvx4pNYI0MY01NL54Unw+jvs7rMtsCXXNFyXVhdghi573nDYBQFdTsBf/GdOKK9Xfdp/GFD3qEEEKIQiBBmbh1svRSrEINBgl2dpI6cdLrLDt+jOChQ2jRaL5L2zARX4RXtL+Cb3V964aG+auovG7P69YekrkupOMrg7HcG8jlfBHccAnx4ARu209DaI3H/sAvwG2vgq6HYaoH/BHY9VJvWPaN0vQrwzPH9jrPMrPez5Ge9ZaCOiYkJ71Tjqp7b4YXu89KvVlEEp6tya/u+1VesOMFfP3C1/m3nn9jLjtHia+El7a+lF/Z/Ss0lTQtXjZz4SK4DlplJXp1dR6rXj+KqmLU1mDU1uCkUl6X2fAIbjZLtq+fbF8/WnkZRkMDenW1dJkViuQUxM55zxPgvc6o3gsltfmtS6zZPTur+PvH+q76964L9+ys3MSKhBBCiKuToEzcOhnmL65C9fsJdR5dnFmWOnGCwMGD6BUV17/yFvXG297It7q+dd3Lqaj4dT+/uPsXr3+jtuUtZ1wMxmaunC+mqF6YtLxjTDPANEn7+m4+yNZ02P2ym7vu1agaBMu8U47jQHZuqessPeuFaY619HPnKNpCeJbrPot64ZmEG9fUGGnk3Z3v5t2d777qZcyxcZyZaVBUfDt3bmJ1m0cNBvG3t+NrbcWenMQcHsaanMKensGenkExdIy6OvT6hm3fBVuwzJTXQTY34n2v6lDZAWWt8u98i3rR3hpaK0MMTKewL1t+qSpQFvLxms6mq1xbCCGE2FwSlIlb4ziwsFOedJSJ1SiG4e2G+cwz3m6Yp04R2LcPo2YNXUlbyJ6KPfzhvX/IB374gcVdKC+nKiq6ovM/Xvg/qAquskObmV7ZLZaZg8tvRzUWArEy72ugbGu/gVQXgr7Asq431/U6zRbDs/hSeJae8U45iuqFZbklm/5SL0y7fHmouCrXsshcughAtqpy288VVFQVvboavboaJ53GHB7BHBn2ZpkNDJIdGEQri2LU16PX1GzpJahbhmN73atT3UsfBkSboGo36MWxKcx2pWsqf//WO/nVv3uc/qkk2rKu4Iqwj79/651E/PK2RAghRGGQ30ji1lhpwPXepMqLWHEViq4TOHSI9JmzWLEY6TNnwLIwGhryXdqGeHn7yykPlPPJpz7J+enz3pB/FxwcXFyO1hzlt+/4bfZX7V8Kg5YHY+YqO4MZwZXdYsWw/FBRvLDLXwI0eue5LpjJpSWbue4zx1yahRYfzN2At1w0t2TTv3DS5FffarK9vbiZLEoggFlZXEug1EAAf3sbvrZW7Kkpr8tsYgJ7Jo49E0e5dAm9phalZnssRb2cNT1N/J/+iemvfQ0rNoEaDFL6ildQ/obX429v35wiZoe9LjIr7X0fLIea21aG52JL21EZ4uH3PJ//e2aMf3yyF4jxkVfu4zV3tBAwJIgWQghROOTdgrg1pgzyFzdGUVUCB/aTOX8ec3iE9LnzuLaNr7k536VtiOc1PI/nvep5nI6d5rGRxxiYG8BxHF7a8mKeX3XYC8QGn/a+XjH8fyEgWh6MydJmj6KAL+ydSpcFrdnkyg0D0jNLu3Bm5rxh4Dm+sBeY6WEMa867nGFs+o9SSOz5BNmBAQB8O3fB1FSeK8oPRVHQKyvRKytxMhmskRHMkRGcVBpzaAi7v49gTw/myAh6QwOKvvVfRqVOnaL/P78dZ27OC6IBO5Nh+qtfZfrLX6buwx+m/LW/snEFpOMw/uzS8mo94M0hK63fuPsUeWNoKi8/VI+mODz9VIxXHmmUkEwIIUTB2fqv8ER+5Xa81CUoE9enKAqBvXtRdJ1s/wCZi5dwTXPzOhby4GDFXg4G6xiYfJauyWepmuiCVHrlhRRtaQllbhmldD2tjS/knUrqls4z0wvLNXMBWtxbKp5NQDaBYllUJC6hdD0MwdKVM8/8paD78vfzbLLMhQvggl5dhV5VXN1kV6P6/fhaWzFaWrCnpzGHR7DHRlFTKbLnz+P09qLX1GA0NKCVlua73Jtijo3R/9a34SSTiyHZIttb+jj6oQ+h11RT8oIXrO+dWxmYuLDUAapoUNEOFW2yZFoIIYQQeSXvxMStkY4ycRP8O3ei6DqZ7h6yvX24poV/9y6U7bCU0Ex5nRHJKe9rdh6AsDkP2XmSqt8bsh8sh2CF99VfurXnixUqI+Cdlu+QZ2WWus7mJ7HVhTDMTHqnudGly+qBhfAsuhSibcPOPnNsDHtmBkVT8e/cyfX3ay0uiqKgV1SgV1SgtbWS7etDCQZxs6Y312x4BDUSxmhoxKitQdlC3YnTX/qyF5I5ztUvpKpMfOYz6xeUOQ5ML8whcyzvvJJ6r4tsG/77EkIIIcTWI0GZuDVm0vsqQZlYI19rK+gGmQsXMIeGcC2TwG23oWylwMh1vWV9i/PFppY2t1jOCBGMVIOukPKX4ra+aHuEgluR7odINUSqcUt3MFEygtvxIrCTK7vPzKQ3K2k+DfPjK6+/PDgLlG7p5z/XNMlcvASAr6UFNRjENi9fCixyFJ8Ps7KS0J13oszPe7PMYjGc+QSZCxfIdl3yuszq69HKyvJd7jW5rsv0P/7jtUMyAMch/cwZMhcv4t+169budG4MYueWXjsEot4csmD5rd2uEEIIIcQ6kqBM3BpzYQnZFn6jKPLH19SIYuikz57FGhsnbdsE9u8v3N3lHNsLU3IdY+mZpY6IRYoXnix2jJWB7ifguqhDSRzXIWWlCMkusYVDMyBQBeFlO5Da1rKdNhfCs2zCC0KtcUiMr7z+5eGZL7z5P8dNyPb24mazqKEgxo4d+S5nS9HLy9HLy3FNE3N0FHN4BCeRwBwZxRwZRQ2HMRrqMWprUXyFt4zXzWRw4vEbvnx2aOjmg7LMnDeHLDnpfa/7oWqPN2dQPjQQQgghRIGRoEzcGukoE7fIqK1F0TRSzzyDNTFJ6uRJggcPFsbyJSu7cjfKdBy4bI6PqnszxUILyygD0VXn6yiKQlAPkjATJK2kBGWFTtO9/6ehiqXzHPuyDQPikJn3NgNITninHNXwNmQILJt55gsXVChgz8+THfTmQ/l3795a3ZwFRDEMfM3N+JqbseNxr8tsfBwnkSBz8RKZri706mqMhgb08sLpnFIMw3s8Xj6b7CrUwE0si7SyMHkRZgZY3CG7vM2bRSZzGIUQQghRoORVirh5rru0zEyG+YtboFdVETpyhNSpU9gzcVInThA4fBh1s7swsomlUCw5tRQEryjWv2w3ygovDLnB8CNkhEiYCVJmCuSfzNajakv/73McxwvNlgdomTlvJ9PUlHfKUbSVXWf+0jU9ftbb0gD/avSKiutfQVyXFo2iRaO4O3dijo1jDg/hzCewxsaxxsa9zr2GBvS6us1/fltF4NAh0qdPX3f5pRIMEjx06MZv2HEg3g8Tl5Z29Y3UenPIfPIhgRBCCCEKmwRl4uaZKRY/Idb9+a5GbHFaWRnBo0dJnTiJPTdP6tgxgkeO3FwXw43IBRzLO8bs7JWX80WWBWPlt/QmL6R7101aqwRwYmtS1YUdS8uWznMcbxOHFd1ns+DaS4+1HEX1wrLcbpuBUvCVbPjmDuboKPZM3Bvgv2vnht5XMVIMA19TI76mRuzZWcyREayxMZxkisylLjLd3eiVVRiNDWjl5Zs+s9AcGyfbdYnw3XeTPnny2hfWVMp+6ZdQQzf43JeYgPGz3gcP4D2+a/at7M4UQgghhChgEpSJm2ctzCfTAwW1nEhsXVpJCaHbO0mdOIGTTHlh2eHDqOF1mPdkW95MscVgbMYLLpZTVC+sWB6Maeu3BFSCsiKhqgtLLkshunCe63rBQW7JZi5Ac6yF7+MQH/Auq6heQHt599kqS3pvhmuaZC51Ad6mGhsWRgsAtNJStNJS3I4OrPFxzJER7PgsViyGFYuhBgMY9fXo9fWo/o390MmemyNz8SL2jDebLPScO8gO/Bxz3/7OVYrX8LW3Uf3ud133tjU7gzJ0DDILXZSaAVW7IdosrxGEEEIIsaVIUCZu3uJ8MllGIdaPGgoR7OwkdeIkTjJJ8thxgkcOo5WUrO2GzPTKbrHMHFfOFzMWArGyhfliZRvayZObS5ZcbUmn2N4UBfwR71Ta4J3nut7z6PKus3TcW6qWW865dAPejLPF8Gxh7tlNzHnK9PQsDPAPYTQ3r8/PJ65L0XWMhgaMhgbs3I6ZY2M4qTSZ7h4yPT3oVVXejpmVlTfUZeZkMqTPnsVNp9GrqvDt3Lnq9Zxslmx3N+bwiFeLpuJracFobiZ8991M7t7D5Of/Die+7DGn60Rf8QpqP/B+tEjk6kXYJsTOUzX/LCSCoBtQ3gKVO9f1gwYhhBBCiM0iQZm4eYs7Xko3glhfaiBAqPMoqZMLyzCPHydw8ODVB2G7LmTmCGYnYPQUmHMLS4MvYwQvW0YZ2dROh1xHmemYmLaJIW8ii5uyEH75wkD90vlmaik0y3Wg2VlvOWd2HhheuqwRWlqymdt58xqPK3tuDnNoCAD/Hhngny9aJIK2e7fXZRaLYQ4PY8/EsWITWLEJFL8Po74Bo6F+1Y4/a3qayc99jpl//N84c3OL5/t37aLiLW8h+guvQVEUXMfBHBz0dje1vA5avbYG/86dK7rXqn7j16m47y0kfvQjrPFx1HCY8L33XnvzAdeF+CBMnEfJLIxiCFdD/QEvEBZCCCGE2KIkKBM3TzrKxAZSfD5vZtmp09gzM95umAcOoFdVeTOgFpdRel+VbIrS1ADKbA3oOqB4s3GWB2N5DnU1VcOv+cnYGZJWkqgWvf6VRPExgt6ppHbpPDO9bObZwtJNK+09D5tJmBtZef3lXWeBUtD9uK67NMC/tqagdmAsVoqmYdTVYdTV4SQSmMPDmKNjuJks2d5esr296JUVGA0NXpeZqmIOD9P7xl/FGh8He+Xy8cylS4z83u+ReOIJat7zANnubpyk96GBVlqCf+dOtLKyVWtRfT5KXvjCGys8OeXNIcsshHS+MNOhDtzG26EQdiwWQgghhLgFEpSJm5fr2DFk+z6xMRRdJ3j4EOlTJ7FG+kj96LsEdlRiRFRwL9ulTdXI6hHcyg4oqfGWUd7EsrSNFjJCXlBmJon6JSgTN8gIeKdIzdJ5VnbZzLOF7jMztXSaH1u6rO7HipvYA5MowQj+ls7N/xnENanhMP5du/At7zKbnsGanMKanELx+dBraxh897tXDckAr8sLmP2XfwHXpfRnfgbF58Pf0Y5eV3frmwZkkxA7t/TYUg2o7MCNNJA9+91bu20hhBBCiAJReO8ixdZhLQRluiy9FOssm1ycLaakpgkEZkkzgTUXJ31mFLelHl9D3YpuMVcLMn3BhMpdBd3RENJDTDMtA/3FrdN9oFdBuGrpPNtc2XWWmYVsAjedJHP+Ilg2vrJa1KFHQfOt7Drzl97Srq5ifSiqilFbi1Fbi5NMYo6MYI6M4mazzP7bv2F299zQ7cw99BAV972FwK5dKPotvtyzLZjqhumehQ8pFChr9p5vdR+Y5q3dvhBCCCFEAZGgTNwc1102o0zeWIlb4Lrem/nlg/etzIqLKKpKYO9uMqPzmFNJMukI6B34GluXLrRF3qjlBvqnrFVmqAlxqzQDwpXeKce2yJw+hhueRzVcjOYG74MOOwuJmHfKUQ0UPURJaghmR6Ck0nuOl10L80INhfB3dOBra8OamGDy7z7nbTjiONe9rptKYfb0ELzttpsvwHVhdhgmzi89L4cqoXqvF64KIYQQQmxDEpSJm2OlARcUFfSN3c5ebDOOvThXjNS0N2vMsS67kOJ1uizrGFN0H4F2ULp7yPb2kunuwbUs/Dt35uGHuHm5gf4JM5HnSkSxsBNJzKkERGrxHz2CUl7u/TtcnHm28DU77+24mZwklB1HGT0JEzqo+squs0Dppm+EUewUVcWoqcGanLqhkAwAXSPb33/zd5qahvFz3nM0eGMWqm9bOTtPCCGEEGIbkqBM3Bxz2bJLebMkrsXKLARjUwvB2CzgrryMqq8cuh+IgqqtenP+9jYUQydz8RLZ/gEvLNuzZ8N/jPWS6yhLW2kc10FVZNdBsXEWB/gDRl3t0gB/VVv695bjOJCdw52fIunrg2AZWEkvyE5NeaccRfM2y1gMz6JeeCa7aG6oNe1S6nLV59FrMtNeB9nswu6qqg4V7VDeJv9/hRBCCFEUJCgTN0cG+YurySa8HdFyHWPmKrO4dD8EK5beqPtL1hS4+pqbUXSd9LlzmMMjuKaFtnvXOv4QG8ev+dEUDdu1SVvpxeBMiI1gDQ9jz86h6Bq+63VfqqoXeGkh5oLNuM13eTvIZue9gDu3YUB6Flzb6zTKdRuB12Hsi3jhWW72mb9UwpV1FDx0iGxf3+qD/C9n2wTWsuzSsWG6Fya7vP+/AKWNULU77zsGCyGEEEJsJgnKxM2RoEyA14GyOF9sIRyzV5kV5i9Z2TG2Do8bo74eRddJnTmDFYthZdI39uaxAAT1IPPmPEkrKUGZ2DBuNkumuxsAX3s7qs+39htRFO/fr78Eoo0LN+x6gfjlSzcd0/tzZhbig7kbAH/ksqWbV+8YFddW/obXE//mN2/oskZDA+F7nndjNzw3CuPPLoxVwNs1uOY2r6tQCCGEEKLIrDkoa21t5a1vfStvectb2LFjx0bUJLaC3CByeZNfXGzL6yDJdYyl40udBznKQlfKYsdYmTdgfAPo1dUEDx8mfeoU1vQ0gf4BXNMs6F0vwVt+OW/OkzSTIFmz2CCZ7m5c00KNhDEaG9fvhpVc+BWB0oal87PJpdAsHfd23rRNyMx5p9mhpcv6wkuhWSDqBXEb9DyxnQQOHiT8/OeT+OEPrzurrPqBB66/VDM96wVkuWW1ut8b1L/8/6sQQgghRJFZc1D2W7/1W3zxi1/kox/9KC94wQt429vexmte8xr8fhnoXlSWzygT25eZXrYb5RRk5rlyvpgBodxssTLvtIlLrfTycoJHjjB37BhaKknqxAn0229HLeDnpLARBiBprbIsVYh1YMfjmMMjAAR270bZjFmSvpB3KqlbOs9Mrew6y8S9uYXZhHeaG1m6rBFa2XXmLwX9JrrgtjFFUWj65J8x8JvvJPn441fugKlp4DjU/u7vEH3Fy69+Q1YGJi5CfGDhhlVvDllFu3T7CSGEEKLo3VRQ9lu/9VscO3aML37xi/zX//pf+c3f/E3e8IY38Na3vpXOzs6NqFMUmtzcKVl6uX24rjeLKDXtdYylZ5YC0eWM4EKn2ELHmC+c9w0dtGiU4NGjuE89jZtIkDp2jOCRI6jBwnx8BnWvruRq89uEuEUrBvjX16GVleWvGCPonZbvlGhlloVnM96frbT3e8VMessAc/TAwsyzsqXlm0W+07IaDrPj7z7H7Hf/nekvfYnUiRMAKD4fpa94BeVvfAPB/ftXv7LjwEyvN4cst9twST1U75Hf50IIIYQQC256RllnZyednZ382Z/9GX/1V3/F7/zO7/DXf/3XHDx4kHe9613cd999m/MJtth8rut1GoG8sN7KHMd7k5qaXtiVctqbMbSCssp8scLsIlTDYVKtLSiBAE4qTfLYMYKHj6BFwvku7Qoh3VuyLB1lYiOYQ8PYc/Moho6voyPf5VxJ90Ok2jvlWNmF4GzZhgFm0gvQ5tMwP77y+v7osu6z0qL7XaToOtFXvJzoK16Ok8ngptOokQiKdo1usPlxb5llLqD3l3pzyEIVm1O0EEIIIcQWcdNBmWmafOMb3+ALX/gCDz30EHfddRdve9vbGBwc5AMf+AD/8R//wVe+8pX1rFUUCiuDt/xOkaWXW4mVXRaM5eaLXTbjRtG8mWLBZUspta2z54fr8xE4ehTr7Fmc+QSp48cIHjqEFo3mu7QVcgP8Lccia2fxabK8TKwPJ5sl27MwwL/tJgf454PuA70KwlVL5+Xmmy0Pz7IJ73eQNQ6JZeGZZlwZnvkKLyTfCKrfD9daap6Z9wKy5IT3vebzdrKMNuW9G1gIIYQQohCt+R3wsWPH+MIXvsBXv/pVVFXlTW96E5/61KfYu3fv4mVe85rX8JznPGddCxUFZHHZZUBeZBeybHLZfLFpb1nl5TTfym4xf+mmzhfbCKrfT+joUVKnTmHHZ0mdOEHg4EH0isLpmlAVlYAeIG2lSZpJCcrEusl2deGaFlpJBKNxiw9k1wyv22l5x5NjrwzOMrNeEGSbXhCUC4PAm5+4PDjzlxbEUvFNY5veHLKZfsD15pCVt0JFx5b6AEQIIYQQYrOt+ZXSc57zHF7ykpfw13/917z61a/GWGV3uba2Nl73utetS4GiAJmy42XBcV3vDePyYMzKXHk5X3hlMLZNOy4UwyB45Ajp06expqZJnTpFYN8+jJqafJe2KKSHvKDMSlJGWb7LEduAPTODOeLN9/Jv1gD/zaZqq4RnzkJgtjw8m/OWkicnvVOOol0ZnvlLtld45rpeODZxcWk5faTG281ymz7nCyGEEEKspzUHZd3d3bS0tFzzMuFwmC984Qs3XZQocNbCfDJZdpk/jg2JZcFYemZpMHOOonpvApcHY0W0g5yiaQQOHSJ95ixWLEb6zBmwLIyGwuiyCekhppiSgf5iXbiuSzo3wL+hvuCWG28oVV1YMl62dJ7jeF20K7rP5sC1l543cxTVC8tyu20GSsFXsjW7axOTMH52qYPYF/HmkC1f0iqEEEIIIa5pzUHZ+Pg4o6Oj3HnnnSvOf/zxx9E0jTvuuGPdihMFanHppXSUbRor472xm4tRMX8epUuDy4c2q/rKUCwQ9bovipiiqgQO7Cdz/jzm8Ajpc+dxbRtfc3O+S1ucUyYD/cV6MAcHceYTKIaOv7093+Xkn6ou7JZZunRebmff9GXdZ47lBWrpOMQHvMsqqhcyXd59VqiyCYidW9r0QDWgaheU7dhe3XJCCCGEEJtgzUHZO9/5Tt73vvddEZQNDQ3xx3/8xzz++OPrVpwoUIs7XkpH2YbJzK9cRrkQTiqWhWEnvTd8euCy+WLbbPnQOlEUhcDevSi6TrZ/gMzFS7immfcwQXa+FOvFyWTI9vQA4G9vR9kqA/w3m7Kwi6+/BGj0znNd7/l1eXiWjntLFnPLOZduAEXzE032wnQvRCq98Cyf875sC6a6vHpcB1C8cKxqlzfjTQghhBBCrNmaX92dPXuWzs7OK84/evQoZ8+eXZeiRIGTjrL15TiQia8Mxmzzysv5S3AjJcSDE7htz4dQAXc3FCD/zp0ouk6mu4dsbx+uaeHfvStvc5xyHWVpK43jOqjKFlzmJQpC5tIlXMtGKy1BL5ClxVuGonhzu3xhoH7p/GxyZddZOg52FjLzBMxplNg5mF54CWWElpZs5nbe3OiQynUhPggTF7y6AEJVULN3IQgUQgghhBA3a81Bmd/vZ2xsjPbLujFGRkbQddlFadtz3aUZZUYwv7VsVbYJqZll88Xi3tyc5RQVAmXLOsbKvDdepkna1yvH/ib5WltBN8hcuIA5NIRrmQRuuw0lD7OIfJoPXdWxHIukmSTii2x6DWLrs6ansca85XbbdoB/PvhC3qmkbuk8M407P8n8M0PecHwr4S2LN5PeaW5k6bJGcNnMs6j35xudEek48Oy34MnPQuyCd/3Dr4M73uptYpCcgvFnl7rdjJA3hyxSOJuVCCGEEEJsZWtOtl760pfy/ve/n3/5l38hujAseGZmhg984AO85CUvWfcCRYGxMkvLO2SY/40x05CaWgrGMnNXXkYzLltGGd2ag6S3AF9TI4qhkz57FmtsnLRtE9i/H+XymW+bIKSHmM3OkrQkKBNr5zoOmQsXATAaG9BKpct0QxkBiNSQCNTjNnSCYXi/EzNzSzPOMrPeztC50/zY0vV1/0JoFl2aeXb5CAPHgX/5TTj5VW+HTteGxDj8vz+EJ/8OXvnnSx+sqDpUdkBZq/y+EEIIIYRYR2sOyv70T/+Un/7pn6alpYWjR48CcOLECWpra/mHf/iHdS9QFJjFZZcBmYe1Gtf13jQtX0aZ68BbzghdFoxJSLKZjNpaFE0j9cwzWBOTpE6eJHjwIIqxuTN9QoYXlKWs1Kber9gezMFBnEQCxTDyPnOvaOl+77R8V0nbXFiyGV+aeWYmvVBtfnxp4D6A5lvqOAuUwvnveiEZrOw0dh2YG4WHPwwv/CBEm6Bqt3ffQgghhBBiXa05KGtsbOTUqVN8+ctf5uTJkwSDQe677z5e//rXY2zym0yRB7nQR5elf4D36X96ZlkwNuMNgV5B8d4ALe5GWSYbIRQAvaqK0JEjpE6dwp6JkzpxgsDhw6ibOAg9N9A/YSY27T7F9uBkMmR7ewHwd7RvesgrrkEzIFzpnXJsa2nWWW72WTbhzRdLxLwTwI8/BSiAu8oNOzB2Bkrqoe7gJvwgQgghhBDF6aaGioXDYX791399vWsRW4G50PlSrDOyrOzCXLGZZfPFnJWXUbSV3WKBaH53RRNXpZWVETx6lNSJk9hz86SOHSN45AhqYHOCzNxA/6QpO1+KtclcXBjgHy1Fr6+//hVEfmm6N18sVLF0nmNftmHALMwMsHpItkx8ABqObGS1QgghhBBF7abfvZ89e5b+/n6y2eyK81/1qlfdclGigC0uvSySoCybXOgUW5gxll2l80fzLYVioQpvCY0sS90ytJISQrd3kjpxAieZ8sKyw4dRw+ENv+9cR5ksvRRrYU1PY42PgyID/Lc0ddmHKjlGCLKrzLFcTnacFkIIIYTYUGsOyrq7u3nNa17D6dOnURQF1/U++cy9ULdt+1pXF1uduY13vHRd71P95NRSx5iVufJyvvDCm5sK76tP3rRsdWooRLCzk9SJkzjJJMljxwkeOYxWUrKh9xvQAygo2K5Nxs7g12TekLg213HInL8AgNHYuOGPUbHJDv4iHP8SONbqfx+IQuu9m1uTEEIIIUSRWfM2Se9+97tpa2tjfHycUCjEmTNneOSRR7jjjjv4/ve/vwElioKS6yjbDjPKbAsSkzBxCQaehIsPQd+jEDvnDU22MqCo3kyx8jZo6ISOF0HbT3vzYaKNEpJtI2ogQKjzKFpJBNc0SR0/jjU9vbH3qagEFnaPleWX4kaYAwM4ySSKz4e/rS3f5Yj19rx3geb3lvCv5j99QAb4iy1taCbFn/3f8/zl9y4B8OmHztMdm89zVUIIIcRKa+4oe+yxx/je975HVVUVqqqiqir33nsvH//4x3nXu97F8ePHN6JOUQhcd2mY/1bsKLMyK3ejTM9yxSwY1YBg2cr5YupV3rCIbUfx+byZZadOY8/MeLthHjiAXlV1/SvfpLARJmWlSFpJyim//hVE0XLS6aUB/js7ZID/dlTZAW/5V/j622C6Z+l8I+iFZHf+Rv5qE+IWuK7Lnz98kT9/+CIKoCoud5fCl58Y4HOPDvBrd7Xw4VftR1NlKbkQQoj8W3NQZts2JQtLPaqqqhgeHmbPnj20tLRw/vz5dS9QFBArszC4XgF9C+zamJlfGYyt1rGjB1YO3veXyHyxIqfoOsHDh0ifOYM1MUnq9GkC+/Zh1NZuyP0FF7ozpaNMXE/m4iVc20Eri2LU1eW7HLFRGm+Hdx2H3h/B5EXvA5tdL/V+PwmxRf3V97v49H9cBLyPKHN5mO24gMKXftKHqsBHfv5A3moUQgghctYclB04cICTJ0/S1tbGnXfeySc+8Ql8Ph9/+7d/S3t7+0bUKDZJ1s7yUN9D/NOFf2JgfgDbsakKVvGK9lfw8zt/nqizsLuj7gd1zat2N5bjQCa+MhizzSsv5y9ZGYxtxc44seEUTSNw4ADpZ5/FGhsnfeYsrmnha2pc9/vKDfRPWhKUiauzpqawYrHFAf5im1MUaPsp7yTEFhdPmvz5Qkh2NS7w94/18bZ729lRKWMthBBC5Neag7L/9t/+G4mEt/PfRz/6UV7xilfwUz/1U1RWVvLggw+ue4Fic3yn+zv84eN/yGx2FlVRcVwvFIulYpybOsenj32ad7b/AvdVHkUthHDJNiE1s2wZ5cxCt9syuflii8FYGWiyVEncGEVVCezbR0Y3MIeGyFy4AJaJr7V1Xe8ntLCDnXSUiatxHcd7/AG+pia0SCTPFQkhxI375+ODmLZz3cupqsJXnujnd3927yZUJYQQQlzdmoOyl73sZYt/3rlzJ+fOnWNqaory8nLZon6L+tq5r/GHj//h4vfOZYGTi4vpmPzTs19CrzjDr931Oze2C0TsAjz5OTj1j2AmIFgJd7wFbn8LlKxx2ZCZWtktlpm78jKacdkyymjhdb6JLUVRFAJ7dqMYBtneXjLdPbiWhX/nznW7j1xHWcbOYDs2mszEE5cx+/txkikUnw+fDPAXQmwx50fn0FQFy3GveTnbcTk/OrtJVQkhhBBXt6agzDRNgsEgJ06c4MCBpRkCFRUV616Y2BxPjj7Jxx7/2A1dNui6PDn2BOG+7/JLDUeufeFj/wD/+i5AAdf2zpsfgR/8Mfz4L+AND159SYnrQmaOYCYGIyfBnFvaRGA5I3RZMCZdFmJj+NvbUAydzMVLZPsHvLBsz551+XDA0AwM1cB0TJJWkhKfzCESS5xUimxfHwD+XTtR9DV/viWEEHmlKAque2VIFtCvPE+VYf5CCCEKwJrabQzDYMeOHdi2va5FfOYzn6G1tZVAIMCdd97JE088cdXLfvGLX0RRlBWnQGALDJYvUJ87/TlU5cYeBv6FTwK/eukbmKvN/8rp+h586796SyHdyx4rrgNWCr7yKzDZ5Z3n2JCc8r4ffAouPYzS92NK04MocyMLIZniDTQub4WGo9DxQmh/PtQfgrJmCcnEhvM1NxO4bS8oYA6PkH7mDK5z/aUkN0KWX4qryVzKDfAv27ANJYQQYqPYjsuOiiD25ZuMAx2X7Z+kKtC5Q3Z/FkIIkX9rXpf2e7/3e3zgAx9gampqXQp48MEHeeCBB/jQhz7EsWPHOHz4MC972csYHx+/6nVKS0sZGRlZPPUtfNou1mZgboDHhh/DvjzMuorAwqeB4+YcD/c/fPUL/uAT19450nW88Ov7fwR9j8Gl/4CBx2HiAiRi4Jig6mT1EtzKndD0XNj5Ymh5HtTc5i3b1P1r+VGFWBdGfT3BAwdAVbBiMdKnTuFa1i3frgz0F6uxJiawYhMywF8IseXYjkv/ZJIfX5qgpSJMQF/5lkNRrnypqCgKv3JH8yZWKYQQQqxuzUHZX/7lX/LII4/Q0NDAnj176OzsXHFaq09+8pO8/e1v57777mPfvn38zd/8DaFQiM9//vNXvY6iKNTV1S2eauVT9pvyw8EfrunyuaDMVDW+P/j91S801QP9j105WP9yrgNnvwnJSe/Put8LwGpug5bn4Xa8iOnwTqjcCeFK0GS5kSgMenU1wcOHUTQVa2qa1MmTuOY1OixvgARl4nKubZO56O0S52tuRouE81yREEJc3/KA7MLYHFnLIRoyeN/P7Lnudd/3sj1Ul8gHoUIIIfJvzenDq1/96nW782w2y9NPP8373//+xfNUVeXFL34xjz322FWvNz8/T0tLC47j0NnZycc+9jH279+/6mUzmQyZTGbx+9lZb0ioaZqYt/jmdj3kashHLbPpWUJKCMtd2RHjLvtPWfjP57r48Wq08TGfml+95ql+UFcuhVWu0rHmOkBJM1S0evPGlsnncSlkclyublOPTSSCvv8A6dOnsKemMJ98ksChQ6j+m3uBb2BgWRZzqbl1r18eM1dXyMcm29uLOT+P4vOjNDZuao2FfFzyTY7N6uS4XF2xHBvHcRmKp+ibTJGxvNd9AUOjrTJEXWmA57ZEKQ9q/OG3n2UuYxFeeAfiV12Cusa7X7SLN97VnJfjtN3/3wghhFg7xV1tuuYmGR4eprGxkUcffZS777578fz3ve99/OAHP+Dxxx+/4jqPPfYYFy9e5NChQ8Tjcf70T/+URx55hDNnztDU1HTF5T/84Q/zkY985Irzv/KVrxAKha44X4Dt2kw4E0w704vnVTg+9qam0NQAsdID17j2lWrjxxf/PBNqJ6uFcVXpEBNbn5LJEOzrR7FMHJ+P9I4duD7fmm8n62bptrpRUNit75YdhIucks0S6uoG1yHd1IRdWprvkoQQYlWOC9MZiKUVrIXFBIYK1UGXMp83d6zQJZNJ3vCGNxCPxymV51shhBBswaDscqZpctttt/H617+eP/iDP7ji71frKGtubmZiYqIgfhmapslDDz3ES17yEgzD2NT7fnT4Ud7z/fdc9e+Xd5aV2g47zCy2onLRH+JN+9/Efzn8X668kpWFv+yE1MziWcs7ylxFWzhTg8aj8GvfXPW+83lcCpkcl6vL17FxUinSJ0/iptMoPj+Bw4dQw2tbJue6Lj8a/hGO6/DcuucS1IPrVp88Zq6uUI9N+pTXqaiVlxM4fHjT779Qj0shkGOzOjkuV7ddj43juAzH0/ROJhc7yPy6RltViPrSwHV3ryyk4zI7O0tVVZUEZUIIIRatua1HVdVrdjusZUfMqqoqNE1jbGxsxfljY2PU1dXd0G0YhsHRo0e5dOnSqn/v9/vxr7IcyjCMvP9iXi4f9dzbfC+lwVJiqdh1L6u7FqZiE1dd0qS5t/ledF2/8rFgGND5q/CjT64+p8xd1t7+3Ld5l7+GQvv/VCjkuFzdph8bw8B47nNJnTyJM5/APH2a4KFDaNHomm6mJFBCwkxgKdaG1C+PmasrpGNjxWIQn0UzDMK33Yaax7oK6bgUGjk2q5PjcnXb5dh4AVmK3okkadMGFMIBH62VYRrLgtcNyC5XCMcl3/cvhBCi8Kx5mP83vvEN/vmf/3nx9OCDD/K7v/u71NfX87d/+7drui2fz8ftt9/Oww8v7aDoOA4PP/zwig6za7Ftm9OnT1NfX7+m+xagqRqv3/t61Bt4GEQdhxLbxlZU9lfsJ56N8+Tok0ykJq688E+9BxqOel1jq1Lg0Otg/2tu7QcQokCofj+ho0fRoqW4pkXqxAmsNe4MnBvonzATG1Gi2AIuH+C/1s5EIYTYKI7jMjid5NGuSc6NzJE2bfyGyp66Eu7pqKK5IrTmkEwIIYQoVGvuKPv5n//5K877pV/6Jfbv38+DDz7I2972tjXd3gMPPMCb3/xm7rjjDp773Ofy6U9/mkQiwX333QfAm970JhobG/n4xz8OwEc/+lHuuusudu7cyczMDH/yJ39CX18f//k//+e1/igC+NV9v8rD/Q9zbuoc9lWG7gMEHJcgGhE9zHuf814ydoakleSZiWco85fRXtZOqW+hXd0Xgjf/Kzz0ITj+D2Cll91QGTzvfrj3PVfuCy7EFqYYBsEjR0ifPu3thnnqFIF9+zBqam7o+kEjCClImrLzZbHK9vXhpDOoAT++1tZ8lyOEEDiOy8hsmp5YYqGDDHy6SlvVzXWQCSGEEFvBuk1Uv+uuu/j1X//1NV/vta99LbFYjN///d9ndHSUI0eO8N3vfpfa2loA+vv7UdWljqfp6Wne/va3Mzo6Snl5ObfffjuPPvoo+/btW68fpagE9SD/8yX/k3c+/E5Oxk6iouJw5ZLJoAslvgh/+uLP0F53B6ZjMjA7wOD8IDOZGY6NHaMmVENbtM2br+QLw8v/FF70Qfjhn0E2CeFquPe3QJetv8X2pGgagUOHSJ85ixWLkT5zBiwLo6HhutfNdZSlrNRGlykKkJNMku3vB8C3cyeKdrWOXCGE2Hi5gKx3IkEquxSQtVaGaSwPoklAJoQQYhtbl6AslUrxF3/xFzQ2Nt7U9e+//37uv//+Vf/u+9///orvP/WpT/GpT33qpu5HrC7qj/L5l32eb3d/my8/+2XOT59f8fc1oRp+uf4I99bfTaR6PwCGatBe1k5DpIGeeA9jyTHGk+NMpCZoCDfQEm3BUA0IRGHHwjJazSchmdj2FFUlcGA/mfPnMYdHSJ87j2vb+Jqbr3m9kOEFZUlLOsqKUebCBXBc9MqKG+5CFEKI9ea6LiPxND0SkAkhhChiaw7KysvLVwxwd12Xubk5QqEQX/rSl9a1OLF5fJqP1+x6Da/Z9RouTF9geH4Y27GpDFZysHwPWvf3vQtethtfQA9wW+VtNJc00xXvYjo9zeD8IKPJUVpKWmgsaVz7IDwhtjhFUQjs3Yui62T7B8hcvIRrmvjb2696ndxOl1k7i+mYXtAsioI5Po41NQ2qgn/XrnyXI4QoQrmArHciQXIhIDN0ldbKEE3lIQnIhBBCFJU1B2Wf+tSnVgRlqqpSXV3NnXfeSXl5+boWJ/Jjd/ludpfvXjojNeN91f2grh57RXwRDlcfZjI1SXe8m4SZoCvexVBiiLZsnBqjFHmJJYqNf+dOFF0n091DtrcP17Tw79616s7Bhmrg03xk7SwpK4Xhk6CsGLi2TXZh12bfjhbUUCjPFQkhionruowuzCCTgEwIIYTwrDkoe8tb3rIBZYiClhvGb1z/DVxlsJKKQAVjyTG6492krTTPJoYY1KboiDRRtrGVClFwfK2tKIZB+vwFzKEhXMskcNttKKuEziE9RNbOkjSTS5tjiG0t29vrDfAPBvC17Mh3OUKIIuG6LmOzGbpj8ysCspaKEE3lQXRN1gMIIYQoXmsOyr7whS8QiUT45V/+5RXn/+///b9JJpO8+c1vXrfiRIHI7cKnB27o4oqiUBeuozpYzeD8IP1Dx5mzU5yY66Eydpr2snbCRngDCxaisBiNjaDrpM+exRobJ23bBPbvv2Jge1APMpOZkTllRcJJJMgODAAL3YcywF8IscEWA7KJeZIZLyDTNYWWyjDNEpAJIYQQAGsfH/Xxj3+cqqqqK86vqanhYx/72LoUJQqMubAL3w10lC2nqRotpS3cGd1Jg78cBYXJ9CRPjT7F+anzZO3sBhQrRGEyamsJHjoEqoI1MUnq5Elc01xxmdxA/5QpO18Wg3RugH9VJXp1db7LEUJsY67rMhpP81j3JM8MxUlmbHRNoaMmwr07q2irCktIJoQQQixYc0dZf38/bW1tV5zf0tJC/8LW9mKbMXNLL2+so+xyPlVnd6ieJly6g1VMpCYYSYwwlhyjuaSZ5pJmdHVdNmAVoqDplZWEjhwhdeoU9kyc1IkTBA4fRvX5AG/pJcjOl8XAHBvHnp6RAf5CiA3lui7jcxm6YwkSGQuQDjIhhBDietb827GmpoZTp05dcf7JkyeprKxcl6JEgcktvVxjR9nlQpqfA1UHOFpzlFJfKY7r0DfbxxMjTzA8P4zruutQrBCFTSsrI3j0KIphYM/Nkzp2DCfthdGLHWVWSv49bGOuZZG5dBEAX0sLajB4nWsIIcTauK7L+Gyan3RPcXowTiJjoWsK7dVh7pEOMiGEEOKa1tzG8/rXv553vetdlJSU8NM//dMA/OAHP+Dd7343r3vd69a9QFEAFof5r8+buag/SmdtJ+PJcXriPaSsFBemLzA0P0RbtI2q4JVLe4XYTrSSEkK3d5I6cQInmSJ17BjBw4cJhEKoiorjOqSs1GJwJraXbG8vbia7MMC/Jd/lCCG2Edd1ic1l6J5IMJ/2Osg0TWFHRYgdFSEMCceEEEKI61pzUPYHf/AH9Pb28qIXvQhd967uOA5vetObZEbZdmRlwfFeaKGvb9dDTaiGqmAVQ/ND9M32kTATPDPxDGX+MnaEZfc3sb2poRDBzk5SJ07iJJMkjx0neOQwQT1IwkyQtJISlG1D9vyyAf67d6+6+6kQQtyM8bk03TEJyIQQQohbteagzOfz8eCDD/Lf//t/58SJEwSDQQ4ePEiLfCq+PVkLQ8U1H2zAGzpVUWkuaaYuXMfA7ACD84PMZGaYSEwwbA2TslIYhrHu9ytEIVADAUKdR0mdPOktwzx+nFCTj4SxMNBfVuRtO5kLF8AFvboKXcYVCCHWwfhcmp5YgrllAVlzeYiWSgnIhBBCiJtx0xPUd+3axS4ZQLz93eSOl2tlqAbtZe00RBroifcwNDvErDvLU2NP0VLWwo7SHRiqBGZi+1F8PoJHj5I6dRp7Zgb/s32ozUGSYRnov92YY2PYMzMomioD/IUQtyw2l6E7Nn9FQLajIoRPl4BMCCGEuFlr/i36i7/4i/zxH//xFed/4hOf4Jd/+ZfXpShRQBaDss1pbQnoAW6rvI3Omk5CSgjHdRiYG+DxkccZmBvAcZ1NqUOIzaToOsHDh9CrKvGrPvTzfSRHBvNdllhHrmmSuXgJWBjgH7i5XYSFECI2l+GJnilODswwl7bQVIXWqhD3dFSxsyYiIZkQQghxi9b8m/SRRx7h537u5644/2d/9md55JFH1qUoUUA2OSjLKfGVsEPfwcGqg4SNMJZj0TXTxROjTzCWGJMdAcW2o2gagQMHCNU3getinj1HdnAo32WJdZLt7cXNZlFDQYwdMoNRCLF2E/NLAdlsylwKyHZWsbOmRAIyIYQQYp2seenl/Pw8Pp/vivMNw2B2dnZdihIFJE9BWU5FoIKaSA2jiVF6ZntIW2menXqWwflBOqIdlAXK8lKXEBtBUVVKDx7Bmb2ANTpJ8txZsEx8ra35Lk3cAnt+nuyg1yEoA/yFEGs1Oe/tYhlPmgBoqkJTeZAdlSH8upbn6oQQQojtZ82v1g8ePMiDDz54xflf+9rX2Ldv37oUJQpIbpj/Ou94uRaKolAfqefOujtpi7ahKRpz2TlOxE5wOnaapCmznMT2YWgG2s527KYaMnaGTHcPmUuX8l2WuAVLA/yr0Ssq8l2OEGKLmJzP8GTvFMf7Z4gnTVQVdlSGeN7OSnbVlkhIJoQQQmyQNXeUffCDH+QXfuEX6Orq4oUvfCEADz/8MF/5ylf4+te/vu4FijzLc0fZcpqq0VLaQn24nt7ZXkbmR5hMTzI1OkVduI62aBs+7cpuRyG2mpARYrq5DjNRAsOzZPsHcC0L/549KIqS7/LEGpgjI9gz8YUB/jvzXY4QYguYnM/QM5FgZqGDTFWhaWEXSwnHhBBCiI235qDsla98Jd/85jf52Mc+xte//nWCwSCHDx/me9/7HhXySfn2YpvgeDspFUJQluPTfOwu301TpInueDcTqQlGEiOMJcfYUbKD5pJmNFVeSIqtK6SHmGaabF05gWgD6XPnMIdHcE2LwP59snRvi3BNk8ylLgB8ra0ywF8IcU1TiSzdsfkVAVljmReQBQx5XSOEEEJsljUHZQAvf/nLefnLXw7A7OwsX/3qV3nve9/L008/jW3b61qgyKPckkbNBwUYPIWMEAeqDhDPxLk0c4m57By9s70Mzw/TFm2jLlwn3TdiSwoZIQCSVhKjvh1F10mdOYMVi5E+dYrAgQMo+k09fYtNlOnuwTVN1FAIo7k53+UIIQrUdCJL98Q80wkJyIQQQohCcNNtCY888ghvfvObaWho4M/+7M944QtfyE9+8pP1rE3km5n2vhZQN9lqov4ot9fezr7KfQT0AFkny/np8zw19hQTqYl8lyfEmoV0LyhLmAnAm20VPHwYRVOxpqZJnTyJa5r5LFFchz03hzns7Vrq3yMD/IUQV5pOZHm6b4qn+6aZTngzyJoqgjyvo4o9dSUSkgkhhBB5sqaWhNHRUb74xS/yd3/3d8zOzvIrv/IrZDIZvvnNb8og/+0o11FW4EFZTk2ohqpgFUPzQ/TN9pEwEzwz8Qxl/jI6yjoo8ZXku0QhbkiuoyxtpXFcB1VR0cvLCR45QurUKez4LMljxwkeOYzq9+e5WnE513WXBvjX1qCXl+e7JCFEAZlJZumKJZhOZAGvg6yhLEhrZVjCMSGEEKIA3PBH3K985SvZs2cPp06d4tOf/jTDw8P8j//xPzayNpFvVq6jLJTfOtZAVVSaS5q5s/5OmkuaURWVmcwMT489zdnJs6Ryu3gKUcD8mh9N0XBxSef+HQJaNErwaCeK34eTSJA6dgwnJY/pQmONjGDHZ1F0Df9OGeAvhPDMJLMc65/mqd5pphNZb4lluddBtreuVEIyIYQQokDccEfZv/3bv/Gud72Ld7zjHezatWsjaxKFItdRpm+9AdSGatBR1kFjpJGeeA9jyTHGk+NMpCZojDSyo3QHhmrku0whriqoB5k350laycUOMwAtEibU2UnqxAmcVJrksWMEDx9Bi4TzWK3IcU2TTFc3AL62Nun4E0KQtOD4wAyzaQcARYH6aJD2aukgE0IIIQrRDXeU/ehHP2Jubo7bb7+dO++8k7/8y79kYkLmP21r5kKnyhbqKLtcQA9wW+Vt3F57O2X+MhzXYWBugMdHHmdgbgDHdfJdohCrWhzonwusl1GDQYKdnaiRMG4mS+r4Mex4fLNLFKvIdHd7A/zDYYzGxnyXI4TIo3jK5MTADN2zClOJLIriLbF8XkcV+xqkg0wIIYQoVDcclN1111189rOfZWRkhN/4jd/ga1/7Gg0NDTiOw0MPPcTc3NxG1inyYXGY/9brKLtcia+EIzVHOFh1kLARxnIsuma6eGL0CcaT4/kuT4gr5Ab6J60rgzIA1e8ndPQoWrQU17RInTiBNTW1mSWKy9izs5hDwwD4d++SAf5CFKl4yuR4/zRP9kwxmciCAg3RpYAs6JOATAghhChka34VHw6Heetb38qPfvQjTp8+zXve8x7+6I/+iJqaGl71qldtRI0iH2wTnIVd9bZwR9nlKoOV3FF7B3vK9+DTfKStNGcnz/L02NPMpGfyXZ4Qi67VUZajGAbBI0fQK8pxbYfUqVOY4xL85sPiAH/AqKuVAf5CFKHZtNdB9mTPFJPz2YUllgF2lbrcVl8iAZkQQgixRdzSx9179uzhE5/4BIODg3z1q19dr5pEIcgtu9QMULfXCztFUaiP1HNn3Z20lraiKipz2TlOxE7wzMQz1wwmhNgs1+soy1E0jcChQ+jV1eC4pM+cwRwe3owSxTLW8DD27ByKruGTAf5CFJVcQPZE9xQTcxkUBeqiAe7uqGRffSn+7fUySgghhNj2bniY/7VomsarX/1qXv3qV6/HzYlCsA3mk12Ppmq0RltpiDTQE+9hNDHKRGqCydQkdeE62qJt+DRfvssURSrXUWY5Flk7e83HoqKqBA7sJ3P+PObwCOlz53FtG19z82aVW9TcbJZM98IA//Z2VJ88bwhRDObSJt2xBLG5DOAN6a8tDdBeHSbk815im6aZzxKFEEIIcRPWJSgT29AW3vFyrXyajz0Ve2guaaY73s1EaoKRxAjjyXGaS5ppLmlG22ZddaLwqYpKQA+QttIkzeR1Q1tFUQjs3Yui62T7B8hcvOQNlZewbMN5A/wt1IgM8BeiGFwekIHXQdZWFSbsl5fWQgghxFYnv83F6qzcIP/t21F2uZAR4kDVAWbSM3TFu5jLztE728twYpi20jbqwnUoipLvMkURCekhLyizkpRRdkPX8e/ciaLrZLp7yPb2oaRS4LobW2gRs+NxzOERAAK7d8tzhBDb2FzapGciwfisBGRCCCHEdia/1cXqch1l22DHy7UqC5TR6e8klorRHe8mbaU5P32ewflB2qPtVAYr812iKBIhPcQUU2uem+drbUUxDNLnL2AND+MfHsZ1nA2qsnitGOBfX4dWVpbfgoQQG2I+Y9Edm18RkNWWBmirDhORgEwIIYTYduS3u1idWXwdZcspikJNqIaqYBVD80P0zfaRMBOcnjhNmb+MjrIOSnwl+S5TbHOLO19eZ6D/aozGRtB1EqdPo8fjZM6cwTh8GEWTZcTrxRwaxp6bRzF0fB0d+S5HCLHO5jMWPbEEY7PpxfMkIBNCCCG2P/ktL1a3OMw/mN868kxVVJpLmqkL19E/28/Q/BAzmRmeHnua2lAtbdE2AkUwx03kx43ufHk1Rm0tfteFp57GnpwkdfIkwYMHUQxjPcssSk42S7ZnYYB/mwzwF2I7SWQseiYSjMaXArKaUj9tVWFKAvL8KYQQQmx3EpSJK9kWOAu7NOnFHZTlGKpBR1nH4g6Z48lxxpJjxFIxGiON7CjdgaHKi2exvnIdZWkrjeM6qIq65tvQKytJtewATcOeiZM6cYLA4cMS7NyibFcXrmmhlUQwGhvyXY4QYh2sFpBVl/hpr5aATAghhCgmEpSJK+XmIakGaPIQWS6oB9lXuY/mkma6ZrqYycwwMDfASGKEltIWGiONNxVmCLEan+ZDV3UsxyJpJon4Ijd1O04oRODIEawzZ7Hn5kkdO0bwyBHUgHRD3gx7ZgZzZBQAvwzwF2LLS2YtuheWWOb2PpGATAghhChe8o5eXGlxx0vpJruaEl8JR2qOcKDqAGEjjOVYdM108cToE4wnx/NdnthGcssvU1bqlm5HKykhdHsnasCPk0yROnYMJ5FYjxKLiuu6pHMD/Bvq0aLRPFckhLhZyazFM0NxHuuaZDTuhWTVJX6e217B4eYyCcmEEEKIIiXtQuJKizteSlB2PVXBKioDlYwmRumJ95C20pydPMvg3CAdZR1E/fImWtyakBFiNjt703PKllNDIYKdnaROnMRJJkkeO07wyGG0EtmY4kaZg4M48wkUQ8ff3p7vcoQQNyGZXVpimesgq1roICuVcEwIIYQoehKUiSuZ0lG2FoqiUB+ppzpUzeDcIP1z/cxmZzk+fpyqYBXt0fbFWVNCrFWuoyxhrk/3lxoIEOo8SurkSW8Z5vHjBA4eRC8vX5fb386cTIZsTw8A/vZ2FJnzJsSWksradE/MrwjIKiM+2qsjRIMSkAkhhBDCI0GZuJJ0lN0UXdVpjbYuDvwfTYwykZpgMjVJfaSe1tJWfJq8sRZrkwtZb3Xp5XKKz0fw6FFSp05jz8x4u2EeOIBeVbVu97EdZS5dwrVstNIS9AYZ4C/EVpHK2vRMJBiJp1YGZFURoiEJyIQQQgixkgRl4krmwhty2fHypvg0H3sq9iwO/J9MTzI8P8xYYowdpTtoijShqVq+yxRbRK6jLGne+tLL5RRdJ3j4EOkzZ7AmJkmdPk1g3z6M2tp1vZ/twpqexhrz5g/KAH8htoa0adMdWxmQVUR8dEhAJoQQQohrkKBMXCnXuSIdZbckZIQ4WH2QmfQMXfEu5rJz9MR7GJofoq20jbpwnbzZFtcV0AMoKNiuTcbO4Nf863bbiqYROHCAzLlzmKNjpM+cxTUtfE2N63Yf24HrOGQuXATAaGxEKy3Nc0VCiGtJm0sdZI7jnVcR8dFeFaYsJJ3dQgghhLg2CcrESrYFtun9WYKydVEWKKPT38l4cpyeWW/g//np8wzOD9IebacyWJnvEkUBUxWVgB4gZaVImsl1DcoAFFXFf9ttoOuYg0NkLlwAy8TX2rqu97OVmYODOIkEimHgb2/LdzlCiKtImza9kwmGZ5YCsvKwj45qCciEEEIIceMkKBMr5brJVAM0WZawXhRFoTZcS3WomqH5Ifpm+0iYCU5PnKbMX0ZHWQclPtl5UKwupIe8oMxKUs76D91XFIXA7t0oukG2t5dMdw+uZeHfuXPd72urcTIZsr29APg72lEMeV4UotBcLSBrrwpTHpaATAghhBBrI0GZWCk3n8wI5LeObUpVVJpLmqkL19E/28/Q/BAzmRmeHnua2lAtbdE2Aroce7FSyAgxmZ5c9zlll/O3t6EYOpmLl8j2D3hh2Z49Rb1EOHNxYYB/tBS9vj7f5QghlkmbNn2TSYZmkssCMoP2qogEZEIIIYS4aRKUiZVMmU+2GQzVoKOsY3GHzPHkOGPJMWKpGE2RJppLmzFU6VwRnsWB/tbGBmUAvuZmFF0nfe4c5vAIrmkR2L8PRVU3/L4LjTU9jTU+DgpFHxgKUUhWC8jKQgbt1REqJCATQgghxC2SoEystBiUhfJbR5EI6kH2Ve6jqaSJ7pluZjIz9M/1M5IYYUfpDhojjahK8QUUYqWQsTE7X16NUV+PouukzpzBisVInzpF4MABFL14fmW4jkPm/AVgYYB/JJLnioQQGcsLyAanJSATQgghxMYpnnc94sbkZpTJ8r9NVeor5UjNESZSE3TPdJO0knTNdDE8P0xbtI2aUE2+SxR5lOsoy9gZbMdGU7UNv0+9uprg4cOkT53CmpomdfIkwUOHimZGlzkwgJNMovh8+NtkgL8Q+ZQLyIamU9iOC0A0ZNBeFaYysr4bnAghhBBCSFAmVpKOsryqClZRGahkJDFCb7yXlJXi7ORZBucG6SjrIKTK/5diZGgGhmpgOiZJK7lpGz/o5eUEjxwhdeoUdnyW5LHjBI8cRvVv7zemTjq9NMB/Z0fRhINCFJqMZdM/mWRQAjIhhBBCbCIJysRKuaVdMqMsbxRFoSHSQE2ohoG5AQbmBpjNznJ8/DhlRhkZN5PvEkUehIwQ8UycpLl5QRmAFo0SPNpJ6uQJnESC1LFjBI8cQQ1u3+eIzMVLuLaDVhbFqKvLdzlCFJ2s5dA3mVgRkJUGDdqrw1RJQCaEEEKIDSbDj8QSxwbb9P4sQVne6apOW7SNO+vvpD5cj4LCRGqCHquHizMXydrZfJcoNtFmDvS/nBYJE+rsRA0GcFJpkseOYc8nNr2OzWBNTWHFYt4A/927812OEEUlazlcGp/jx5cm6JtMYjsupUGDIzvKeG5bhYRkQgghhNgU0lEmluSWXaoGaLLUqFD4NT97KvbQVNLEhQlvuPjw/DCTmUl2lO6gKdK0KTOrRH7lMygDUINBgp2dpE6exJlPkDp+jOChQ2jRaF7q2Qiu45C54P0b8zU1yQB/ITZJ1nLon0oyMJ3Etr0OspKATnt1hOoSCceEEEIIsbkkKBNLFueTySD/QhQ2whyoOsA57RwRX4S0k6Yn3sPQ/BDt0XZqQ7UoipLvMsUGCS50eW7WzperUf1+QkePLs4sS504QeDgQfSKirzVtJ7M/n6cZArF78MnA/yF2HCm7dA3eWVA1lYdpqZEXosIIYQQIj8kKBNLZD7ZlhBSQ3RWdzJtTtMd7yZjZzg3dY6BuQHao+1UBivzXaLYALmOspSVwnXdvIWiimEQPHKE9OnT3m6Yp04R2LcPo2Zr78zqpFJkFgf470TR5dejEBvFtL0Osv6ppYAsEtBpl4BMCCGEEAVA3gmIJVba+6pLUFboFEWhNlxLdaiaobkh+ub6SJgJTk+cpjxQTke0g4hPlo1tJ0E9iKqoOK5Dxs4Q0PP3ZlLRNAKHDpE+cxYrFiN95gxYFkZDQ95qulWZS5fAcdHKyjBqa/NdjhDb0lUDsqow1SV+6YoWQgghREGQoEwskY6yLUdVVJpLm6mL1NEX72M4Mcx0epqn0k9RG6qlLdqW10BFrB9FUQjqQRJmgqSVzPv/V0VVCRzYT+b8eczhEdLnzuPaNr7m5rzWdTOsiUms2IQM8Bdig5i2w8BCQGYtBGRhv05HtQRkQgghhCg8EpSJJeZCR5kRym8dYs0M1WBn+U4aSxrpifcwnhxnLDlGLBWjKdJEc2kzhiobNGx1IT1EwkyQMBNUBPI/F0xRFAJ796LoOtn+ATIXL+GaJv729nyXduMch+yli6iAr7kZLRLOd0VCbBvWsg6y5QGZt8RSAjIhhBBCFCYJysSSxY4y6UDaqoJ6kH2V+2gqaaJ7ppuZzAz9c/2MJEZoKW2hIdKAqqj5LlPcpKARhJQ3p6yQ5GZ6Zbp7yPb24ZoW/t27tsSbYGNyEjcSQQmF8LW25rscIbYFy3YYmE7RN5lYDMhCfo2O6ogEZEIIIYQoeBKUCY9jg531/iwdZVteqa+UIzVHmEhN0D3TTdJKcmnm0uIOmdWh6nyXKG5CbqB/Pne+vBpfayuKYZA+fwFzaAjXMgncdhuKWrjBrJNM4puYhJYW/Dt3yQB/IW6RZTsMxhNXBGTtVRFqSyUgE0IIIcTWIO8KhMdc6FBRddBkid52URWsojJQyUhihN54LykrxZnJM5TOldJR1kHUH813iWINQgshdtIqvKAMwGhsBF0nffYs1tg4adsmsH8/iqblu7RVZS9dAtdBKy/HqN3au3YKkU+W7RBLwWPdUzh44XjIp9FeLQGZEEIIIbYeCcqEJ7fjpQzy33YURaEh0kBNqIaBuQEG5gaYzc5yfPw41cFq2qJtiwGMKGzBhR1ps3YWy7HQ1cJ7Cjdqa1F0ndTp01gTk6ROniR48CCKUVgBvBWLYU9NgaLi27Ur3+UIsSXZjsvgdJKusVnGUgpNtkNp0KCtOkxdaUACMiGEEEJsSQWxJuYzn/kMra2tBAIB7rzzTp544okbut7XvvY1FEXh1a9+9cYWWAxyS7l0Ccq2K13VaYu2cWf9ndSH61FQiKViPDn6JBemL2DaZr5LFNdhqAY+1QcUblcZgF5ZSejIERRdw56JkzpxAiebzXdZi1zbJnPxIgDZygrUkATFQqyF7bj0TSb40aUJLo7Nk7UdfBrsqy/l7o5K6qNBCcmEEEIIsWXlPSh78MEHeeCBB/jQhz7EsWPHOHz4MC972csYHx+/5vV6e3t573vfy0/91E9tUqXbXG7ppXSUbXt+zc+eij3cUXcHlYFKXFyG54f5ychP6Jvtw3bsfJcormFx+WUBzilbTisrI3j0KIphYM/Nkzp2DCedzndZAGT7+nDSGRR/ALOqKt/lCLFl2I5L/2SSHy8EZKblEPRp7KsvZVepS31UusiEEEIIsfXlPSj75Cc/ydvf/nbuu+8+9u3bx9/8zd8QCoX4/Oc/f9Xr2LbNG9/4Rj7ykY/Q3t6+idVuY4tBmex4WSzCRpiD1Qc5XH2YiBHBdm164j08Pvo4o4lRXNfNd4liFbnll4XcUZajlZQQur0TNeDHSaa8sCyRyGtNTjJJtr8fAN+unVDAmw0IUSiWB2QXxubILgRktzWUcnd75UJAlu8qhRBCCCHWR14H3GSzWZ5++mne//73L56nqiovfvGLeeyxx656vY9+9KPU1NTwtre9jR/+8IfXvI9MJkMmk1n8fnZ2FgDTNDHN/C81y9WQ71qU9BxYFi4GrHMtimUt/EnDvcHbLpTjUmg24rhEtAiHKg4RS8Xome0hmUnyzPgz9Bq9tEfbKQ+Ur9t9baRiecz4FB+WZTGXmsMMXf9nzftxMQz0gwdJnzqFnUhgPvkkgUOH0EpK8lJO+uxZbNNEq6jAjXqbWWz3x8xa5f0xU8CK7dg4jstQPEXfZIqM5XUbBwyNtsoQdaUBVFXBtq2iOy5rIcdmdYV0XAqhBiGEEIVFcfPYNjI8PExjYyOPPvood9999+L573vf+/jBD37A448/fsV1fvSjH/G6172OEydOUFVVxVve8hZmZmb45je/uep9fPjDH+YjH/nIFed/5StfISRzaRZVzz6D6ppMhndj6eF1ve3a+HEAHEUnVnpwXW9brC/HdZh2ppl0JnFwAAgrYaq1agKKdBsWgnlnnkF7EB8+2o0t1FFrWQT7+1HTaVxVI93chBNe3+ea69FmZwkMDoKikuxox/X5NvX+hdgqHBemMxBLK1jerwIMFaqDLmU+UKV7TGwjyWSSN7zhDcTjcUpLS/NdjhBCiAJQeFumXcPc3By/9mu/xmc/+1mqbnCuzPvf/34eeOCBxe9nZ2dpbm7mpS99aUH8MjRNk4ceeoiXvOQlGPnaFc51UC56y4/cjheCtr5vHpULC0ubdD9u+wtu6DoFcVwK0GYdF9M26Z/rZzgxjON675LqwnW0lrbi1/wbdr+3olgeMykrxROjT6AqKvc23HvdeUCFdFxcyyL9zDM4MzOgqvj37Uevqtyc+7ZtUo8/gVtXh9HSgq+traCOTSGR43J12/3YOI7LcDxN72SSoGXTgNdB1loZon6hg2w12/243Ao5NqsrpOOSW20ihBBC5OQ1KKuqqkLTNMbGxlacPzY2Rl1d3RWX7+rqore3l1e+8pWL5zmO9yZe13XOnz9PR0fHiuv4/X78/ivf2BuGkfdfzMvltZ5sAnQdFA0CG9DhoS88zDQN1vgzFtr/p0Kx0cfFMAz2BvbSUt5Cd7ybWDLGRGaCqYkpmkqaaC5pxlAL8//Ldn/M6LqOz/DhuA6WYi0O97+egjguhoHR2Un6zBmsiUmsc8+i79uHUVu74Xed6e9HtW3USIRQRweKpi0rqwCOTQGS43J12+3YeAFZit6JJGnTBhTCAR+tlWEay4JXDcgut92Oy3qSY7O6Qjgu+b5/IYQQhSevU4x9Ph+33347Dz/88OJ5juPw8MMPr1iKmbN3715Onz7NiRMnFk+vetWreMELXsCJEydobm7ezPK3D9nxUlxFUA+yv3I/nbWdRP1RHNehf7afJ0aeYHBucLHbTGweRVG21ED/yymaRuDAAYy6WnAhfeYs2cGhDb1PJ5FYHODv37VrRUgmRDFzHJfB6SSPdk1ybmSOtGnjN1T21JVwT0cVzRWhGw7JhBBCCCG2i7wvvXzggQd485vfzB133MFzn/tcPv3pT5NIJLjvvvsAeNOb3kRjYyMf//jHCQQCHDhwYMX1y8rKAK44X6zBYlAmM9vE6kp9pRytOcpEaoLumW6SVpJLM5cYmh+iPdpOdag63yUWlZARImEmSJkp2IL5tqKq+G+7DXQdc3CIzIULYJn4Wls35P7SFy6AC3pVJfoNLtsXYjtzHJeR2TQ9scRCBxn4DXXNHWRCCCGEENtR3oOy1772tcRiMX7/93+f0dFRjhw5wne/+11qF5bi9Pf3o6p5bXzb/haDMhnWLq6tKlhFZaCSkcQIvfFeUlaKM5NniM5HaY+2E/VH811iUQjpXqi9FTvKchRFIbB7N4pukO3tJdPdg2tZ+HfuXNf7McfGsadnQFXw79q1rrctxFaTC8h6JxKksl5A5tMXArLyIJoEZEIIIYQQ+Q/KAO6//37uv//+Vf/u+9///jWv+8UvfnH9Cyo25sKbbVl6KW6Aoig0RBqoCdUwMDfAwNwA8Uyc4+PHqQ5W0xZtu+G5WeLmbIegLMff3oZi6GQuXiLbP+CFZXv2XHeTghvhWhaZSxcB8LW0oAblOU4UJ9d1GYmn6ZGATAghhBDiugoiKBN5ZqW9r7q8iRQ3Tld12qJtNEQa6I33MpoYJZaKMZGaoCHSQGtpK4YmA3I3Qi6ITJpbPygD8DU3o+g66XPnMIdHcE2LwP59KLfYTZzt7cXNZFFDQXwtLetUrRBbRy4g651IkFwWkLVUhmgqD0lAJoQQQgixCgnKhHSUiVvi1/zsqdhDU0kTXTNdTKWnGJofYjQxyo7SHTRFmtBUGZ6+nnLD/E3HxLTNbRFIGvX1KLpO6swZrFiM9KlTBA4cQNFv7teUPZ8gOzAALAzwlyX8ooi4rsvowgyyXEBm6CqtEpAJIYQQQlyXvHModo4DVsb7swRl4haEjTCHqg9xuPowESOC7dr0xHt4fPRxRhOjuK6b7xK3DV3V8Wt+YHssv8zRq6sJHj6MoqlYU9OkTp7ENc2buq1MboB/dRV6ZeU6VypEYfI6yFI81jXJmaFZklkbQ1fZWRPhno5KWirDEpIJIYQQQlyHBGXFzloY5K9ooPvzW4vYFsoD5dxeezt7K/bi1/xk7Sznps7x1NhTTKWn8l3etpFbfpnK/RveJvTycoJHj6IYOnZ8luSx4ziZzJpuwxwbw56ZQdFUGeAvioLruozG0zzWvRSQ6ZqyGJC1VoXRNXnJJ4QQQghxI2TpZbGTHS/FBlAUhbpwHTWhGgbnBumf6ydhJjgVO0V5oJyOaAcRXyTfZW5pIT3ENNPbZk7ZclppKcGjnaROnsBJJEgdO0bwyJEbGsbvmiaZi5eAhQH+AXluE9uX67qMzWbonpgnmfGWWOqaQktlmObyoIRjQgghhBA3QYKyYrcYlMkuhWL9qYrKjtId1Ifr6ZvtYzgxzHR6mqfST1EXrqMt2ra4hFCszeJA/2209HI5LRIm1NlJ6uRJnGSK5LFjBA8fQYuEFy+TPn+e9DNncLMZtLIywvfcgzU2hpv1BvgbO3bk8ScQYuO4rsv4XIbuWIJExgIkIBNCCCGEWC8SlBW7XFCmS9eF2DiGZrCzfCcNkQZ6ZnuIJWOMJkYZT47TVNLEjpId6Ko8Ha1FSPeCsoSZyHMlG0cNBgkePeqFZfMJUsePETx0iMRjP2Hy858nferUyisYBqHnPpfSl76Ukpe8WAb4i23HdV1icxm6LgvIdlSEaK4IYUhAJoQQQghxy+SdabHLzTeSQf5iE4SMEPsr9xOPxOmOdxPPxOmf7WdkfoTWaCv14XpURd7o3YhcR1naSuO4zrY9bqrfT+joUVKnTmHF4wz/3n9j/j/+A1YLwUyT5GOPkXr6afy7d6FXVGx+wUJsgFxA1j2RYD7tBWSaptAiAZkQQgghxLqTV1bFTpZeijyI+qMcrTnKgaoDBPUgpmNycfoiT44+SSwZy3d5W4Jf86MqKi4uaSud73I2lGIYBI8cIfHID72QDLwde1fjOLiZDAP/+e1k+/o2r0ghNsj4XJrHe6Y4NRhnPm2haQpt1WHu3VlFe3VEQjIhhBBCiHUmr66KnQzzF3lUFaziOXXPYVf5Lnyqj5SV4szkGY6PHyeeiee7vIKXW365XeeULeckEsz80z/d2IVdFyedZuJ//u3GFiXEBhqfS/OT7klODSwFZK1VXkDWIQGZEEIIIcSGkVdZxcxxwMp4f5aOMpEnqqLSGGnkufXPZUfpDlRFJZ6Jc3z8OGcmzmzLXR3Xy+JA/yI4RvFv/gtY1o1fwbaZ/dd/xY5L4Cq2lthchsdXCcju6ahiZ40EZEIIIYQQG01mlBUzKw24oKigy86DIr90Vac92k5jpJHeeC8jiRFiqRgTqQkaIg20lrZiaEa+yywoxdRRNvvd7675Oq5pMv/II0Rf+coNqEiI9RWby9AzkWA2ZQKgqQrNFUF2VITx6RKOCSGEEEJsFgnKipkpg/xF4fFrfvZU7KEx0kh3vJup9BRD80OMJkZpKW2hMdKIpmr5LrMgFFNHmTU5Ca67tiupCvb09MYUJMQ6mZjP0B2TgEwIIYQQolBIUFbMcjte6hKUicIT8UU4VH2I6fQ0l2YukTATdMe7GZofoi3aRm2oFkVR8l1mXhVTR5nqv4muV8dF8cv8RVGYJua9DrJ4cikgayoPsqMyhF+XDwOEEEIIIfJFgrJiJh1lYgsoD5RzR+0djCXH6In3kLEznJs6x8DcAB1lHVQEKvJdYt4EF0Juy7HI2ll8mi/PFW2cwMGDZLq6wLbXdr19t21QRULcnMn5DN0SkAkhhBBCFCwJyopZbrmWBGWiwCmKQl24jupgNUPzQ/TN9pEwE5yKnaI8UE5HtAO/Unxz9jRVI6AHSFtpkmZyWwdl5a9/PfEb3fUSQFXx795N4MCBjStKiDWYXOggm1kIyFQVmspDtEhAJoQQQghRUCQoK2Zm2vsqQZnYIjRVY0fpDurD9fTN9jGcGGY6Pc1T6aeo8ldhuma+S9x0IT3kBWVWkjLK8l3Ohgke2E/w8GFSzzxzY11ljkPlfW8p+uW5Iv+mElm6Y/NXBGQ7KkIEDAnIhBBCCCEKjUyJLWbSUSa2KEMz2Fm+kztq76A6VA3AaGKUbqubnngPlmPlucLNszinrAgG+jf++afRKypAu364UPbaX6H0Va/ahKqEWN10IsvTfVMc65tmJmmiqtBcEeJ5HVXsri2RkEwIIYQQokBJR1mxcl2wMt6fZZi/2KJCRoj9lfuJR+JcmLyAi0v/XD+xTIzWaCv14XpUZXt/HrC482URDPQ36upo/d//yOC73kX61GkvMFveXaYooGlU/frbqbr/fukmE3kxncjSPTHPdGKpg6yhLEhrZVjCMSGEEEKILUCCsmJlpgAXFBX04pvtJLaXqD/KkeojnNfOE9SDmI7JxemLDM4N0lHWQVWwKt8lbphi2vkSFsKyBx8kffo001/9KqmTp3DTabTyckp/7meJ/sIvoJeX57tMUYRmklm6YgmmE1lAAjIhhBBCiK1KgrJiZS3MJ9MDXheGENtAiVrCHbV3MJGdoDfeS8pK8czEM0T9Udqj7UT90XyXuO5yHWVpK43jOtu+gw68zR2Chw4RPHQo36UIwUwyS/dEgqn5pYCsPhqkrUoCMiGEEEKIrUiCsmK1OJ8slN86hFhnqqLSGGmkNlRL/1w/g3ODxDNxjo8fpzpUTVtp22K4tB34NB+6qmM5FikrRdgI57skIYpCPGnSNTG/GJApitdBJgGZEEIIIcTWJkFZsVrc8TKQ3zqE2CC6qtMebacx0khPvIfRxCixZIzJ1CT14XpaS1sxNCPfZa6LkB5iNjtL0kxKUCbEBktacHxghtm0A3gBWa6DLOiTgEwIIYQQYquToKxYSUeZKBJ+zc/eir00RZroincxnZ5maH6IseQYO0p20BhpRFO39pvbkLEQlBXJnDIh8iGeMrkwMkP3rEJ5Iouh6xKQCSGEEEJsQxKUFSsz5X01ZMdLURwivgiHqw8zlZ6ia6aLhJmgO97N0PwQbdE2akO1W3aXxMWB/qYEZUKst3jKpGciwcRcBsu2QIH6aIBddVFCPnkZJYQQQgix3cgrvGJlLQRluiy9FMWlIlBBeW05Y8kxuuPdZOwM56bOLe6QWR7Yejsm5mauSUeZEOtnNm3SHfMCMsgtsQywq9RlX30phiEvoYQQQgghtiN5lVeMXHfZjDJZeimKj6Io1IXrqA5WMzQ/RN9sH/PmPCdjJ6kIVNAebSfii+S7zBsmHWVCrJ/ZtElPLEFsWUBWWxqgvTqMobj0yipLIYQQQohtTYKyYmSlARcUFXR/vqsRIm80VWNH6Q7qw/X0zvYyPD/MVHqK6fQ0deE6WqOt+LXC/zcS0AMoKNiuTcbObImahSg0cwsdZJcHZG1VYcJ+7+WSaZr5LFEIIYQQQmwCCcqKkbls2eUWnckkxHoyNINd5bsWd8iMpWKMJEYYS47RVNLEjpId6GrhPl2qikpAD5CyUiTNpARlQqzBXNqbQTY+m1k8ry66MiATQgghhBDFQ14BFiMZ5C/EqkJGiP1V+4ln4nTHu4ln4vTP9jM6P0pLtIX6cD2qoua7zFWF9JAXlFlJytl6c9aE2GzzGYvu2PyKgKy2NEBbdZiIBGRCCCGEEEVLXgkWIwnKhLimqD/K0ZqjxJIxuuPdpKwUF6cvMjQ3RHtZO1XBqnyXeIWQEWIyPSlzyoS4jvmMRU8swdhsevE8CciEEEIIIUSOvCIsRrkdL2WQvxDXVB2qpjJYyUhihN54L0kryTMTzxD1R2mPthP1R/Nd4qLFgf6y86UQq0pkLHomEozGlwKymlI/7dURCciEEEIIIcQieWVYjJbPKBNCXJOqqDRGGqkN1dI/18/g3CDxTJzj48epDlXTVtpGqABC51wN0lEmxEpXC8jaqsKUBIw8ViaEEEIIIQqRBGXFKPdGeqOXXqZm4ORX4fg/QDYFwTJITMC+nwdDQjqxteiqTnu0fXHg/2hilFgyxmRqkoZwAy2lLRha/t505zrKMnYG27HRVC1vtQhRCJJZi+6FJZau651XXeKnvVoCMiGEEEIIcXUSlBUb1wVrYXDxRgZlx/4BvvMesLLAwjuUaQW+8evw3d+BX/5f0P78jbt/ITaIX/Ozt2IvTZEmuuJdTKenGZwfZDQ5yo6SHTSVNOVl4L+hGRiqgemYJK0kJb6STa9BiEJwtYCsrTpMqQRkQgghhBDiOiQoKzZWBlwHUDZu6eXxL8G37l/lLxbesaTj8KVfhDf/K7TcvTE1CLHBIr4Ih6sPM5Weomumi4SZoDvezdD8EG3RNmpDtSiKsqk1hYwQ8UycpClBmSg+yezSEstcQFa10EEmAZkQQgghhLhREpQVm8VllwHYiDfx6Vn49nuvfRnX8b5+67/C/U9uTB1CbJKKQAXlteWMJcfojneTsTOcmzrH4NwgHWUdlAfKN62WkL4QlMlAf1FEUlmb7on5KwKytqow0aAEZEIIIYQQYm0kKCs25gbveHnqQbDS17+c68DkReh7FFrv2ZhahNgkiqJQF66jOljN4Pwg/bP9zJvznIydpCJQQUdZB2EjvOF1yM6XopiksjY9EwlG4qnFgKwy4qO9OiIBmRBCCCGEuGkSlBWbXIi1UcsuL/7fG7+sqnuXl6BMbBOaqtFS2kJDuIHe2V6G54eZSk8xPTpNXbiO1mgrfs2/YfcfXJg7KDtfiu1stYCsIuKjoypCNCQBmRBCCCGEuDUSlBWbxaWXG9RRlo6zOIvsuhTIJjamDiHyyNAMdpXvWtwhM5aKMZIYYSw5RnNJM80lzejq+j/95jrKUlYK13U3fUaaEBspbS4FZM7CCn4JyIQQQgghxHqToKzYmAsdZRu142WkFhQNXPv6l3UdCFdtTB1CFICQEWJ/1X7imThdM13MZmfpm+1jZH6E1mgr9eH6dQ2zgnoQVVFxXIeMnSGwUZ2jQmyi1QKy8rCPjuowZSFffosTQgghhBDbjgRlxWaxo2yDgrIDvwjPfuvGLuvasP8XNqYOIQpI1B+ls7aTWDJGd7yblJXiwvQFBucGaS9rpyq4PoGxoigE9SAJM0HSSkpQJra0tGnTO5lgeEYCMiGEEEIIsXkkKCsmrrs0o2yjgrK9L/e6yhIT1+4qUzRvNln17o2pQ4gCVB2qpjJYyfD8MH2zfSStJM9MPEOZv4z2snZKfaW3fB8hPeQFZWaSikDFOlQtxOZKmzZ9k0mGZpLLAjKD9qoI5WEJyIQQQgghxMaSoKyYWBlvuSPKxg3z1wx47Zfgf70CbFYPyxQNQpXw6r/emBqEKGCqotJU0kRtuJaBuQEG5waZycxwbOwY1aFq2qPtBPWbD7KDRhBSsvOl2HpWC8jKQgbt1REqJCATQgghhBCbRIKyYrK47DIAGznku/m5cN934VvvgrHT3u6WKF5I59rQ9tPw838J0aaNq0GIAmeoBu3R9sUdMkcTo8SSMSZTkzSEG2gpbcHQ1j6gPDfQX3a+FFtFxvICssFpCciEEEIIIUT+SVBWTHLLLm+hW+WGNXbCO34EQ0/DhX+HzDyEKmD/a6CyY+PvX4gtIqAH2Fuxl6ZIE13xLqbT0wzODzKaHKWlpIXGkkZURb3h2wst7GgrHWWi0F0tIGurClMZ8ee3OCGEEEIIUbQkKCsmZsr7ulHzyVbTeLt3EkJcU8QX4XD1YabSU3TNdJEwE3TFuxicH6Q92k5NqOaGdsjMLdvM2lksx9rosoVYs4xl0z+ZZHA6he24AERDBu0SkAkhhBBCiAIgQVkxyUdQJoRYk4pABeW15Ywlx+iOd5OxMzw79SwDcwN0lHVQHii/5vUN1cCn+sg6WZJWkqAi/95FYchaDn2TiRUBWWnQoL06TJUEZEIIIYQQokBIUFZMJCgTYktQFIW6cB3VwWoG5wfpn+1n3pznZOwklYFK2svaCRvhVa+bslI8MfoEZybP8E8X/om6QB0HOIDt2BisfeaZELcqazn0TyUYmJKATAghhBBCFD4JyopJbrj3ZswoE0LcMk3VaCltoT5cT+9sLyPzI0ymJ5kanaIuXEdrtBW/5gUNlmPxVyf+ii8/+2WSVhIVFReXoBLkQPQAr/nWa/iNo7/Bq3e+Or8/lCgaqwVkJQGd9uoI1SUSkAkhhBBCiMIkQVmxcN2lYf7SUSbEluLTfOwu301TpImeeA+xVIyRxAhjyTGaS5qpC9fx2z/4bR4ZfAQXL5Bw8Kaj264NQCwZ44M//iAjiRHecfgdeftZxPZn2g59k0kGppPYtgRkQgghhBBia5GgrFjYWXAdQAE9kO9qhBA3IWSE2F+1n3gmTtdMF7PZWfpm+/iLY3/BDwZ/cEO38Vcn/oq2aBs/0/ozG1ytKDarBWSRgE57dZiaEvm9I4QQQgghtgYJyorF4rJLP6hqfmsRQtySqD9KZ20nsWSM0xOn+X8D/++ql811mOUoKPztyb/lZS0vu6FdNIW4HtN26J9K0j91ZUBWHfHL40wIIYQQQmwpEpQVi8VB/qH81iGEWDfVoWoG5wYXl1feCBeXizMXORk7yZGaIxtXnNj2TNthYCEgsxYCsrBfp6M6THWJBGRCCCGEEGJrkqCsWCwGZbL8RYjt5MnRJ1FQrugcuxZN0Xhy9EkJysRNsZZ1kC0PyLwllhKQCSGEEEKIrU2CsmIhHWVCbEtz5tyaQjIARVFImIkNqkhsV5btMDCdom8yIQGZEEIIIYTYtiQoKxa5HS9lkL8Q20qpr3TNHWWu6xLxRTawKrGdrBaQhfwa7VURakslIBNCCCGEENuLBGXFIjfMXzrKhNhW7qq/i+8PfP+qf69wZYhhuzZ31d+1cUWJbcGyHYbiCfr+//buPTqq+t7//2vuk0lICEkMFyMhwQsogsrBg7bV9oBYe2xpPUq1R4HT2q5WVi+02lqtiFqxVi0eRT2lon7bWj21aHtai9IsaX8KFculy2AW8gAAQKZJREFUKogXEi4qgSRckswkk7l8fn9MMhCZUQKZ2Xtmno+1svZkZ++Z97wZJ5mXn89n7w0pEo1LIiADAABA/rPF5Q+XLFmi2tpa+f1+nX322Vq7dm3aY5cvX67Jkydr6NChKi4u1qRJk/TLX/4yi9XmKNYoA/LSxfUXy+vyHvHxTjk1btg4nVZ5WgarQi6LxY1auqQ1jXv1zp5ORaJxBbwunTqqVFPrKjS8zE9IBgAAgLxleVD25JNPav78+VqwYIHWr1+viRMnasaMGdqzZ0/K44cNG6YbbrhBa9as0auvvqq5c+dq7ty5eu6557JceQ6JhiWTGA0gd5G1tQAYVEO8Q3TV+KuO+Pi44vraxK9lsCLkqljcaHtbUKu3tml3l0M9sUMCsvoKjSgrIiADAABA3rM8KLvnnnt09dVXa+7cuRo/frweeughBQIBLVu2LOXx559/vj7/+c9r3Lhxqq+v17e+9S2dfvrpevHFF7NceQ7pG03m9klOy//JAQyyayZdowtGX/Chx/RNwfzuWd/Vv53wb9koCzkiFjfa0RbSS++06u3dneqJxeV1SeNHEJABAACg8FiamvT09GjdunWaNm1acp/T6dS0adO0Zs2ajzzfGKOGhga9+eab+sQnPpHJUnMbV7wE8prL6dKdn7hT3zzjmyrzlkmS3A63XA6X3I7EUpSjhozSTz/xU805bY6FlcJODg3I3trdoZ5oXEVel8aPKNXYUqMRTLEEAABAAbJ0Mf/W1lbFYjFVV1f3219dXa0tW7akPe/AgQMaNWqUwuGwXC6XHnjgAU2fPj3lseFwWOFwOPl9e3u7JCkSiSgSiQzCszg2fTVktJaudjmiURm5JRs85yORlb7kIPqSHr2R5oyboy+d9CWteneVXm15VV3RLpW5y6Q90q9n/Fper7eg+/NBhfqaicWN3tvfpe17Q+rpXaS/yONSbUVAw0v9isWicjoKry9HolBfMx+FvqRHb1KzU1/sUAMAwF4cxhhj1YO///77GjVqlFavXq2pU6cm91933XX661//qpdffjnlefF4XI2Njers7FRDQ4NuvfVWPfPMMzr//PMPO/bmm2/WwoULD9v/+OOPKxAojBFWQ7p2KtDTqqCvWp3+kVaXAwCwQNxI+8JSS7dDvfmYPE6pqshoqFdyMngMQAEKhUK64oordODAAZWWllpdDgDABiwNynp6ehQIBPTUU09p5syZyf2zZ8/W/v379fvf//6I7ucrX/mKdu7cmXJB/1QjympqatTa2mqLX4aRSEQrV67U9OnT5fF4MvIYjvfWScEWmerTpLLjM/IYgy0bfclF9CU9epMafUmvUHoTjxu9f6Bb29pCCkdjkiR/7wiyEaV+OT+QkBVKX44GvUmNvqRHb1KzU1/a29tVWVlJUAYASLJ06qXX69VZZ52lhoaGZFAWj8fV0NCgefPmHfH9xOPxfmHYoXw+n3w+32H7PR6P5b+YD5XRekxEcruloiGSjZ7zkbDbv5Nd0Jf06E1q9CW9fO1NvHeK5ba2oMKRuCSHiv1e1VYUa9TQosMCsg/K174MBnqTGn1Jj96kZoe+WP34AAD7sTQok6T58+dr9uzZmjx5sqZMmaLFixcrGAxq7ty5kqSrrrpKo0aN0qJFiyRJixYt0uTJk1VfX69wOKxnn31Wv/zlL/Xggw9a+TTsLbmYf5G1dQAAMi4xgqxL21pD6o4kRpD5PM4jDsgAAACAQmZ5UDZr1iy1tLTopptuUnNzsyZNmqQVK1YkF/jfsWOHnM6DF+cMBoP6xje+oXfffVdFRUU65ZRT9Ktf/UqzZs2y6inYW7RHMokPSnITlAFAvorHjXa1d6upJUhABgAAABwly4MySZo3b17aqZarVq3q9/1tt92m2267LQtV5YlIKLF1+6RDAkcAQH7oC8i2tQbV1ZMIyLxup8ZUEpABAAAAA2WLoAwZFO1ObD2FcYVPACgU6QKy2opijSovkouADAAAABgwgrJ8lxxR5re2DgDAoDDGaNeBREAWIiADAAAABhVBWb5LLuTPiDIAyGXGGDX3rkHWF5B53E7VVgR0fHmAgAwAAAAYBARl+S7SN/WSEWUAkIsIyAAAAIDsISjLd31TLxlRBgA5xRij3e1hNbZ2KhQ+GJCNHhbQ8eVFcru4QAsAAAAw2AjK8l1yMf8ia+sAAByRVAGZ2+XQ6Ipi1RCQAQAAABlFUJbPoj1SPJq47SYoAwA7M8ZoT0dYjS1BBcOJ924CMgAAACC7CMryWbR3IX+XV3LyAQsA7ChdQHbCsIBqhgXkISADAAAAsoagLJ9xxUsAsC1jjFo6wmpsDaqzm4AMAAAAsAOCsnyWDMqYdgkAdrKno1uNLQcDMldvQHYCARkAAABgKYKyfEZQBgC2kiogqykPaHQFARkAAABgBwRl+YygDABsYU9Ht5paguogIAMAAABsjaAsn/Ut5s8VLwHAEi0dYTW2dB4MyJwO1fROsfS6CcgAAAAAuyEoy2eMKAMAS7R2Jq5i2d4VkdQXkBXphGHFBGQAAACAjRGU5atYRIonRjAQlAFAdrR2htXUGtSB0MGA7PjyIo2uICADAAAAcgFBWb6KhBJbl1dyuqytBQDyXFtnWI0pArITKgLyuXkPBgAAAHIFQVm+inQntowmA4CMaesdQba/NyBzOqXjexfpJyADAAAAcg9BWb7qG1FGUAYAg25vsEeNLZ0EZAAAAECeISjLV9G+EWUBa+sAgDyyN9ijptZO7QseDMhGDU0EZH4PARkAAACQ6wjK8lXfiDK339o6ACAP7Av2qJGADAAAAMh7BGX5KtKV2DKiDACO2v5Qj7a2BLUv2CMpEZCNHFqk2opiAjIAAAAgDxGU5avkYv6MKAOAgUoVkI0oK9KYSgIyAAAAIJ8RlOWjWESKJ6YHMaIMAI7cgVBEW1s7tbeTgAwAAAAoRARl+ahv2qXLIzn5YAcAH+WDAZnDkQjI6qoIyAAAAIBCQlCWj1ifDACOSCgqbdy5Xwe645IOBmRjKotV5CUgAwAAAAoNQVk+ivYGZVzxEgBSOtAV0Vu79qux3aHyYI88breGl/lVV1lCQAYAAAAUMIKyfMSIMgBIqb07osaWoFo7worGopJDGlHm14nDyxTw8isRAAAAKHR8KshHkVBiyxUvAUBSIiBragmqpSMsKTHFcnipXyeWGo0fUSqPh1+HAAAAAAjK8lOkO7FlRBmAAtfRO4Ls0ICsutSvuqpieRxG2zdaWx8AAAAAeyEoy0fJqZdF1tYBABbp6I6oqTWoPe3h5L7hZX6NqSxWsS/xqy8SiVhVHgAAAACbIijLN7GoFO/98OcmKANQWI4kIAMAAACAdPjUkG/61idzeSQX/7wACkNnOKqmlqB2t3cn91WX+jWmqlglBGQAAAAAjhCfHvJNtPdDIqPJABSAVAHZcaU+1VWVEJABAAAAGDA+ReSb5BUvCcoA5K9gOKqm1qCaD/QPyMZUFmuI32NhZQAAAAByGUFZvkle8ZKgDED+CfVE1dg7gsyYxL6qIT7VVRGQAQAAADh2BGX5hhFlAPJQuoBsTFWxSgnIAAAAAAwSgrJ8E+lKbFmjDEAeSBWQVfaOICMgAwAAADDYCMryTbQ3KGNEGYAc1tUTU2Nrp5oP9A/IxlQWq6yIgAwAAABAZhCU5ZNYVIpFErcJygDkoK6emJpag9p1oCsZkFWUeFVXVUJABgAAACDjCMrySd9oMqdHcvGBEkDuSBuQVZaoLMD7GQAAAIDsICjLJ33rk3n81tYBAEeoO5IIyN7ffzAgG1biVT0BGQAAAAALEJTlkwjrkwHIDX0B2a4DXYrHE/uGlXhVV1msoQGvtcUBAAAAKFgEZfkkGZQFrK0DANLojsS0rS0xgqwvICsv9qq+ioAMAAAAgPUIyvJJ3xplbqZeArCX1AGZR3WVJSovJiADAAAAYA8EZfmEEWUAbKY7EtP2tpDe2x8iIAMAAABgewRl+SQSSmxZowyAxbojMe3YG9K7+w4GZEMDHtVVlWgYARkAAAAAmyIoyxfxmBSLJG4TlAGwSDiaGEFGQAYAAAAgFxGU5Yu+aZdOj+TyWFsLgILTF5C9t69LsbiRJJUFPKqrLFZFic/i6gAAAADgyBCU5Yvk+mQs5A8ge3qicW1vC+pdAjIAAAAAeYCgLF+wPhmALOqJxrVjb1A79x4MyEqLPKqrKlYlARkAAACAHEVQli+i3Ymtm6AMQOakC8jGVBaraggBGQAAAIDcRlCWLxhRBiCDIrG4treFtHNfSLFYIiAb4nerrqqEgAwAAABA3iAoyxeR3hFlnoC1dQDIK+kCsjFVxTpuCGsiAgAAAMgvBGX5IjmijA+uAI5dJBbXjr0h7dh7MCAr8btVR0AGAAAAII8RlOWDeEyK9SRuM6IMwDHoC8h27g0pemhA1rsGmcPhsLhCAAAAAMgcgrJ8EOlKbJ1uyeWxthYAOSkSi2tn7wiyvoCs2OdWfRUBGQAAAIDCQVCWD/queMlC/gAGKBqLa+e+Lm1vC/YLyBJTLAnIAAAAABQWgrJ80Lc+mZugDMCRSRWQBXwu1VeVEJABAAAAKFgEZfmgb+olI8oAfIRoLK5393Vp+96QItG4pERAVldZoupSAjIAAAAAhc1pdQGStGTJEtXW1srv9+vss8/W2rVr0x67dOlSffzjH1d5ebnKy8s1bdq0Dz2+ICSDMq5EByC1WNxoW2tQL21t0zt7OhWJxhXwunTaqDJNravQ8DI/IRkAAACAgmd5UPbkk09q/vz5WrBggdavX6+JEydqxowZ2rNnT8rjV61apcsvv1wvvPCC1qxZo5qaGl1wwQV67733sly5jSSDMq54CaC/WNyotVta/YGA7NRRpZpaT0AGAAAAAIeyPCi75557dPXVV2vu3LkaP368HnroIQUCAS1btizl8b/+9a/1jW98Q5MmTdIpp5yiX/ziF4rH42poaMhy5TYSZeolgP5icaPtbUGtbmxTc8ihnlgiIBs/MhGQjSgrIiADAAAAgA+wdI2ynp4erVu3Ttdff31yn9Pp1LRp07RmzZojuo9QKKRIJKJhw4Zlqkx7i8elaDhxm8X8gYIXixu9t69L29qC6onGFY3F5XFJ40eUqqaihHAMAAAAAD6EpUFZa2urYrGYqqur++2vrq7Wli1bjug+vv/972vkyJGaNm1ayp+Hw2GFw+Hk9+3t7ZKkSCSiSCRylJUPnr4ajrqWnqAc0ajkdMkYh2SD5zQYjrkveYq+pFfovYnFjd7bn1ikv6d3kf4ij0u15UXqKjWqDLgUjUYtrtJeCv01kw59SY/epEZf0qM3qdmpL3aoAQBgLw5jjLHqwd9//32NGjVKq1ev1tSpU5P7r7vuOv31r3/Vyy+//KHn33HHHbrzzju1atUqnX766SmPufnmm7Vw4cLD9j/++OMKBHJ/TS9vpF3loa2KOv1qGzLO6nIAZFncSPvCUku3Q735mDxOqarIaKhXcjKADACAtEKhkK644godOHBApaWlVpcDALABS0eUVVZWyuVyaffu3f327969W8OHD//Qc++66y7dcccd+stf/pI2JJOk66+/XvPnz09+397enrwAgB1+GUYiEa1cuVLTp0+Xx+MZ+B0ceFeO3UOl4uNkRp056PVZ5Zj7kqfoS3qF1pt43Oj9A93a1hZSUTSmkZL8HpdqKwIaUeqXszchK7S+DAS9SY2+pEdvUqMv6dGb1OzUl77ZJgAA9LE0KPN6vTrrrLPU0NCgmTNnSlJyYf558+alPe/OO+/Uj3/8Yz333HOaPHnyhz6Gz+eTz+c7bL/H47H8F/OhjroeE5HcbqmoRLLR8xksdvt3sgv6kl6+9ybeO8VyW1tQ4UhckkPFfq/GVBZrZFlRMiD7oHzvy7GgN6nRl/ToTWr0JT16k5od+mL14wMA7MfSoEyS5s+fr9mzZ2vy5MmaMmWKFi9erGAwqLlz50qSrrrqKo0aNUqLFi2SJP3kJz/RTTfdpMcff1y1tbVqbm6WJJWUlKikpMSy52GZSCix5YqXQF5LjCDr0rbWkLojMUmSz+NUbUWxRg1NH5ABAAAAAI6c5UHZrFmz1NLSoptuuknNzc2aNGmSVqxYkVzgf8eOHXI6ncnjH3zwQfX09Og//uM/+t3PggULdPPNN2ezdHuIdie2XPESyEsEZAAAAACQPZYHZZI0b968tFMtV61a1e/7bdu2Zb6gXMKIMiAvxeNGu9q71dQSJCADAAAAgCyxRVCGoxSPS9Fw4jZBGZAX+gKyba1BdfUkAjKvuzcgKy+Si4AMAAAAADKGoCyXRbsSW4dLch9+wQIAucMYo10HutVEQAYAAAAAliEoy2WR3vXJPH5r6wBw1PoCsm2tQYV6AzKP26naioCOLw8QkAEAAABAFhGU5bLk+mQBa+sAMGDGGDX3rkFGQAYAAAAA9kBQlssivVMvWZ8MyBnGGO1uD6uxpbNfQDZ6WEDHlxfJ7XJ+xD0AAAAAADKFoCyX9a1R5mbqJWB3yYCstVOhcCIgc7scqq0oJiADAAAAAJsgKMtlyRFlTL0E7MoYoz0dYW1t6R+Qja4oVg0BGQAAAADYCkFZLksGZYwoA+ymLyBrbAkqGI5KIiADAAAAALsjKMtV8bgUDSduM6IMsA1jjFo6wtr6gYDshGEB1QwLyENABgAAAAC2RVCWq6LdkozkcEpun9XVAAWvLyBrbA2qszsRkLl6A7ITCMgAAAAAICcQlOUqrngJ2Maejm41thCQAQAAAECuIyjLVckrXhKUAVbZ09GtppagOg4JyGrKAxpdQUAGAAAAALmIoCxXMaIMsExLR1iNLZ2HBWQnDAvI6yYgAwAAAIBcRVCWqyKhxJagDMialo6wmlqDau+KSJJcTodqhhXphGHFBGQAAAAAkAcIynJVpDuxJSgDMq61M6zGFgIyAAAAAMh3BGW5ihFlQMa1dSauYnkgdDAgO768SCdUBORzuyyuDgAAAAAw2AjKcpExUjScuM1i/sCgSxWQjSov0mgCMgAAAADIawRluSjSJclIDqfk9lldDZA39gZ71NjSqf29AZnTKR3fexVLAjIAAAAAyH8EZbko2rs+mdsvORzW1gLkgXQB2QnDAvJ7CMgAAAAAoFAQlOWi5PpkAWvrAHLcvmCPGls7tS94MCAbNTQxgoyADAAAAAAKD0FZLkpe8dJvbR1Ajtof6tHWlqD2BXskJQKykUOLVFtRTEAGAAAAAAWMoCwXMaIMOCoEZAAAAACAD0NQlosiXYmthyteAkdif6hHja1B7e08GJCNKCvSmEoCMgAAAADAQQRluSjaG5S5mXoJfJgDoYi2tnYmAzKHIzGCjIAMAAAAAJAKQVmuMeaQNcqYegmkcqArosaWTrUdEpD1jSAr8hKQAQAAAABSIyjLNdFuSUZyOCW3z+pqAFsJRaWNO/frQHdcEgEZAAAAAGBgCMpyTeSQaZcOh7W1ADbR3h3RW7sOqLHdofJgjzxut4aX+TWmslgBL29zAAAAAIAjwyfIXMNC/kBSe3dEjS1BtXaEFY1FJYc0vNSvk0aUEZABAAAAAAaMT5K5hqAMUEdvQNbSEZaUGFw5vNSvE0uNTh1ZKo+HtzYAAAAAwMDxaTLX9F3xkoX8UYA+GJBJSk6x9DqNtm+0rjYAAAAAQO4jKMs1h65RBhSIju6ImlqD2tN+eEBW7Eu8jUUiEavKAwAAAADkCYKyXBMJJbZMvUQB6AxH1dQS1O727uS+6lK/xlQVq8TH2xcAAAAAYHDxSTOXGCNFe0fUEJQhjxGQAQAAAACswCfOXBINSyYuycHUS+SlYDiqptagmg8cDMiOK/VpTGWxhvg9FlYGAAAAACgEBGW5JDnt0p+4zB+QJwjIAAAAAAB2QFCWSyJc8RL5JdQTVWPvFEtjEvuqhvhUV0VABgAAAADIPoKyXBLtHW3DtEvkuHQB2ZiqYpUSkAEAAAAALEJQlkuSUy8ZUYbcFOo5OMWyLyCr7B1BRkAGAAAAALAaQVkuifSOKOOKl8gxXT0xNbUGtetAVzIgqyjxqq6qRGVFBGQAAAAAAHsgKMslyRFlBGXIDWkDssoSlQUIyAAAAAAA9kJQliuMObhGGUEZbK47ElNjS/+AbFiJV/UEZAAAAAAAGyMoyxXRsGTikhws5g/b6o4cHEEWjyf2DSvxqq6yWEMDXmuLAwAAAADgIxCU5YrktEu/5HBYWwvwAd2RmLa1BfX+/oMBWXmxV/VVBGQAAAAAgNxBUJYr+qZdupl2CfsgIAMAAAAA5BOCslwR6UpsWZ8MNtAdiWl7W0jv7Q8dEpB5VFdZovJiAjIAAAAAQG4iKMsVBGWwgVQB2dCAR3VVJRpGQAYAAAAAyHEEZbmCoAwWCkcTAdm7+wjIAAAAAAD5i6AsVyQX8w9YWwcKSl9A9t6+LsXiRpJUFvCorrJYFSU+i6sDAAAAAGBwEZTlAmMOWczfb20tKAjhaEw72kJ6l4AMAAAAAFBACMpyQaxHMnFJDoIyZFRPNK4de4PaufdgQFZa5FFdVbEqCcgAAAAAAHmOoCwX9E27dPskp9PaWpCXCMgAAAAAACAoyw3JhfxZnwyDKxGQhbRzX0ixWCIgG+J3q66qRFVDCMgAAAAAAIWFoCwXJIMypl1icERicW1vOzwgG1NVrOOG8DoDAAAAABQmgrJcwIgyDJJILDGCbMfegwFZid+tOgIyAAAAAAAIynICV7zEMeoLyHbuDSn6gYCsqsQnh8NhcYUAAAAAAFiPoCwX9C3mz4gyDFAkFtfO3hFkfQFZsc+t+qpiVQ0hIAMAAAAA4FAEZbmANcowQNFYXDv3dWl7W7BfQJaYYklABgAAAABAKgRldhcNSyaeuO0usrYW2F6qgCzgc6m+qoSADAAAAACAj0BQZnd9o8ncPsnptLYW2Fa6gKyuskTVpQRkAAAAAAAcCcuTlyVLlqi2tlZ+v19nn3221q5dm/bYTZs26ZJLLlFtba0cDocWL16cvUKtwhUv8SGisbi2tQb10tY2bd3TqWjMKOBz6bRRZZpaV6HhZX5CMgAAAAAAjpClQdmTTz6p+fPna8GCBVq/fr0mTpyoGTNmaM+ePSmPD4VCqqur0x133KHhw4dnuVqL9C3kzxUvcYhY3Gh7WyIge2dPpyLRuAJel04dVUpABgAAAADAUbI0KLvnnnt09dVXa+7cuRo/frweeughBQIBLVu2LOXx//Iv/6Kf/vSn+uIXvyifz5flai0S7U5sGVEGJQKy1m5pdWOb3t79gYCsvkIjyooIyAAAAAAAOEqWBWU9PT1at26dpk2bdrAYp1PTpk3TmjVrrCrLfpJTL1nIv5DF4kY72kJa3dim5pBDPdG4irwujR9JQAYAAAAAwGCxbDH/1tZWxWIxVVdX99tfXV2tLVu2DNrjhMNhhcPh5Pft7e2SpEgkokgkMmiPc7T6akhXi6O7Q4pGZeSWbFBvtnxUXwpFLG703v4ubd8bUk80rlg0Jo9LGltZpJphJXI6HYpGo1aXaQu8ZlKjL+nRm9ToS3r0JjX6kh69Sc1OfbFDDQAAe8n7q14uWrRICxcuPGz/888/r0DAPtMZV65cmXL/cQf+KYfian2zSzFXgUw3PUS6vuS7uJH2haWWboei8cQ+j0uq8hudWCptWvv/aZO1JdpWob5mPgp9SY/epEZf0qM3qdGX9OhNanboSygUsroEAIDNWBaUVVZWyuVyaffu3f327969e1AX6r/++us1f/785Pft7e2qqanRBRdcoNLS0kF7nKMViUS0cuVKTZ8+XR6Pp/8PYz1ybE3MjjUnXiA5LL9IadZ8aF/yWDxu9N6BLm1v61JRNKaRkvwel8ZUBDS81K9YLFqQfTkShfqa+Sj0JT16kxp9SY/epEZf0qM3qdmpL32zTQAA6GNZUOb1enXWWWepoaFBM2fOlCTF43E1NDRo3rx5g/Y4Pp8v5cL/Ho/H8l/Mh0pZTzQoud2S2yd5C280mWS/f6dMifdOsdzWFlQ4EpfkUInfp9rKgEaWFcnpTKw/FokktoXSl6NBb1KjL+nRm9ToS3r0JjX6kh69Sc0OfbH68QEA9mPp1Mv58+dr9uzZmjx5sqZMmaLFixcrGAxq7ty5kqSrrrpKo0aN0qJFiyQlLgCwefPm5O333ntPGzduVElJicaOHWvZ88gYrniZ9+Jxo/cPdGlba0jdkZgkyedxqraiWKOGHgzIAAAAAABA5lkalM2aNUstLS266aab1NzcrEmTJmnFihXJBf537Nghp/PgdMP3339fZ5xxRvL7u+66S3fddZfOO+88rVq1KtvlZ16kd80Et9/aOjDoCMgAAAAAALAfyxfznzdvXtqplh8Mv2pra2WMyUJVNhHpSmwZUZY34nGjXe3damoJEpABAAAAAGAzlgdl+BCRvqmXjCjLdX0B2bbWoLp6EgGZ190bkJUXyUVABgAAAACA5QjK7Kxv6iUjynKWMUa7DnSriYAMAAAAAADbIyizs+Ri/kXW1oEB6wvItrUGFSIgAwAAAAAgJxCU2VW0R4pHE7fdBGW5whij5t41yPoCMo/bqdqKgI4vDxCQAQAAAABgYwRldhXtXcjf5ZUOufIn7MkYo93tYTW2dPYLyEYPC6hmGAEZAAAAAAC5gKDMrrjiZU5IBmStnQqF+wdkx5cXye0i5AQAAAAAIFcQlNlVMihj2qUdpQrI3C6HRlcUq4aADAAAAACAnERQZlcEZbZkjNGejrAaW4IKhhNryBGQAQAAAACQHwjK7IqgzFaMMWrpCGvrBwKyE3rXIPMQkAEAAAAAkPMIyuyqbzF/rnhpqb6ArLE1qM5uAjIAAAAAAPIZQZldMaLMcns6utXYcjAgc/UGZCcQkAEAAAAAkJcIyuwoFpHiiXCGoCz7UgVkNeUBja4gIAMAAAAAIJ8RlNlRJJTYuryS02VtLQWkpSOsxpZOdRCQAQAAAABQkAjK7CjSndgymiwrWjrCamoNqr0rIklyOR2qGVakE4YVy+smIAMAAAAAoFAQlNlR34gygrKMau0Mq7GFgAwAAAAAACQQlNlRtG9EWcDaOvJUW2fiKpYHQgcDsuPLizS6goAMAAAAAIBCRlBmR30jytx+a+vIM+kCshMqAvK5WQsOAAAAAIBCR1BmR5GuxJYRZYOirTOxBtn+3oDM6ZSO712kn4AMAAAAAAD0ISizo+Ri/owoOxZ7gz1qbOk8LCA7YVhAfg8BGQAAAAAA6I+gzG5iESmeCHYYUXZ09gV71NjaqX3BgwHZqKGJEWQEZAAAAAAAIB2CMrvpm3bp8khOQp2BICADAAAAAADHgqDMblifbMD2h3q0tSWofcEeSYmAbOTQItVWFBOQAQAAAACAI0ZQZjfR3qCMK15+pP2hHjW2BrW382BANqKsSGMqCcgAAAAAAMDAEZTZDSPKPtKBUERbWzuTAZnDkRhBRkAGAAAAAACOBUGZ3URCia2nyNo6bChVQNY3gqzIS0AGAAAAAACODUGZ3US6E1uCsqQDXRE1tnSqjYAMAAAAAABkEEGZ3SSnXhKUhaLSP989oP1dMUmJgGx4mV91lSUEZAAAAAAAYNARlNlJLCrFI4nb7sINytq7I3pr1wE1tjtU3hmWx+3W8DK/xlQWK+DlJQsAAAAAADKD1MFO+tYnc3kkV+H903R0R9TYElRLR1jRWFRySMNL/TppRBkBGQAAAAAAyDjSBzuJ9q5PVmCjyQ4NyKTeKZalfp1YanTqyFJ5PLxMAQAAAABA5pFA2EmBXfGyozuiptag9rSHk/v6plh6nUbbN1pXGwAAAAAAKDwEZXZSIFe87AxH1djSmTIgK/YlXpKRSMSq8gAAAAAAQIEiKLOTPB9R1hmOqqklqN3t3cl91aV+jakqVomPlyIAAAAAALAW6YSdRLoS2zxboywYjqqpNajmAwcDsuNKfaqrKiEgAwAAAAAAtkFKYSfR3qAsT0aUpQvIxlQWa4jfY2FlAAAAAAAAhyMos4t4VIr1rsuV40FZqCeqxt4plsYk9lUN8amuioAMAAAAAADYF0GZXfRNu3R6JFduhknpArIxVcUqJSADAAAAAAA2R1BmF5HcnXYZ6jk4xbIvIKvsHUFGQAYAAAAAAHIFQZldRHvX8fL4ra1jALp6Ymps7TwsIBtTWayyIgIyAAAAAACQWwjK7CI5oixgbR1HoKsnpqbWoHYd6EoGZBUlXtVVlRCQAQAAAACAnEVQZhd9V7x023dEWaqAbFiJV/WVJSoLEJABAAAAAIDcRlBmEw4bjyjrjhwMyOLxxD4CMgAAAAAAkG8Iyuwi0iU5ZKvF/NMFZHWVxRoa8FpbHAAAAAAAwCAjKLMDE5diPZLbbYugrDsS07a2oN7ffzAgKy/2qr6KgAwAAAAAAOQvgjIbcMV7JLklp0dyWTeVsTsS0/a2kN7bHzokIPOorrJE5cUEZAAAAAAAIL8RlNlAIigLSB5rFvInIAMAAAAAACAoswVXPJy4keVpl+FoIiB7d9/BgGxowKO6qhINIyADAAAAAAAFhqDMBlymJ3HDnZ2gLF1ANqayWBUlvqzUAAAAAAAAYDcEZTaQmHqpjI8oC0dj2tEW0rv7uhSLG0lSWcCjOgIyAAAAAAAAgjI7OBiUBTJy/z3RuLa3BfsFZKVFHtVVFauSgAwAAAAAAEASQZktHAzKBncx/55oXDv2BrVzLwEZAAAAAADARyEos1o8JqeJJm4P0oiyREAW0s59IcViiYBsiN+tuqoSVQ0hIAMAAAAAAEiFoMxq0a7E1umWXJ5juqtILK7tbQRkAAAAAAAAR4OgzGqR7sT2GBbyTxWQlfjdqqsq1nFDBnc6JwAAAAAAQL4iKLNa34gy98CDskgsMcVyx14CMgAAAAAAgGNFUGa1SCIoMwMYURaJxbWzNyCLHhqQVRaraohPDocjI6UCAAAAAADkM4IyC2x9dbX2Pv9TjelcJ6/iUt1XtOk316tk+g81duK5ac+LHjKCrC8gK/a5VV9FQAYAAAAAAHCsCMqyqHnnOwo98gXVx7erzkgOhxR1eCVJ4ztfkffpi9T4+xMUmLNcw084MXleNBbXzn1d2t4W7BeQJaZYEpABAAAAAAAMBoKyLHm/aYuGPHq+qtUlORIhWUIi+HI4EjfHxHao8+GP6b3Zq1Q9+uTDArKAz6W6yhJVlxKQAQAAAAAADCan1QVI0pIlS1RbWyu/36+zzz5ba9eu/dDjf/vb3+qUU06R3+/XhAkT9Oyzz2ap0qMTjfTI9diFKlGXPirbcjgkv+lR86NX6W9bmrV1T6eiMaOAz6XTRpVpal2Fhpf5CckAAAAAAAAGmeVB2ZNPPqn58+drwYIFWr9+vSZOnKgZM2Zoz549KY9fvXq1Lr/8cn35y1/Whg0bNHPmTM2cOVOvv/56lis/cq/+5Teq1r60IZmRU3I4FDMObYtXa7U5TftMid5e/4ICXpdOHVVKQAYAAAAAAJBhlgdl99xzj66++mrNnTtX48eP10MPPaRAIKBly5alPP7ee+/VhRdeqGuvvVbjxo3TrbfeqjPPPFP3339/lis/ciX/uPdDfx51eLU9XqUX4xP0jhmliNwKqFsTGn+uqfUVGlFWREAGAAAAAACQYZYGZT09PVq3bp2mTZuW3Od0OjVt2jStWbMm5Tlr1qzpd7wkzZgxI+3xVusOdeqk2NaUP4sZqcWUqanLp3fiI5MB2XjnNk11btY5ZqO6Q51ZrhgAAAAAAKAwWbqYf2trq2KxmKqrq/vtr66u1pYtW1Ke09zcnPL45ubmlMeHw2GFw+Hk9+3t7ZKkSCSiSCRyLOUfkf1te1Th9Kf82ZuxEepSkbqNVx6ndKJ2qdqxX06HUVSJc/a17Zbbm/r8fNb3b5ONf6NcQl/Sozep0Zf06E1q9CU9epMafUmP3qRmp77YoQYAgL3k/VUvFy1apIULFx62//nnn1cgEMhOERN/nnK3CbYo2LlfQwN+tU+6RhtSza7csCnxVaBWrlxpdQm2RF/Sozep0Zf06E1q9CU9epMafUmP3qRmh76EQiGrSwAA2IylQVllZaVcLpd2797db//u3bs1fPjwlOcMHz58QMdff/31mj9/fvL79vZ21dTU6IILLlBpaekxPoOPFo/F1PWTk1Xs6E7587CjSA2n36vpr31Tnnj/Y0LGJ//335LT5cp4nXYTiUS0cuVKTZ8+XR6Px+pybIO+pEdvUqMv6dGb1OhLevQmNfqSHr1JzU596ZttAgBAH0uDMq/Xq7POOksNDQ2aOXOmJCkej6uhoUHz5s1Lec7UqVPV0NCgb3/728l9K1eu1NSpU1Me7/P55PP5Dtvv8Xiy84vZ49H64Z/Vv77//1Jf9bJ3lThPvLtfUGaMtHnkpZrqL7xpl4fK2r9TjqEv6dGb1OhLevQmNfqSHr1Jjb6kR29Ss0NfrH58AID9WH7Vy/nz52vp0qV67LHH9MYbb+jrX/+6gsGg5s6dK0m66qqrdP311yeP/9a3vqUVK1bo7rvv1pYtW3TzzTfrH//4R9pgzQ7q/32+zADPMZLqPvOdTJQDAAAAAACAFCxfo2zWrFlqaWnRTTfdpObmZk2aNEkrVqxILti/Y8cOOZ0H87xzzjlHjz/+uG688Ub98Ic/1IknnqhnnnlGp512mlVP4SMdN2qM1pz4XU195+4jPuflsd/W1OPrM1gVAAAAAAAADmV5UCZJ8+bNSzsibNWqVYftu/TSS3XppZdmuKrBNfU/b9Kax8Ka2nS/jFHqaZi91oz+hqZeefgFCAAAAAAAAJA5lk+9LCRTZ/9Yb332D9pQfK5ipn9SFjMOrQ+co7c++3tNnbvIogoBAAAAAAAKly1GlBWSk848TzrzPO1r2aWdm1arJ9QuRaSOr/5DZ4443uryAAAAAAAAChZBmUXKq0ao/PxLFIlE9N6zz2poZbXVJQEAAAAAABQ0pl4CAAAAAAAAIigDAAAAAAAAJBGUAQAAAAAAAJIIygAAAAAAAABJBGUAAAAAAACAJIIyAAAAAAAAQBJBGQAAAAAAACCJoAwAAAAAAACQRFAGAAAAAAAASCIoAwAAAAAAACQRlAEAAAAAAACSCMoAAAAAAAAASQRlAAAAAAAAgCSCMgAAAAAAAEASQRkAAAAAAAAgiaAMAAAAAAAAkERQBgAAAAAAAEgiKAMAAAAAAAAkEZQBAAAAAAAAkgjKAAAAAAAAAEkEZQAAAAAAAIAkgjIAAAAAAABAEkEZAAAAAAAAIImgDAAAAAAAAJBEUAYAAAAAAABIktxWF5BtxhhJUnt7u8WVJEQiEYVCIbW3t8vj8Vhdjm3Ql9ToS3r0JjX6kh69SY2+pEdvUqMv6dGb1OzUl77PBH2fEQAAKLigrKOjQ5JUU1NjcSUAAAAA7KCjo0NlZWVWlwEAsAGHKbD/fRKPx/X+++9ryJAhcjgcVpej9vZ21dTUaOfOnSotLbW6HNugL6nRl/ToTWr0JT16kxp9SY/epEZf0qM3qdmpL8YYdXR0aOTIkXI6WZUGAFCAI8qcTqeOP/54q8s4TGlpqeV/KNgRfUmNvqRHb1KjL+nRm9ToS3r0JjX6kh69Sc0ufWEkGQDgUPxvEwAAAAAAAEAEZQAAAAAAAIAkgjLL+Xw+LViwQD6fz+pSbIW+pEZf0qM3qdGX9OhNavQlPXqTGn1Jj96kRl8AAHZWcIv5AwAAAAAAAKkwogwAAAAAAAAQQRkAAAAAAAAgiaAMAAAAAAAAkERQBgAAAAAAAEgiKMu4JUuWqLa2Vn6/X2effbbWrl37ocf/9re/1SmnnCK/368JEybo2WefzVKl2TeQ3mzatEmXXHKJamtr5XA4tHjx4uwVmmUD6cvSpUv18Y9/XOXl5SovL9e0adM+8jWWywbSm+XLl2vy5MkaOnSoiouLNWnSJP3yl7/MYrXZM9D3mT5PPPGEHA6HZs6cmdkCLTSQ3jz66KNyOBz9vvx+fxarzZ6Bvmb279+va665RiNGjJDP59NJJ52Ut7+fBtKb888//7DXjMPh0Gc+85ksVpwdA33NLF68WCeffLKKiopUU1Oj73znO+ru7s5StdkzkL5EIhHdcsstqq+vl9/v18SJE7VixYosVps9f/vb33TxxRdr5MiRcjgceuaZZz7ynFWrVunMM8+Uz+fT2LFj9eijj2a8TgAAUjLImCeeeMJ4vV6zbNkys2nTJnP11VeboUOHmt27d6c8/qWXXjIul8vceeedZvPmzebGG280Ho/HvPbaa1muPPMG2pu1a9ea733ve+Y3v/mNGT58uPnZz36W3YKzZKB9ueKKK8ySJUvMhg0bzBtvvGHmzJljysrKzLvvvpvlyjNvoL154YUXzPLly83mzZvNO++8YxYvXmxcLpdZsWJFlivPrIH2pU9TU5MZNWqU+fjHP24+97nPZafYLBtobx555BFTWlpqdu3alfxqbm7OctWZN9C+hMNhM3nyZHPRRReZF1980TQ1NZlVq1aZjRs3ZrnyzBtob9ra2vq9Xl5//XXjcrnMI488kt3CM2ygffn1r39tfD6f+fWvf22amprMc889Z0aMGGG+853vZLnyzBpoX6677jozcuRI86c//cls3brVPPDAA8bv95v169dnufLMe/bZZ80NN9xgli9fbiSZp59++kOPb2xsNIFAwMyfP99s3rzZ3HfffXn5OxsAkBsIyjJoypQp5pprrkl+H4vFzMiRI82iRYtSHn/ZZZeZz3zmM/32nX322eZrX/taRuu0wkB7c6jRo0fnbVB2LH0xxphoNGqGDBliHnvssUyVaJlj7Y0xxpxxxhnmxhtvzER5ljmavkSjUXPOOeeYX/ziF2b27Nl5G5QNtDePPPKIKSsry1J11hloXx588EFTV1dnenp6slWiZY71feZnP/uZGTJkiOns7MxUiZYYaF+uueYa86lPfarfvvnz55tzzz03o3Vm20D7MmLECHP//ff32/eFL3zBfOlLX8ponVY7kqDsuuuuM6eeemq/fbNmzTIzZszIYGUAAKTG1MsM6enp0bp16zRt2rTkPqfTqWnTpmnNmjUpz1mzZk2/4yVpxowZaY/PVUfTm0IwGH0JhUKKRCIaNmxYpsq0xLH2xhijhoYGvfnmm/rEJz6RyVKz6mj7csstt+i4447Tl7/85WyUaYmj7U1nZ6dGjx6tmpoafe5zn9OmTZuyUW7WHE1f/vCHP2jq1Km65pprVF1drdNOO0233367YrFYtsrOisF4D3744Yf1xS9+UcXFxZkqM+uOpi/nnHOO1q1bl5yG2NjYqGeffVYXXXRRVmrOhqPpSzgcPmw6d1FRkV588cWM1poLCuVvYABAbiAoy5DW1lbFYjFVV1f3219dXa3m5uaU5zQ3Nw/o+Fx1NL0pBIPRl+9///saOXLkYX9s5rqj7c2BAwdUUlIir9erz3zmM7rvvvs0ffr0TJebNUfTlxdffFEPP/ywli5dmo0SLXM0vTn55JO1bNky/f73v9evfvUrxeNxnXPOOXr33XezUXJWHE1fGhsb9dRTTykWi+nZZ5/Vj370I91999267bbbslFy1hzre/DatWv1+uuv6ytf+UqmSrTE0fTliiuu0C233KKPfexj8ng8qq+v1/nnn68f/vCH2Sg5K46mLzNmzNA999yjt99+W/F4XCtXrtTy5cu1a9eubJRsa+n+Bm5vb1dXV5dFVQEAChVBGZAn7rjjDj3xxBN6+umn83YB8oEaMmSINm7cqFdeeUU//vGPNX/+fK1atcrqsizT0dGhK6+8UkuXLlVlZaXV5djO1KlTddVVV2nSpEk677zztHz5clVVVel//ud/rC7NUvF4XMcdd5x+/vOf66yzztKsWbN0ww036KGHHrK6NFt5+OGHNWHCBE2ZMsXqUiy3atUq3X777XrggQe0fv16LV++XH/605906623Wl2ape69916deOKJOuWUU+T1ejVv3jzNnTtXTid/jgMAYCduqwvIV5WVlXK5XNq9e3e//bt379bw4cNTnjN8+PABHZ+rjqY3heBY+nLXXXfpjjvu0F/+8hedfvrpmSzTEkfbG6fTqbFjx0qSJk2apDfeeEOLFi3S+eefn8lys2agfdm6dau2bdumiy++OLkvHo9Lktxut958803V19dntugsGYz3GY/HozPOOEPvvPNOJkq0xNH0ZcSIEfJ4PHK5XMl948aNU3Nzs3p6euT1ejNac7Ycy2smGAzqiSee0C233JLJEi1xNH350Y9+pCuvvDI5um7ChAkKBoP66le/qhtuuCEvgqGj6UtVVZWeeeYZdXd3q62tTSNHjtQPfvAD1dXVZaNkW0v3N3BpaamKioosqgoAUKhy/y8Vm/J6vTrrrLPU0NCQ3BePx9XQ0KCpU6emPGfq1Kn9jpeklStXpj0+Vx1NbwrB0fblzjvv1K233qoVK1Zo8uTJ2Sg16wbrNROPxxUOhzNRoiUG2pdTTjlFr732mjZu3Jj8+uxnP6tPfvKT2rhxo2pqarJZfkYNxmsmFovptdde04gRIzJVZtYdTV/OPfdcvfPOO8lQVZLeeustjRgxIm9CMunYXjO//e1vFQ6H9Z//+Z+ZLjPrjqYvoVDosDCsL2g1xmSu2Cw6lteL3+/XqFGjFI1G9bvf/U6f+9znMl2u7RXK38AAgBxh9dUE8tkTTzxhfD6fefTRR83mzZvNV7/6VTN06FDT3NxsjDHmyiuvND/4wQ+Sx7/00kvG7Xabu+66y7zxxhtmwYIFxuPxmNdee82qp5AxA+1NOBw2GzZsMBs2bDAjRoww3/ve98yGDRvM22+/bdVTyIiB9uWOO+4wXq/XPPXUU2bXrl3Jr46ODqueQsYMtDe33367ef75583WrVvN5s2bzV133WXcbrdZunSpVU8hIwbalw/K56teDrQ3CxcuNM8995zZunWrWbdunfniF79o/H6/2bRpk1VPISMG2pcdO3aYIUOGmHnz5pk333zT/PGPfzTHHXecue2226x6ChlztP89fexjHzOzZs3KdrlZM9C+LFiwwAwZMsT85je/MY2Njeb555839fX15rLLLrPqKWTEQPvy97//3fzud78zW7duNX/729/Mpz71KTNmzBizb98+i55B5nR0dCT/bpNk7rnnHrNhwwazfft2Y4wxP/jBD8yVV16ZPL6xsdEEAgFz7bXXmjfeeMMsWbLEuFwus2LFCqueAgCggBGUZdh9991nTjjhBOP1es2UKVPM3//+9+TPzjvvPDN79ux+x//v//6vOemkk4zX6zWnnnqq+dOf/pTlirNnIL1pamoykg77Ou+887JfeIYNpC+jR49O2ZcFCxZkv/AsGEhvbrjhBjN27Fjj9/tNeXm5mTp1qnniiScsqDrzBvo+c6h8DsqMGVhvvv3tbyePra6uNhdddJFZv369BVVn3kBfM6tXrzZnn3228fl8pq6uzvz4xz820Wg0y1Vnx0B7s2XLFiPJPP/881muNLsG0pdIJGJuvvlmU19fb/x+v6mpqTHf+MY38jIQGkhfVq1aZcaNG2d8Pp+pqKgwV155pXnvvfcsqDrzXnjhhZR/n/T1Y/bs2Yf9DffCCy+YSZMmGa/Xa+rq6swjjzyS9boBADDGGIcxeTIGHgAAAAAAADgGrFEGAAAAAAAAiKAMAAAAAAAAkERQBgAAAAAAAEgiKAMAAAAAAAAkEZQBAAAAAAAAkgjKAAAAAAAAAEkEZQAAAAAAAIAkgjIAQA5YtWqVHA6H9u/fb3UpAAAAAPIYQRkAIOPmzJkjh8Mhh8Mhj8ejMWPG6LrrrlN3d7fVpSXdfPPNmjRpktVlAAAAALCQ2+oCAACF4cILL9QjjzyiSCSidevWafbs2XI4HPrJT35idWkAAAAAIIkRZQCALPH5fBo+fLhqamo0c+ZMTZs2TStXrpQkhcNhffOb39Rxxx0nv9+vj33sY3rllVcOu4+XXnpJp59+uvx+v/71X/9Vr7/+evJnqUaELV68WLW1tcnvV61apSlTpqi4uFhDhw7Vueeeq+3bt+vRRx/VwoUL9c9//jM58u3RRx+VJDkcDv3iF7/Q5z//eQUCAZ144on6wx/+0O9xXn/9dX36059WSUmJqqurdeWVV6q1tTX586eeekoTJkxQUVGRKioqNG3aNAWDwQ+tCQAAAED2EZQBALLu9ddf1+rVq+X1eiVJ1113nX73u9/pscce0/r16zV27FjNmDFDe/fu7Xfetddeq7vvvluvvPKKqqqqdPHFFysSiRzRY0ajUc2cOVPnnXeeXn31Va1Zs0Zf/epX5XA4NGvWLH33u9/Vqaeeql27dmnXrl2aNWtW8tyFCxfqsssu06uvvqqLLrpIX/rSl5K17d+/X5/61Kd0xhln6B//+IdWrFih3bt367LLLpMk7dq1S5dffrn+67/+S2+88YZWrVqlL3zhCzLGfGhNAAAAALKPqZcAgKz44x//qJKSEkWjUYXDYTmdTt1///0KBoN68MEH9eijj+rTn/60JGnp0qVauXKlHn74YV177bXJ+1iwYIGmT58uSXrsscd0/PHH6+mnn06GUh+mvb1dBw4c0L//+7+rvr5ekjRu3Ljkz0tKSuR2uzV8+PDDzp0zZ44uv/xySdLtt9+u//7v/9batWt14YUX6v7779cZZ5yh22+/PXn8smXLVFNTo7feekudnZ2KRqP6whe+oNGjR0uSJkyYIEnau3fvh9YEAAAAILsYUQYAyIpPfvKT2rhxo15++WXNnj1bc+fO1SWXXKKtW7cqEono3HPPTR7r8Xg0ZcoUvfHGG/3uY+rUqcnbw4YN08knn3zYMekMGzZMc+bM0YwZM3TxxRfr3nvv1a5du47o3NNPPz15u7i4WKWlpdqzZ48k6Z///KdeeOEFlZSUJL9OOeUUSdLWrVs1ceJE/du//ZsmTJigSy+9VEuXLtW+ffuOuSYAAAAAg4+gDACQFcXFxRo7dqwmTpyoZcuW6eWXX9bDDz88aPfvdDpljOm374PTMh955BGtWbNG55xzjp588kmddNJJ+vvf//6R9+3xePp973A4FI/HJUmdnZ26+OKLtXHjxn5fb7/9tj7xiU/I5XJp5cqV+vOf/6zx48frvvvu08knn6ympqZjqgkAAADA4CMoAwBkndPp1A9/+EPdeOONqq+vl9fr1UsvvZT8eSQS0SuvvKLx48f3O+/QAGnfvn166623klMVq6qq1Nzc3C8s27hx42GPfcYZZ+j666/X6tWrddppp+nxxx+XJHm9XsVisQE/lzPPPFObNm1SbW2txo4d2++ruLhYUiJYO/fcc7Vw4UJt2LBBXq9XTz/99EfWBAAAACC7CMoAAJa49NJL5XK59OCDD+rrX/+6rr32Wq1YsUKbN2/W1VdfrVAopC9/+cv9zrnlllvU0NCg119/XXPmzFFlZaVmzpwpSTr//PPV0tKiO++8U1u3btWSJUv05z//OXluU1OTrr/+eq1Zs0bbt2/X888/r7fffjsZtNXW1qqpqUkbN25Ua2urwuHwET2Pa665Rnv37tXll1+uV155RVu3btVzzz2nuXPnKhaL6eWXX9btt9+uf/zjH9qxY4eWL1+ulpYWjRs37iNrAgAAAJBdLOYPALCE2+3WvHnzdOedd6qpqUnxeFxXXnmlOjo6NHnyZD333HMqLy/vd84dd9yhb33rW3r77bc1adIk/d///V/yypnjxo3TAw88oNtvv1233nqrLrnkEn3ve9/Tz3/+c0lSIBDQli1b9Nhjj6mtrU0jRozQNddco6997WuSpEsuuUTLly/XJz/5Se3fv1+PPPKI5syZ85HPY+TIkXrppZf0/e9/XxdccIHC4bBGjx6tCy+8UE6nU6Wlpfrb3/6mxYsXq729XaNHj9bdd9+tT3/609q9e/eH1gQAAAAguxzmgwu6AAAAAAAAAAWIqZcAAAAAAACACMoAAAAAAAAASQRlAAAAAAAAgCSCMgAAAAAAAEASQRkAAAAAAAAgiaAMAAAAAAAAkERQBgAAAAAAAEgiKAMAAAAAAAAkEZQBAAAAAAAAkgjKAAAAAAAAAEkEZQAAAAAAAIAkgjIAAAAAAABAkvT/A+o/F2p04crFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_bicrit_metric(results, select=slice(None, None, 2))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/fedot_ind/core/metrics/interval_metrics.py b/fedot_ind/core/metrics/interval_metrics.py index f4a5f6544..d3436cdbb 100644 --- a/fedot_ind/core/metrics/interval_metrics.py +++ b/fedot_ind/core/metrics/interval_metrics.py @@ -134,5 +134,4 @@ def nab(boundaries, predictions, mode='standard', custom_coefs=None): score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], coefs) return score - - + \ No newline at end of file diff --git a/fedot_ind/core/metrics/metrics_implementation.py b/fedot_ind/core/metrics/metrics_implementation.py index 8803f18dd..dac5e888d 100644 --- a/fedot_ind/core/metrics/metrics_implementation.py +++ b/fedot_ind/core/metrics/metrics_implementation.py @@ -222,7 +222,7 @@ def smape(a, f, _=None): (np.abs(a) + np.abs(f)) * 100) def rmse(y_true, y_pred): - return np.sqrt(mean_squared_error(y_true, y_pred)) + return mean_squared_error(y_true, y_pred, squared=False) def mape(A, F): @@ -370,15 +370,22 @@ def metric(self) -> float: if len(self.predicted_labels.shape) == 1: self.predicted_labels = self.predicted_labels[None, ...] self.predicted_probs = self.predicted_probs[None, ...] - + print(f''' + target shape {self.target.shape} + prediction {self.predicted_labels.shape} + predicted_probs (scores) {self.predicted_probs.shape} + ''') n_metrics = len(self.metric_list) + (self.mode == 'robust') n_est = self.predicted_labels.shape[0] result = np.zeros((n_est, n_metrics)) + print(result.shape) if self.mode == 'robust': mask = self.predicted_probs >= 0 + print('mask', mask.shape) if not mask.any(): return result robustness = mask.sum(-1) / self.predicted_probs.shape[-1] + print('rob', robustness.shape) result[:, 0] = robustness.flatten() else: mask = np.ones_like(self.predicted_probs, dtype=bool) @@ -401,6 +408,7 @@ def metric(self) -> float: self.weights /= self.weights.sum() result = result @ self.weights.T + result[np.isnan(result)] = self.default_value if not self.reduce: return pd.DataFrame(result, columns=self.columns) else: diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 9c8e12cb8..83c3a5bba 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -1,26 +1,40 @@ from typing import Optional, List -from fedot_ind.core.architecture.settings.computational import backend_methods as np +from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation +from fedot.core.operations.operation_parameters import OperationParameters from sklearn.preprocessing import StandardScaler from sklearn.base import ClassifierMixin, BaseEstimator from sktime.classification.dictionary_based import WEASEL -from fedot.core.operations.operation_parameters import OperationParameters +from fedot_ind.core.architecture.settings.computational import backend_methods as np + +class EarlyTSClassifier(ClassifierMixin, BaseEstimator): + """ + Base class for Early Time Series Classification models + which implement prefix-wise predictions via traiing multiple slave estimators. -class BaseETC(ClassifierMixin, BaseEstimator): - def __init__(self, params: Optional[OperationParameters] = None): - if params is None: - params = {} + Args: + ``interval_percentage (float in (1, 100])``: define how much points should be between prediction points. + ``consecutive_predictions (int)``: how many last subsequent estimators should classify object equally. + ``accuracy_importance (float in [0, 1])``: trade-off coefficient between earliness and accuracy. + ``prediction_mode (str in ['last_available', 'best_by_metrics_mean', 'all'])``: + - if 'last_available', returns the latest estimator prediction allowed by prefix length; + - if 'best_by_metrics_mean', returns the best of estimators estimated + with weighted average of accuracy and earliness + - if 'all', returns all estiamtors predictions + ``transform_score (bool)``: whether or not to scale scores to [-1, 1] interval + ``min_ts_step (int)``: minimal difference between to subsequent prefix' lengths + """ + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__() self.interval_percentage = params.get('interval_percentage', 10) self.consecutive_predictions = params.get('consecutive_predictions', 1) self.accuracy_importance = params.get('accuracy_importance', 1.) - - self.prediction_mode = params.get('prediction_mode', 'last_available') - self.transform_score = params.get('transform_score', True) self.min_ts_length = params.get('min_ts_step', 3) self.random_state = params.get('random_state', None) + + self.prediction_mode = params.get('prediction_mode', 'last_available') + self.transform_score = params.get('transform_score', True) self.weasel_params = {} - assert self.consecutive_predictions < self.interval_percentage, 'Not enough checkpoints for prediction proof' def _init_model(self, X, y): max_data_length = X.shape[-1] @@ -69,7 +83,7 @@ def _compute_prediction_points(self, n_idx): def _select_estimators(self, X, training=False): offset = 0 - if not training and self.prediction_mode == 'best_by_harmonic_mean': + if not training and self.prediction_mode == 'best_by_metrics_mean': estimator_indices = [self._chosen_estimator_idx] elif not training and self.prediction_mode == 'last_available': last_idx, offset = self._get_applicable_index(X.shape[-1] - 1) @@ -100,6 +114,13 @@ def _consecutive_count(self, predicted_labels: List[np.array]): return consecutive_labels # prediction_points x n_instances def predict_proba(self, *args): + """ + Args: + X (np.array): input features + Returns: + predictions as a numpy array of shape (2, n_selected_estimators, n_instances, n_classes) + where first subarray stands for probas, and second for scores + """ predicted_probas, scores, *_ = args if self.transform_score: scores = self._transform_score(scores) @@ -110,6 +131,13 @@ def predict_proba(self, *args): return prediction def predict(self, X): + """ + Args: + X (np.array): input features + Returns: + predictions as a numpy array of shape (2, n_selected_estimators, n_instances) + where first subarray stands for labels, and second for scores + """ prediction = self.predict_proba(X) labels = prediction[0:1].argmax(-1) scores = prediction[1:2, ..., 0] diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index b83cc3254..64bb3f8bd 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -2,13 +2,18 @@ from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from fedot_ind.core.models.early_tc.base_early_tc import EarlyTSClassifier from sklearn.metrics import confusion_matrix from sklearn.model_selection import cross_val_predict -class ECEC(BaseETC): - def __init__(self, params: Optional[OperationParameters] = None): +class ECEC(EarlyTSClassifier): + """ + The Effective Confidence-based Early Classification algorithm + from J. Lv, X. Hu, L. Li, and P.-P. Li, “An effective confidence-based early classification + of time series,” IEEE Access, vol. 7, pp. 96 113–96 124, 2019 + """ + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.__cv = 5 @@ -53,6 +58,7 @@ def _score(self, y, y_pred, alpha): for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred + accuracy_for_candidate[np.isnan(accuracy_for_candidate)] = 0 cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) self._chosen_estimator_idx = np.argmin(cfs.mean(0)) return candidates[np.argmin(cfs, axis=0)] # n_pred diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index 481cad97b..675f87362 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -2,18 +2,22 @@ from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from fedot_ind.core.models.early_tc.base_early_tc import EarlyTSClassifier from sklearn.cluster import KMeans from sklearn.metrics import confusion_matrix from sklearn.model_selection import cross_val_predict -class EconomyK(BaseETC): - def __init__(self, params: Optional[OperationParameters] = None): - if params is None: - params = {} +class EconomyK(EarlyTSClassifier): + """ + Model described in + A. Dachraoui, A. Bondu, and A. Cornu´ejols, “Early classification of time series as a non myopic sequential decision + making problem,” in the European Conf. on Machine Learning and Knowledge Discovery in Databases, ser. LNCS, vol. + 9284. Springer, 2015, pp. 433–447. + """ + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.prediction_mode = params.get('prediction_mode', 'last_available') - self.lambda_ = params.get('lambda', 1.) + self.lambda_ = params.get('lambda_', 1.) self._cluster_factor = params.get('cluster_factor' , 1) self._random_state = 2104 self.__cv = 5 @@ -81,12 +85,11 @@ def predict_proba(self, X): return super().predict_proba(probas, times) def _transform_score(self, time): - idx = self._estimator_for_predict[-1] - scores = (1 - (time - self.prediction_idx[idx]) / self.prediction_idx[-1]) # [1 / n; 1 ] - 1 / n) * n /(n - 1) * 2 - 1 - n = self.n_pred - scores -= 1 / n - scores *= n / (n - 1) * 2 + scores = 1 - (time - self.prediction_idx[self._estimator_for_predict, None]) / (self.prediction_idx[-1] - self._estimator_for_predict)[:, None] + assert ((0 <= scores) & (scores <= 1)).all() + scores *= 2 scores -= 1 + scores[np.isnan(scores)] = 0 return scores diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index b72a927f1..946d4a2c2 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -2,12 +2,14 @@ from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from fedot_ind.core.models.early_tc.base_early_tc import EarlyTSClassifier -class ProbabilityThresholdClassifier(BaseETC): - def __init__(self, params: Optional[OperationParameters] = None): - if params is None: - params = {} +class ProbabilityThresholdClassifier(EarlyTSClassifier): + f""" + Two-tier Early time-series classification model + uniting consecutive prediction comparison and thresholding by predicted probability. + """ + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.probability_threshold = params.get('probability_threshold', None) @@ -15,11 +17,18 @@ def _init_model(self, X, y): super()._init_model(X, y) if self.probability_threshold is None: self.probability_threshold = 1 / len(self.classes_[0]) + eps = 1e-7 + if self.probability_threshold == 1: + self.probability_threshold -= eps + if self.probability_threshold == 0: + self.probability_threshold += eps def predict_proba(self, X): _, predicted_probas, non_acceptance = self._predict(X, training=False) - predicted_probas[non_acceptance] = 0 scores = predicted_probas.max(-1) + scores[~non_acceptance & (scores < self.probability_threshold)] = self.probability_threshold + \ + (1 - self.probability_threshold) * self.consecutive_predictions / self.n_pred + predicted_probas[non_acceptance] = 0 return super().predict_proba(predicted_probas, scores) def _predict(self, X, training=True): diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 0350d0886..8c44c0c2e 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -2,13 +2,20 @@ from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.early_tc.base_early_tc import BaseETC +from fedot_ind.core.models.early_tc.base_early_tc import EarlyTSClassifier from sklearn.model_selection import GridSearchCV from sklearn.svm import OneClassSVM -class TEASER(BaseETC): - def __init__(self, params: Optional[OperationParameters] = None): +class TEASER(EarlyTSClassifier): + """ + Two-tier Early and Accurate Series classifiER + + from “TEASER: early and accurate time series classification,” + Data Min. Knowl. Discov., vol. 34, no. 5, pp. 1336–1362, 2020 + """ + + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self._oc_svm_params = (100., 10., 5., 2.5, 1.5, 1., 0.5, 0.25, 0.1) @@ -56,14 +63,27 @@ def _predict(self, X, training=False): final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # mark as accepted final_verdicts[i] = final_verdict - non_acceptance[non_acceptance & (final_verdict > 0)] = False + (non_acceptance[non_acceptance & (final_verdict > 0)], + final_verdicts[non_acceptance], + final_verdicts[~non_acceptance & (final_verdicts < 0)] + ) = False, -1, self.consecutive_predictions / self.n_pred return predicted_labels, predicted_probas, non_acceptance, final_verdicts def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) - predicted_probas[non_acceptance] = final_verdicts[non_acceptance, None] + predicted_probas[non_acceptance] = 0 #final_verdicts[non_acceptance, None] return super().predict_proba(predicted_probas, final_verdicts) + def predict(self, X): + prediction = self.predict_proba(X) + labels = prediction[0:1].argmax(-1) + scores = prediction[1:2, ..., 0] + labels[scores < 0] = -1 + prediction = np.stack([labels, scores], 0) + if prediction.shape[1] == 1: + prediction = prediction.squeeze(1) + return prediction + def _score(self, X, y, accuracy_importance=None): scores = super()._score(X, y, accuracy_importance) self._chosen_estimator_idx = np.argmax(scores) diff --git a/fedot_ind/core/models/nn/network_impl/base_nn_model.py b/fedot_ind/core/models/nn/network_impl/base_nn_model.py index f285853d0..7c20f5f0b 100644 --- a/fedot_ind/core/models/nn/network_impl/base_nn_model.py +++ b/fedot_ind/core/models/nn/network_impl/base_nn_model.py @@ -122,8 +122,11 @@ def _train_one_batch(self, batch, optimizer, loss_fn): optimizer.step() training_loss = loss.data.item() * inputs.size(0) total = targets.size(0) - correct = (torch.argmax(output, 1) == - torch.argmax(targets, 1)).sum().item() + if targets.ndim == 2: + targets = targets.argmax(-1) + if output.ndim == 2: + output = output.argmax(-1) + correct = (output == targets).sum().item() return training_loss, total, correct def _eval_one_batch(self, batch, loss_fn): @@ -132,8 +135,11 @@ def _eval_one_batch(self, batch, loss_fn): loss = loss_fn(output, targets.float()) valid_loss = loss.data.item() * inputs.size(0) total = targets.size(0) - correct = (torch.argmax(output, 1) == - torch.argmax(targets, 1)).sum().item() + if targets.ndim == 2: + targets = targets.argmax(-1) + if output.ndim == 2: + output = output.argmax(-1) + correct = (output == targets).sum().item() return valid_loss, total, correct def _run_one_epoch(self, train_loader, val_loader, @@ -188,7 +194,6 @@ def _train_loop(self, train_loader, val_loader, loss_fn, optimizer): best_val_loss = float('inf') val_interval = self.get_validation_frequency( self.epochs, self.learning_rate) - loss_prefix = 'RMSE' if self.is_regression_task else 'Accuracy' for epoch in range(1, self.epochs + 1): best_model, best_val_loss = self._run_one_epoch( train_loader, val_loader, diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 604f28660..33cf7e23e 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -1,20 +1,14 @@ -import copy from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel -from typing import Optional, Callable, Any, List, Union +from typing import Optional from fedot.core.operations.operation_parameters import OperationParameters -from fedot.core.data.data import InputData, OutputData -from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY, MULTI_CLASS_CROSS_ENTROPY, RMSE +from fedot.core.data.data import InputData +from fedot_ind.core.repository.constanst_repository import CROSS_ENTROPY import torch.optim as optim -import torch.optim.lr_scheduler as lr_scheduler import torch.nn as nn import torch.nn.functional as F import torch -from tqdm import tqdm from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array -import pandas as pd -from fedot_ind.core.models.nn.network_modules.layers.special import adjust_learning_rate, EarlyStopping -import torch.utils.data as data class SqueezeExciteBlock(nn.Module): def __init__(self, input_channels, filters, reduce=4): @@ -44,7 +38,7 @@ def __init__(self, input_size, input_channels, self.lstm = nn.LSTM(input_size, inner_size, num_layers, batch_first=True, dropout=dropout) - squeeze_excite_size = input_size #if not interval else interval + squeeze_excite_size = input_size self.conv_branch = nn.Sequential( nn.Conv1d(input_channels, inner_channels, padding='same', @@ -82,10 +76,15 @@ def forward(self, x, hidden_state=None, return_hidden=False): class MLSTM(BaseNeuralModel): - def __init__(self, params: Optional[OperationParameters] = None): - if params is None: - params = {} - super().__init__() + f""" + The Multivariate Long Short Term Memory Fully Convolutional Network (MLSTM) + from F. Karim, S. Majumdar, H. Darabi, and S. Harford, “Multivariate LSTM-FCNs for time series classification,” Neural + Networks, vol. 116, pp. 237–245, 2019. + + {BaseNeuralModel.__doc__} + """ + def __init__(self, params: Optional[OperationParameters] = {}): + super().__init__(params) self.dropout = params.get('dropout', 0.25) self.hidden_size = params.get('hidden_size', 64) self.hidden_channels = params.get('hidden_channels', 32) @@ -101,7 +100,7 @@ def __repr__(self): def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) prediction_idx = np.arange(interval_length - 1, n_idx, interval_length) - self.earliness = 1 - prediction_idx / n_idx # /n_idx because else the last hm score is always 0 + self.earliness = 1 - prediction_idx / n_idx return prediction_idx, interval_length def _init_model(self, ts: InputData): @@ -198,6 +197,7 @@ def _predict_model(self, x_test: InputData, output_mode: str = 'default'): pred = self._moving_window_output(torch.tensor(x_test).float()) else: raise ValueError('Unknown prediction mode') + pred = pred.detach() return self._convert_predict(pred, output_mode) def _padding(self, ts: np.array): diff --git a/fedot_ind/core/models/nn/network_impl/transformer.py b/fedot_ind/core/models/nn/network_impl/transformer.py index a8d12fdd8..07d049a33 100644 --- a/fedot_ind/core/models/nn/network_impl/transformer.py +++ b/fedot_ind/core/models/nn/network_impl/transformer.py @@ -72,6 +72,8 @@ class TransformerModel(BaseNeuralModel): self.batch_size: int, the batch size. """ + def __repr__(self): + return 'Transformer' def __init__(self, params: Optional[OperationParameters] = None): super().__init__(params) diff --git a/fedot_ind/core/models/quantile/quantile_extractor.py b/fedot_ind/core/models/quantile/quantile_extractor.py index 17a44e0cd..35fd99857 100644 --- a/fedot_ind/core/models/quantile/quantile_extractor.py +++ b/fedot_ind/core/models/quantile/quantile_extractor.py @@ -87,4 +87,4 @@ def generate_features_from_ts(self, aggregation_df = self._get_feature_matrix( self.extract_stats_features, ts) - return aggregation_df + return aggregation_df \ No newline at end of file diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index b05cb3c95..a7bddecf5 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -372,9 +372,9 @@ }, "mlstm_model": { "meta": "fedot_NN_classification", - "presets": ["ts"], + "presets": [], "tags": [], - "input_type": "[DataTypesEnum.table]" + "input_type": "[DataTypesEnum.multi_ts, DataTypesEnum.ts]" }, "xcm_model": { "meta": "fedot_NN_classification", diff --git a/tests/unit/core/models/model_impl/test_mlstm.py b/tests/unit/core/models/model_impl/test_mlstm.py new file mode 100644 index 000000000..d245e6dbd --- /dev/null +++ b/tests/unit/core/models/model_impl/test_mlstm.py @@ -0,0 +1,37 @@ +import pytest + +from fedot.core.data.data import InputData +from fedot_ind.core.models.nn.network_impl.mlstm import MLSTM +from fedot.core.pipelines.pipeline_builder import PipelineBuilder +from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum +import numpy as np + + +_N_FEATURES = 73 +_N_SAMPLES = 133 +_N_CLASSES = 3 +_INTERVAL_LENGTH = 7 + +@pytest.fixture +def data(): + X, y = np.random.randn(_N_SAMPLES, _N_FEATURES), np.random.randint(0, _N_CLASSES, size=_N_SAMPLES) + return InputData(idx=np.arange(0, len(X)), + features=X, + target=y, + task=Task(TaskTypesEnum.classification), + data_type=DataTypesEnum.table) + +@pytest.mark.parametrize('fitting_mode', ['zero_padding', 'moving_window']) +def test_mlstm_by_mode(data, fitting_mode): + with IndustrialModels(): + ppl = PipelineBuilder().add_node('mlstm_model', + params={'epochs': 5, 'fitting_mode': fitting_mode}).build() + ppl.fit(data) + pred = ppl.predict(data).predict + assert not np.isnan(pred).any() + + + + diff --git a/tests/unit/core/models/test_etc.py b/tests/unit/core/models/test_etc.py new file mode 100644 index 000000000..b1ff2e3de --- /dev/null +++ b/tests/unit/core/models/test_etc.py @@ -0,0 +1,98 @@ +import pytest + +from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier +from fedot_ind.core.models.early_tc.ecec import ECEC +from fedot_ind.core.models.early_tc.economy_k import EconomyK +from fedot_ind.core.models.early_tc.teaser import TEASER +import numpy as np + +_N_FEATURES = 73 +_N_SAMPLES = 133 +_N_CLASSES = 3 +_INTERVAL_LENGTH = 7 +MODELS = { + 'economy_k': EconomyK, + 'ecec': ECEC, + 'teaser': TEASER, + 'proba_threshold_etc': ProbabilityThresholdClassifier +} + +@pytest.fixture +def data(): + X, y = np.random.randn(_N_SAMPLES, _N_FEATURES), np.random.randint(0, _N_CLASSES, size=_N_SAMPLES) + return X, y + +def test_compute_prediction_points(data): + X, y = data + pthr = ProbabilityThresholdClassifier({'interval_percentage': 10}) + pthr._init_model(X, y) + prediction_idx = pthr.prediction_idx + assert len(prediction_idx) == _N_FEATURES // _INTERVAL_LENGTH, 'wrong number of points' + +@pytest.mark.parametrize('training,prediction_mode,expected_num', [ + (True, 'last_available', None), + (False, 'last_available', 1), + (False, 'best_by_metrics_mean', 1), + (False, 'all', None), + +]) +def test_select_estimators(data, training, prediction_mode, expected_num): + X, y = data + pthr = ProbabilityThresholdClassifier({'prediction_mode': prediction_mode}) + pthr._init_model(X, y) + if expected_num is None: + expected_num = pthr.n_pred + idx, _ = pthr._select_estimators(X, training) + assert len(idx) == expected_num, f'selection went wrong: got {len(idx)}, expected {expected_num}' + +@pytest.mark.parametrize('model', + ['proba_threshold_etc', 'ecec', 'economy_k', 'teaser']) +def test_fit_predict(data, model): + X, y = data + model = MODELS[model]({'prediction_mode': 'all'}) + model.fit(X, y) + prediction = model.predict_proba(X) + ind = model._select_estimators(X, training=False)[0] + assert (not np.isnan(prediction).any() and + (prediction.shape == (2, len(ind), len(y), _N_CLASSES))), 'Prediction went wrong' + +# ECEC TESTS +def test_select_thrs(): + model = ECEC() + selection = model._select_thrs(np.random.randn(40)) + assert len(selection), 'No candidates were chosen!' + +# Proba Thr +def test_consecutive(data): + X, y = data + pthr = ProbabilityThresholdClassifier({'prediction_mode': 'last_available', + 'consecutive_predictions': 1}) + pthr.fit(X, y) + prediction, scores = pthr.predict(X) + assert -1 not in prediction, 'Setting uncertainty while it is impossible' + +# Economy K +def test_specific_economyk(data): + X, y = data + model = EconomyK() + model.fit(X, y) + assert not np.isnan( + model._EconomyK__cluster_probas(X, model._clusterizer.cluster_centers_) + ).any(), '__cluster_probas doesn\'t function correctly' + + i = model.n_pred - 1 + times = model._get_prediction_time(X, model._clusterizer.cluster_centers_, i)[0] + assert not np.isnan(times).any() + assert ((model.prediction_idx[0] <= times) & (times <= model.prediction_idx[-1])).all(), \ + f'(_get_prediction_time) case of the last prediction point:' + \ + ' times cannot exceed the limits of time predictions.' + \ + f'current lies in [{times.min()}, {times.max()}]' + +# TEASER +def test_form_X_oc(): + probas = np.random.randint(0, 10, size=(_N_SAMPLES, _N_CLASSES)).astype(float) + probas /= probas.sum(1, keepdims=True) + 1e-5 + model = TEASER() + X_oc = model._form_X_oc(probas) + assert X_oc.shape == (_N_SAMPLES, _N_CLASSES + 1), 'Wrong number of features' + assert ((0 <= X_oc) & (X_oc <= 1)).all(), 'In original paper outputs lie in [0, 1]' From 3d9022c9da4fcba5d1006b16982ca6fd1a6455c6 Mon Sep 17 00:00:00 2001 From: autopep8 bot Date: Tue, 23 Jul 2024 09:41:01 +0000 Subject: [PATCH 39/43] Automated autopep8 fixes --- .../core/models/early_tc/base_early_tc.py | 10 +++--- fedot_ind/core/models/early_tc/ecec.py | 3 +- fedot_ind/core/models/early_tc/economy_k.py | 12 ++++--- .../core/models/early_tc/prob_threshold.py | 8 +++-- fedot_ind/core/models/early_tc/teaser.py | 14 ++++---- .../core/models/nn/network_impl/mlstm.py | 12 ++++--- .../models/nn/network_impl/transformer.py | 1 + .../models/quantile/quantile_extractor.py | 2 +- .../unit/core/models/model_impl/test_mlstm.py | 17 ++++------ tests/unit/core/models/test_etc.py | 34 +++++++++++++------ 10 files changed, 65 insertions(+), 48 deletions(-) diff --git a/fedot_ind/core/models/early_tc/base_early_tc.py b/fedot_ind/core/models/early_tc/base_early_tc.py index 09f24c462..bb5c9d89d 100644 --- a/fedot_ind/core/models/early_tc/base_early_tc.py +++ b/fedot_ind/core/models/early_tc/base_early_tc.py @@ -1,5 +1,4 @@ from typing import Optional, List -from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation from fedot.core.operations.operation_parameters import OperationParameters from sklearn.preprocessing import StandardScaler from sklearn.base import ClassifierMixin, BaseEstimator @@ -9,7 +8,7 @@ class EarlyTSClassifier(ClassifierMixin, BaseEstimator): """ - Base class for Early Time Series Classification models + Base class for Early Time Series Classification models which implement prefix-wise predictions via traiing multiple slave estimators. Args: @@ -24,14 +23,15 @@ class EarlyTSClassifier(ClassifierMixin, BaseEstimator): ``transform_score (bool)``: whether or not to scale scores to [-1, 1] interval ``min_ts_step (int)``: minimal difference between to subsequent prefix' lengths """ - def __init__(self, params: Optional[OperationParameters] = {}): + + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__() self.interval_percentage = params.get('interval_percentage', 10) self.consecutive_predictions = params.get('consecutive_predictions', 1) self.accuracy_importance = params.get('accuracy_importance', 1.) self.min_ts_length = params.get('min_ts_step', 3) self.random_state = params.get('random_state', None) - + self.prediction_mode = params.get('prediction_mode', 'last_available') self.transform_score = params.get('transform_score', True) self.weasel_params = {} @@ -123,7 +123,7 @@ def predict_proba(self, *args): predictions as a numpy array of shape (2, n_selected_estimators, n_instances, n_classes) where first subarray stands for probas, and second for scores """ - predicted_probas, scores, *_ = args + predicted_probas, scores, *_ = args if self.transform_score: scores = self._transform_score(scores) scores = np.tile(scores[..., None], (1, 1, self.n_classes)) diff --git a/fedot_ind/core/models/early_tc/ecec.py b/fedot_ind/core/models/early_tc/ecec.py index b8a743f25..4137faa70 100644 --- a/fedot_ind/core/models/early_tc/ecec.py +++ b/fedot_ind/core/models/early_tc/ecec.py @@ -13,6 +13,7 @@ class ECEC(EarlyTSClassifier): from J. Lv, X. Hu, L. Li, and P.-P. Li, “An effective confidence-based early classification of time series,” IEEE Access, vol. 7, pp. 96 113–96 124, 2019 """ + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.__cv = 5 @@ -57,7 +58,7 @@ def _score(self, y, y_pred, alpha): cfs = np.zeros((len(candidates), n)) for i, candidate in enumerate(candidates): mask = confidences >= candidate # n_pred x n_inst - accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred + accuracy_for_candidate = (matches * mask).sum(1) / mask.sum(1) # n_pred accuracy_for_candidate[np.isnan(accuracy_for_candidate)] = 0 cfs[i] = self.cost_func(self.earliness, accuracy_for_candidate, alpha) self._chosen_estimator_idx = np.argmin(cfs.mean(0)) diff --git a/fedot_ind/core/models/early_tc/economy_k.py b/fedot_ind/core/models/early_tc/economy_k.py index e41d393cf..a67d3b945 100644 --- a/fedot_ind/core/models/early_tc/economy_k.py +++ b/fedot_ind/core/models/early_tc/economy_k.py @@ -10,16 +10,17 @@ class EconomyK(EarlyTSClassifier): """ - Model described in - A. Dachraoui, A. Bondu, and A. Cornu´ejols, “Early classification of time series as a non myopic sequential decision + Model described in + A. Dachraoui, A. Bondu, and A. Cornu´ejols, “Early classification of time series as a non myopic sequential decision making problem,” in the European Conf. on Machine Learning and Knowledge Discovery in Databases, ser. LNCS, vol. 9284. Springer, 2015, pp. 433–447. """ - def __init__(self, params: Optional[OperationParameters] = {}): + + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.prediction_mode = params.get('prediction_mode', 'last_available') self.lambda_ = params.get('lambda_', 1.) - self._cluster_factor = params.get('cluster_factor' , 1) + self._cluster_factor = params.get('cluster_factor', 1) self._random_state = 2104 self.__cv = 5 @@ -88,7 +89,8 @@ def predict_proba(self, X): return super().predict_proba(probas, times) def _transform_score(self, time): - scores = 1 - (time - self.prediction_idx[self._estimator_for_predict, None]) / (self.prediction_idx[-1] - self._estimator_for_predict)[:, None] + scores = 1 - (time - self.prediction_idx[self._estimator_for_predict, None] + ) / (self.prediction_idx[-1] - self._estimator_for_predict)[:, None] assert ((0 <= scores) & (scores <= 1)).all() scores *= 2 scores -= 1 diff --git a/fedot_ind/core/models/early_tc/prob_threshold.py b/fedot_ind/core/models/early_tc/prob_threshold.py index 537bd963e..bbf77b49b 100644 --- a/fedot_ind/core/models/early_tc/prob_threshold.py +++ b/fedot_ind/core/models/early_tc/prob_threshold.py @@ -4,12 +4,14 @@ from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.early_tc.base_early_tc import EarlyTSClassifier + class ProbabilityThresholdClassifier(EarlyTSClassifier): f""" - Two-tier Early time-series classification model + Two-tier Early time-series classification model uniting consecutive prediction comparison and thresholding by predicted probability. """ - def __init__(self, params: Optional[OperationParameters] = {}): + + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.probability_threshold = params.get('probability_threshold', None) @@ -22,7 +24,7 @@ def _init_model(self, X, y): self.probability_threshold -= eps if self.probability_threshold == 0: self.probability_threshold += eps - + def predict_proba(self, X): _, predicted_probas, non_acceptance = self._predict(X, training=False) scores = predicted_probas.max(-1) diff --git a/fedot_ind/core/models/early_tc/teaser.py b/fedot_ind/core/models/early_tc/teaser.py index 5dd4ff437..475e8f33f 100644 --- a/fedot_ind/core/models/early_tc/teaser.py +++ b/fedot_ind/core/models/early_tc/teaser.py @@ -14,8 +14,8 @@ class TEASER(EarlyTSClassifier): from “TEASER: early and accurate time series classification,” Data Min. Knowl. Discov., vol. 34, no. 5, pp. 1336–1362, 2020 """ - - def __init__(self, params: Optional[OperationParameters] = {}): + + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self._oc_svm_params = (100., 10., 5., 2.5, 1.5, 1., 0.5, 0.25, 0.1) @@ -62,16 +62,16 @@ def _predict(self, X, training=False): # if they are not outliers final_verdict = self.oc_estimators[estimator_indices[i]].decision_function(X_to_ith) # mark as accepted - final_verdicts[i] = final_verdict - (non_acceptance[non_acceptance & (final_verdict > 0)], - final_verdicts[non_acceptance], + final_verdicts[i] = final_verdict + (non_acceptance[non_acceptance & (final_verdict > 0)], + final_verdicts[non_acceptance], final_verdicts[~non_acceptance & (final_verdicts < 0)] - ) = False, -1, self.consecutive_predictions / self.n_pred + ) = False, -1, self.consecutive_predictions / self.n_pred return predicted_labels, predicted_probas, non_acceptance, final_verdicts def predict_proba(self, X): _, predicted_probas, non_acceptance, final_verdicts = self._predict(X) - predicted_probas[non_acceptance] = 0 #final_verdicts[non_acceptance, None] + predicted_probas[non_acceptance] = 0 # final_verdicts[non_acceptance, None] return super().predict_proba(predicted_probas, final_verdicts) def _score(self, X, y, accuracy_importance=None): diff --git a/fedot_ind/core/models/nn/network_impl/mlstm.py b/fedot_ind/core/models/nn/network_impl/mlstm.py index 2d9d99396..bc6b1e825 100644 --- a/fedot_ind/core/models/nn/network_impl/mlstm.py +++ b/fedot_ind/core/models/nn/network_impl/mlstm.py @@ -10,6 +10,7 @@ from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.architecture.abstraction.decorators import convert_to_3d_torch_array + class SqueezeExciteBlock(nn.Module): def __init__(self, input_channels, filters, reduce=4): super().__init__() @@ -37,9 +38,9 @@ def __init__(self, input_size, input_channels, super().__init__() self.proj = nn.Linear(input_size * inner_channels + input_channels * inner_size, output_size) self.lstm = nn.LSTM(input_size, inner_size, num_layers, - batch_first=True, dropout=dropout) - - squeeze_excite_size = input_size + batch_first=True, dropout=dropout) + + squeeze_excite_size = input_size self.conv_branch = nn.Sequential( nn.Conv1d(input_channels, inner_channels, padding='same', @@ -84,7 +85,8 @@ class MLSTM(BaseNeuralModel): {BaseNeuralModel.__doc__} """ - def __init__(self, params: Optional[OperationParameters] = {}): + + def __init__(self, params: Optional[OperationParameters] = {}): super().__init__(params) self.dropout = params.get('dropout', 0.25) self.hidden_size = params.get('hidden_size', 64) @@ -101,7 +103,7 @@ def __repr__(self): def _compute_prediction_points(self, n_idx): interval_length = max(int(n_idx * self.interval_percentage / 100), self.min_ts_length) prediction_idx = np.arange(interval_length - 1, n_idx, interval_length) - self.earliness = 1 - prediction_idx / n_idx + self.earliness = 1 - prediction_idx / n_idx return prediction_idx, interval_length def _init_model(self, ts: InputData): diff --git a/fedot_ind/core/models/nn/network_impl/transformer.py b/fedot_ind/core/models/nn/network_impl/transformer.py index 07d049a33..11dd54b09 100644 --- a/fedot_ind/core/models/nn/network_impl/transformer.py +++ b/fedot_ind/core/models/nn/network_impl/transformer.py @@ -72,6 +72,7 @@ class TransformerModel(BaseNeuralModel): self.batch_size: int, the batch size. """ + def __repr__(self): return 'Transformer' diff --git a/fedot_ind/core/models/quantile/quantile_extractor.py b/fedot_ind/core/models/quantile/quantile_extractor.py index 0d92127b6..596f165d6 100644 --- a/fedot_ind/core/models/quantile/quantile_extractor.py +++ b/fedot_ind/core/models/quantile/quantile_extractor.py @@ -90,4 +90,4 @@ def generate_features_from_ts(self, aggregation_df = self._get_feature_matrix( self.extract_stats_features, ts) - return aggregation_df \ No newline at end of file + return aggregation_df diff --git a/tests/unit/core/models/model_impl/test_mlstm.py b/tests/unit/core/models/model_impl/test_mlstm.py index d245e6dbd..be7fcc0ec 100644 --- a/tests/unit/core/models/model_impl/test_mlstm.py +++ b/tests/unit/core/models/model_impl/test_mlstm.py @@ -1,7 +1,6 @@ import pytest from fedot.core.data.data import InputData -from fedot_ind.core.models.nn.network_impl.mlstm import MLSTM from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels from fedot.core.repository.dataset_types import DataTypesEnum @@ -14,24 +13,22 @@ _N_CLASSES = 3 _INTERVAL_LENGTH = 7 + @pytest.fixture def data(): X, y = np.random.randn(_N_SAMPLES, _N_FEATURES), np.random.randint(0, _N_CLASSES, size=_N_SAMPLES) return InputData(idx=np.arange(0, len(X)), - features=X, - target=y, - task=Task(TaskTypesEnum.classification), - data_type=DataTypesEnum.table) + features=X, + target=y, + task=Task(TaskTypesEnum.classification), + data_type=DataTypesEnum.table) + @pytest.mark.parametrize('fitting_mode', ['zero_padding', 'moving_window']) def test_mlstm_by_mode(data, fitting_mode): with IndustrialModels(): - ppl = PipelineBuilder().add_node('mlstm_model', + ppl = PipelineBuilder().add_node('mlstm_model', params={'epochs': 5, 'fitting_mode': fitting_mode}).build() ppl.fit(data) pred = ppl.predict(data).predict assert not np.isnan(pred).any() - - - - diff --git a/tests/unit/core/models/test_etc.py b/tests/unit/core/models/test_etc.py index b1ff2e3de..e8b0647eb 100644 --- a/tests/unit/core/models/test_etc.py +++ b/tests/unit/core/models/test_etc.py @@ -17,11 +17,13 @@ 'proba_threshold_etc': ProbabilityThresholdClassifier } + @pytest.fixture def data(): X, y = np.random.randn(_N_SAMPLES, _N_FEATURES), np.random.randint(0, _N_CLASSES, size=_N_SAMPLES) return X, y + def test_compute_prediction_points(data): X, y = data pthr = ProbabilityThresholdClassifier({'interval_percentage': 10}) @@ -29,12 +31,13 @@ def test_compute_prediction_points(data): prediction_idx = pthr.prediction_idx assert len(prediction_idx) == _N_FEATURES // _INTERVAL_LENGTH, 'wrong number of points' + @pytest.mark.parametrize('training,prediction_mode,expected_num', [ (True, 'last_available', None), - (False, 'last_available', 1), + (False, 'last_available', 1), (False, 'best_by_metrics_mean', 1), (False, 'all', None), - + ]) def test_select_estimators(data, training, prediction_mode, expected_num): X, y = data @@ -45,24 +48,29 @@ def test_select_estimators(data, training, prediction_mode, expected_num): idx, _ = pthr._select_estimators(X, training) assert len(idx) == expected_num, f'selection went wrong: got {len(idx)}, expected {expected_num}' -@pytest.mark.parametrize('model', - ['proba_threshold_etc', 'ecec', 'economy_k', 'teaser']) + +@pytest.mark.parametrize('model', + ['proba_threshold_etc', 'ecec', 'economy_k', 'teaser']) def test_fit_predict(data, model): X, y = data model = MODELS[model]({'prediction_mode': 'all'}) model.fit(X, y) prediction = model.predict_proba(X) ind = model._select_estimators(X, training=False)[0] - assert (not np.isnan(prediction).any() and + assert (not np.isnan(prediction).any() and (prediction.shape == (2, len(ind), len(y), _N_CLASSES))), 'Prediction went wrong' - + # ECEC TESTS + + def test_select_thrs(): model = ECEC() selection = model._select_thrs(np.random.randn(40)) assert len(selection), 'No candidates were chosen!' # Proba Thr + + def test_consecutive(data): X, y = data pthr = ProbabilityThresholdClassifier({'prediction_mode': 'last_available', @@ -72,23 +80,27 @@ def test_consecutive(data): assert -1 not in prediction, 'Setting uncertainty while it is impossible' # Economy K + + def test_specific_economyk(data): X, y = data model = EconomyK() model.fit(X, y) assert not np.isnan( model._EconomyK__cluster_probas(X, model._clusterizer.cluster_centers_) - ).any(), '__cluster_probas doesn\'t function correctly' - + ).any(), '__cluster_probas doesn\'t function correctly' + i = model.n_pred - 1 times = model._get_prediction_time(X, model._clusterizer.cluster_centers_, i)[0] assert not np.isnan(times).any() assert ((model.prediction_idx[0] <= times) & (times <= model.prediction_idx[-1])).all(), \ f'(_get_prediction_time) case of the last prediction point:' + \ - ' times cannot exceed the limits of time predictions.' + \ - f'current lies in [{times.min()}, {times.max()}]' - + ' times cannot exceed the limits of time predictions.' + \ + f'current lies in [{times.min()}, {times.max()}]' + # TEASER + + def test_form_X_oc(): probas = np.random.randint(0, 10, size=(_N_SAMPLES, _N_CLASSES)).astype(float) probas /= probas.sum(1, keepdims=True) + 1e-5 From e68331c4361adb119eb30f4955e39568f630d606 Mon Sep 17 00:00:00 2001 From: Leon_Strelkov <103892559+leostre@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:24:09 +0300 Subject: [PATCH 40/43] Delete tests/unit/core/models/test_teaser.py due to its inclusion into test_etc.py --- tests/unit/core/models/test_teaser.py | 36 --------------------------- 1 file changed, 36 deletions(-) delete mode 100644 tests/unit/core/models/test_teaser.py diff --git a/tests/unit/core/models/test_teaser.py b/tests/unit/core/models/test_teaser.py deleted file mode 100644 index 1cf847f67..000000000 --- a/tests/unit/core/models/test_teaser.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -import numpy as np -from fedot_ind.core.models.early_tc import teaser as TEASER - - -@pytest.fixture(scope='module') -def teaser(): - teaser = TEASER.TEASER({'interval_length': 10, 'prediction_mode': ''}) - return teaser - - -@pytest.fixture(scope='module') -def xy(): - return np.random.randn((2, 23)), np.random.randint(0, 2, size=(2, 1)) - - -def test_get_applicable_index(teaser): - teaser._init_model(23) - idx, offset = teaser._get_last_applicable_idx(100) - assert offset == 100 - 22, 'Wrong offset estimation when right edge' - assert idx == len(teaser.prediction_idx) - 1 - idx, offset = teaser._get_last_applicable_idx(12) - assert offset == 100 - teaser.prediction_idx[idx], 'Wrong offset estimation in the middle' - assert idx == len(teaser.prediction_idx) - 1 - - -def test_compute_prediction_points(teaser): - indices = teaser._compute_prediction_points(23) - assert 2 in indices - assert 22 in indices - assert 23 not in indices - -# def test_consecutive_count(teaser): -# pass - -# def test_score(teaser): From b50e8b133b07f498fd4784208d4fcf1340190c81 Mon Sep 17 00:00:00 2001 From: Leon_Strelkov <103892559+leostre@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:46:22 +0300 Subject: [PATCH 41/43] Delete fedot_ind/core/metrics/interval_metrics.py due to the fucntionality overlap --- fedot_ind/core/metrics/interval_metrics.py | 142 --------------------- 1 file changed, 142 deletions(-) delete mode 100644 fedot_ind/core/metrics/interval_metrics.py diff --git a/fedot_ind/core/metrics/interval_metrics.py b/fedot_ind/core/metrics/interval_metrics.py deleted file mode 100644 index f95586147..000000000 --- a/fedot_ind/core/metrics/interval_metrics.py +++ /dev/null @@ -1,142 +0,0 @@ -from sklearn.metrics import confusion_matrix -import numpy as np -import pandas as pd -from typing import Union, Literal - - -def conf_matrix(actual, predicted): - cm = confusion_matrix(actual, predicted) - return dict(TN=cm[0, 0], FP=cm[0, 1], FN=cm[1, 0], TP=[1, 1]) - - -def average_delay(boundaries, prediction, - point, - use_idx=True, - window_placement='lefter'): - cp_confusion = extract_cp_cm(boundaries, prediction, use_idx=use_idx, use_switch_point=False) - # statistics - statistics = { - 'anomalies_num': len(cp_confusion['TPs']) + len(cp_confusion['FPs']), - 'FP_num': len(cp_confusion['FPs']), - 'missed': len(cp_confusion['FNs']) - } - time_func = { - 'righter': lambda triplet: triplet[1] - triplet[0], - 'lefter': lambda triplet: triplet[2] - triplet[1], - 'central': lambda triplet: triplet[1] - triplet[0] - (triplet[2] - triplet[0]) / 2 - }[window_placement] - - detection_history = { - i: time_func(triplet) for i, triplet in cp_confusion['TPs'].items() - } - return detection_history, statistics - - -def tp_transform(tps): - return np.diff(tps[[1, 0]], axis=0) / np.diff(tps[[-1, 0]], axis=0) - - -def extract_cp_cm(boundaries: Union[np.array, pd.DataFrame], - prediction: pd.DataFrame, - use_switch_point: bool = True, # if first anomaly dot is considered as changepoint - use_idx: bool = False): - if isinstance(boundaries, pd.DataFrame): - boundaries = boundaries.values.T - anomaly_tsp = prediction[prediction == 1].sort_index().index - TPs, FNs, FPs = {}, [], [] - - if boundaries.shape[1]: - - FPs += [anomaly_tsp[anomaly_tsp < boundaries[0, 0]]] # left rest - for i, (b_low, b_up) in enumerate(boundaries): - all_tsp_in_window = prediction[b_low: b_up].index - anomaly_tsp_in_window = anomaly_tsp_in_window & anomaly_tsp - if not len(anomaly_tsp_in_window): # why not false positive? do we expect an anomaly to be in every interval? - FNs.append(i if use_idx else all_tsp_in_window) - TPs[i] = [b_low, - anomaly_tsp_in_window[int(use_switch_point)] if use_idx else anomaly_tsp_in_window, - b_up] - if not use_idx: - FNs.append(all_tsp_in_window - anomaly_tsp_in_window) - FPs.append(anomaly_tsp[anomaly_tsp > boundaries[-1, -1]]) # right rest - else: - FPs.append(anomaly_tsp) - - FPs = np.concatenate(FPs) - FNs = np.concatenate(FNs) - - return dict( - FP=FPs, - FN=FNs, - TP=np.stack(TPs) - ) - -# cognate of single_detecting_boundaries - - -def get_boundaries(idx, actual_timestamps, window_size: int = None, - window_placement: Literal['left', 'right', 'central'] = 'left', - intersection_mode: Literal['uniform', 'shift_to_left', 'shift_to_right'] = 'shift_to_left', - ): - # idx = idx - # cast everything to pandas object fir the subsequent comfort - if isinstance(idx, np.array): - if idx.dtype == np.dtype('O'): - idx = pd.to_datetime(pd.Series(idx)) - td = pd.Timedelta(window_size) - else: - idx = pd.Series(idx) - td = window_size - else: - raise TypeError('Unexpected type of ts index') - - boundaries = np.tile(actual_timestamps, (2, 1)) - # [0, ...] - lower bound, [1, ...] - upper - if window_placement == 'left': - boundaries[0] -= td - elif window_placement == 'central': - boundaries[0] -= td / 2 - boundaries[1] += td / 2 - elif window_placement == 'right': - boundaries[1] += td - else: - raise ValueError('Unknown mode') - - if not len(actual_timestamps): - return boundaries - - # intersection resolution - for i in range(len(actual_timestamps) - 1): - if not boundaries[0, i + 1] > boundaries[1, i]: - continue - - if intersection_mode == 'shift_to_left': - boundaries[0, i + 1] = boundaries[1, i] - elif intersection_mode == 'shift_to_right': - boundaries[1, i] = boundaries[0, i + 1] - elif intersection_mode == 'uniform': - boundaries[1, i], boundaries[0, i + 1] = boundaries[0, i + 1], boundaries[1, i] - else: - raise ValueError('Unknown intersection resolution') - - # filtering - idx_to_keep = np.abs(np.diff(boundaries, axis=0)) > 1e-6 - boundaries = boundaries[..., idx_to_keep] - boundaries = pd.DataFrame({'lower': boundaries[0], 'upper': boundaries[1]}) - return boundaries - - -def nab(boundaries, predictions, mode='standard', custom_coefs=None): - inner_coefs = { - 'low_FP': [1.0, -0.11, -1.0], - 'standard': [1., -0.22, -1.], - 'lof_FN': [1., -0.11, -2.] - } - coefs = custom_coefs or inner_coefs[mode] - confusion_matrix = extract_cp_cm(boundaries, predictions) - - tps = confusion_matrix['tps'] - - score = np.inner([tps, len(confusion_matrix['FP']), len(confusion_matrix['FN'])], - coefs) - return score From c1ab5da891e7efc22d1fb68448c7329647e38269 Mon Sep 17 00:00:00 2001 From: Leon_Strelkov <103892559+leostre@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:53:09 +0300 Subject: [PATCH 42/43] Apply suggestions from code review Co-authored-by: George Lopatenko <81328772+Lopa10ko@users.noreply.github.com> --- fedot_ind/core/models/quantile/quantile_extractor.py | 5 +---- fedot_ind/core/repository/model_repository.py | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/fedot_ind/core/models/quantile/quantile_extractor.py b/fedot_ind/core/models/quantile/quantile_extractor.py index 596f165d6..17a44e0cd 100644 --- a/fedot_ind/core/models/quantile/quantile_extractor.py +++ b/fedot_ind/core/models/quantile/quantile_extractor.py @@ -42,10 +42,7 @@ def __init__(self, params: Optional[OperationParameters] = None): self.stride = params.get('stride', 1) self.add_global_features = params.get('add_global_features', True) self.logging_params.update({'Wsize': self.window_size, - 'Stride': self.stride, - # 'VarTh': self.var_threshold - } - ) + 'Stride': self.stride}) def _concatenate_global_and_local_feature( self, diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py index b67df5d7c..c3e226dc7 100644 --- a/fedot_ind/core/repository/model_repository.py +++ b/fedot_ind/core/repository/model_repository.py @@ -37,10 +37,6 @@ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from xgboost import XGBRegressor -from fedot_ind.core.models.early_tc.ecec import ECEC -from fedot_ind.core.models.early_tc.economy_k import EconomyK -from fedot_ind.core.models.early_tc.prob_threshold import ProbabilityThresholdClassifier -from fedot_ind.core.models.early_tc.teaser import TEASER from fedot_ind.core.models.detection.anomaly.algorithms.arima_fault_detector import ARIMAFaultDetector from fedot_ind.core.models.detection.anomaly.algorithms.convolutional_autoencoder_detector import \ ConvolutionalAutoEncoderDetector From 916f89914691671b031c7f9492a9155908f70fba Mon Sep 17 00:00:00 2001 From: leostre Date: Fri, 26 Jul 2024 15:35:25 +0300 Subject: [PATCH 43/43] changed bump up fedot --- .github/workflows/poetry_unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/poetry_unit_test.yml b/.github/workflows/poetry_unit_test.yml index 2b3032116..b6b197b77 100644 --- a/.github/workflows/poetry_unit_test.yml +++ b/.github/workflows/poetry_unit_test.yml @@ -39,7 +39,7 @@ jobs: run: poetry install - name: Bump up FEDOT to a stable revision (temporary) - run: poetry add git+https://github.com/aimclub/FEDOT.git@e0b4ee7 + run: poetry add git+https://github.com/aimclub/FEDOT.git@master - name: Run tests with pytest run: poetry run pytest --cov=fedot_ind --cov-report xml:coverage.xml tests/unit