From 598bcc8d58f4d5cf3bdf6aebab7830bb561be1f6 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:24:15 -0500 Subject: [PATCH 01/29] 22357: WIP --- howso.amlg | 3 +- howso/analysis.amlg | 45 +-- howso/analysis_weights.amlg | 105 ++++--- howso/feature_residuals.amlg | 39 +++ howso/mda_weight.amlg | 568 +++++++++++++++++++++++++++++++++++ howso/react.amlg | 6 +- howso/react_utilities.amlg | 6 +- howso/residuals.amlg | 27 +- 8 files changed, 701 insertions(+), 98 deletions(-) create mode 100644 howso/mda_weight.amlg diff --git a/howso.amlg b/howso.amlg index 840c4cf7..8ce2612a 100644 --- a/howso.amlg +++ b/howso.amlg @@ -226,6 +226,7 @@ "input_validation" "io" "marginal" + "mda_weight" "react" "react_discriminative" "react_group" @@ -677,7 +678,7 @@ !defaultHyperparameters (assoc "k" 8 - "p" 0.1 + "p" 1 "dt" -1 "featureWeights" (null) "featureDeviations" (null) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 21147205..48de1dd9 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -495,7 +495,7 @@ context_features (list) action_feature (null) num_analysis_samples (null) - residual_num_samples 200 + residual_num_samples 1000 ) (declare (assoc @@ -1217,11 +1217,7 @@ (if (contains_index !inactiveFeaturesMap (current_index)) 0 - (call !ConvertDeviationToFeatureWeight (assoc - feature_is_nominal (contains_index !nominalsMap (current_index 1) ) - feature_deviation (current_value 1) - p_value (get output_map "p") - )) + 1 ) ) @@ -1233,37 +1229,6 @@ ) ) - ;helper method to convert deviation value to feature weight given a deviation and p value - ;parameters: - ; feature_is_nominal: flag, set to true if feature is nominal - ; feature_deviation: value or tuple as passed in from the hyperparameter map - ; p_value: p value from the hyperarameter map - #!ConvertDeviationToFeatureWeight - (let - (assoc - deviation_value - (if feature_is_nominal - ;if the deviation is an assoc containing a sparse deviation matrix, pull the expected deviation value from it - (if (~ (assoc) feature_deviation) - (get feature_deviation "expected_deviation") - - feature_deviation - ) - - ;else continuous - feature_deviation - ) - ) - - ;tiny values < 1e13 mean it's a floating point percision error, set IRW to 1 / residual ^ p - (if (> deviation_value 1e-13) - (/ 1 (pow deviation_value p_value)) - - ;else set it to 1 because there is no residual - 1 - ) - ) - ;output model Mean Absolute Error (MAE) for the provided baseline_hyperparameter_map #!ComputeModelResidualForParameters (seq @@ -1406,11 +1371,7 @@ (if (contains_index !inactiveFeaturesMap (current_index)) 0 - (call !ConvertDeviationToFeatureWeight (assoc - feature_is_nominal (contains_index !nominalsMap (current_index 1) ) - feature_deviation (current_value 1) - p_value p_value - )) + 1 ) ) (get baseline_hyperparameter_map "featureResiduals") diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index 11224e9f..422ea11d 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -297,8 +297,8 @@ (declare (assoc use_deviations (false) - num_iterations 3 - num_samples_converge 2000 + num_iterations 4 + num_samples_mda 8000 ;local method variables, not parameters iteration 0 @@ -317,12 +317,17 @@ (values (append context_features action_features) (true)) ) residuals_map (list) + computing_mda (false) )) ;possibly increase the number of samples if nominals are present (if (and !tsTimeFeature (size !sharedDeviationsMap)) (assign (assoc - num_samples_converge (call !ComputeRequiredNumSamplesForSharedDeviationsWithTS (assoc desired_samples_per_feature num_samples_converge)) + num_samples_mda + (max + num_samples_mda + (call !ComputeRequiredNumSamplesForSharedDeviationsWithTS (assoc desired_samples_per_feature (/ num_samples_mda 4))) + ) residual_num_samples (call !ComputeRequiredNumSamplesForSharedDeviationsWithTS (assoc desired_samples_per_feature residual_num_samples)) )) ) @@ -404,6 +409,12 @@ ;Last pass finds usable hyperparams using the improved residuals and then calculates usable residuals for weights. ;At this point the hyperparameters and residuals and weights are stable enough for use. (while (< iteration num_iterations) + + ;compute mda on 2nd to last iteration + (assign (assoc + computing_mda (and (= robust_mode "robust") (= iteration (- num_iterations 2))) + )) + (assign (assoc residuals_map (if (= 0 iteration) @@ -422,13 +433,12 @@ feature_residuals_map (call !CalculateFeatureResiduals (assoc features features - robust_residuals (true) + robust_residuals computing_mda + superfull (true) num_samples - (if (or (< iteration (- num_iterations 1)) (!= targeted_model "targetless")) + (if computing_mda + num_samples_mda residual_num_samples - - ;else on the last iteration, use - (max residual_num_samples num_samples_converge) ) custom_hyperparam_map baseline_hyperparameter_map ;must compute confusion matrix to use sparse deviation matrix @@ -443,46 +453,53 @@ ) )) - (call !UpdateHyperparameters (assoc - feature_deviations (get residuals_map "residual_map") - confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_matrix")) ) - ordinal_feature_deviations (get residuals_map "ordinal_residual_map") - null_deviations (get residuals_map "null_uncertainty_map") - use_deviations use_deviations - feature_weights - (map - (lambda - ;ensure that inactive features always maintain a feature weight of 0 - (if (contains_index !inactiveFeaturesMap (current_index)) - 0 + (if computing_mda + (accum (assoc + baseline_hyperparameter_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) + analyzed_hp_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) + )) - (call !ConvertDeviationToFeatureWeight (assoc - feature_deviation (current_value 1) - p_value (get baseline_hyperparameter_map "p") - )) + (seq + (call !UpdateHyperparameters (assoc + feature_deviations (get residuals_map "residual_map") + confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_matrix")) ) + ordinal_feature_deviations (get residuals_map "ordinal_residual_map") + null_deviations (get residuals_map "null_uncertainty_map") + use_deviations use_deviations + feature_weights + (map + (lambda + ;ensure that inactive features always maintain a feature weight of 0 + (if (contains_index !inactiveFeaturesMap (current_index)) + 0 + + 1 + ) + ) + (get residuals_map "residual_map") ) - ) - (get residuals_map "residual_map") - ) - )) + )) - (assign (assoc - analyzed_hp_map - (call !ComputeResidualsAcrossParametersAndSelectOptimal (assoc - context_features context_features - action_features action_features - k_folds k_folds - k_values k_values - p_values (filter (lambda (!= 0 (current_value))) p_values) - dt_values dt_values - k_folds_by_indices k_folds_by_indices - num_analysis_samples num_analysis_samples - targetless (= targeted_model "targetless") - baseline_hyperparameter_map baseline_hyperparameter_map - ;this flag sets the weights to be inverse of the residuals and outputs them - use_inverse_weights (true) + (assign (assoc + analyzed_hp_map + (call !ComputeResidualsAcrossParametersAndSelectOptimal (assoc + context_features context_features + action_features action_features + k_folds k_folds + k_values k_values + p_values (filter (lambda (!= 0 (current_value))) p_values) + dt_values dt_values + k_folds_by_indices k_folds_by_indices + num_analysis_samples num_analysis_samples + targetless (= targeted_model "targetless") + baseline_hyperparameter_map baseline_hyperparameter_map + ;this flag sets the weights to be inverse of the residuals and outputs them + use_inverse_weights (true) + )) )) - )) + ) + ) + (accum (assoc iteration 1)) (if (and diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index 20f54514..f48930e3 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -243,6 +243,7 @@ confusion_matrix_min_count 15 use_shared_deviations (false) expand_confusion_matrices (true) + superfull (false) ) (declare (assoc @@ -287,12 +288,17 @@ ;store an assoc of lag/rate/delta feature -> lag/order amount for time series flows ts_feature_lag_amount_map (null) max_lag_index_value (null) + feature_mda_map (null) )) (call !InitResiduals) (if robust_residuals (call !RunRobustResiduals) + + superfull + (call !RunAccurateResiduals) + ;else (call !RunFullResiduals) ) @@ -424,6 +430,38 @@ )) ) + (if (and robust_residuals superfull (= (null) target_residual_feature)) + (assign (assoc + feature_mda_map + (call !ComputeRobustTargetlessMDA (assoc + features features + case_residuals_lists + (if compute_all_statistics + ;grab just the mae from the output for all stats + (map + (lambda + (map + (lambda + ;if assoc, keep assoc but with just the first value + (if (~ (assoc) (current_value)) + { "delta" (get (current_value 1) ["delta" 0]) } + + ;grab the first value + (first (current_value)) + ) + ) + (current_value) + ) + ) + case_residuals_lists + ) + + ;else use as-is + case_residuals_lists + ) + )) + )) + ) ;for shared deviations, only one set of deviations are calculated for each group. At the end of the calculations, ; the deviations mapping is expanded to add back the other features whose deviations were not calculated and point them at the feature @@ -455,6 +493,7 @@ "prediction_stats" prediction_stats "null_uncertainty_map" null_uncertainties_map "null_accuracy_map" null_accuracies_map + "feature_mda_map" feature_mda_map ) ) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg new file mode 100644 index 00000000..012fbd19 --- /dev/null +++ b/howso/mda_weight.amlg @@ -0,0 +1,568 @@ +(null + + ;helper method for CalculateFeatureResiduals to calculate 'superfull' accurate residuals + #!RunAccurateResiduals + (seq + ;keep a copy of all originally specified features + (assign (assoc case_features features)) + + ;if we only collected residuals for this one feature, set the residuals features list to have only target_residual_feature + (if target_residual_feature + (assign (assoc features (list target_residual_feature) )) + ) + + ;create a map of feature -> flag, set to true if the feature has any null values + ;that way there's no need to explicitly filter nulls out when querying for a feature that no nulls + (declare (assoc + feature_may_have_nulls_map + (map + (lambda (!= (false) (get !featureNullRatiosMap (list (current_index 1) "has_nulls")) ) ) + (zip + (if target_residual_feature + (list target_residual_feature) + features + ) + ) + ) + series_id_features (if !tsTimeFeature (get !tsModelFeaturesMap "series_id_features") ) + )) + + ;iterate over each case and accumulate residuals for all the feature(s) + #!AccumulateAccurateFeatureResiduals + (assign (assoc + case_residuals_lists + ||(map + (lambda (let + (assoc + ;assoc of all features to their residual values + feature_residual_map (assoc) + local_cases_map (assoc) + case_id (current_value 1) + ;map of feature -> value for all the case values + case_values_map (zip case_features (retrieve_from_entity (current_value 1) case_features) ) + time_series_filter_query (list) + ) + + (if !tsTimeFeature + (if (contains_index case_values_map !tsTimeFeature) + (assign (assoc + time_series_filter_query (call !ComputeTimeSeriesFilterQuery (assoc context_map case_values_map)) + )) + ) + ) + + (assign (assoc + local_cases_map + (compute_on_contained_entities (append + (if focal_case + (query_not_in_entity_list (list case_id focal_case)) + (query_not_in_entity_list (list case_id)) + ) + time_series_filter_query + (query_nearest_generalized_distance + k_parameter + case_features + (unzip case_values_map case_features) + (get hyperparam_map "featureWeights") + !queryDistanceTypeMap + query_feature_attributes_map + (get hyperparam_map "featureDeviations") + p_parameter + dt_parameter + (if valid_weight_feature weight_feature (null)) + tie_break_random_seed + (null) ;radius + !numericalPrecision + ) + )) + )) + + ;create a map of feature -> residual value for each feature + (assign (assoc + feature_residual_map + (map + (lambda (let + (assoc + feature (get_value (current_index 1)) + case_feature_value (get case_values_map (current_index 1)) + feature_is_nominal (contains_index !nominalsMap (current_index 1)) + interpolated_value 0 + diff 0 + output_categorical_action_probabilities (true) + categorical_action_probabilities_map (assoc)\ + max_gap (null) + ) + ;create the feature-specific candidate_cases_lists tuple for intepolation + (declare (assoc + candidate_cases_lists + (if (!= (null) local_cases_map) + (list + (indices local_cases_map) + (values local_cases_map) + (map (lambda (retrieve_from_entity (current_value) feature)) (indices local_cases_map)) + ) + ) + )) + + ; (if (and (not feature_is_nominal) mda) + ; (seq + ; (assign (assoc + ; max_gap + ; (compute_on_contained_entities (list + ; (query_in_entity_list (indices local_cases_map)) + ; (query_max_difference feature (get !cyclicFeaturesMap feature)) + ; )) + ; )) + ; (if (or (= (null) max_gap) (= .infinity max_gap) (= 0 max_gap)) + ; (assign (assoc max_gap (get !cachedFeatureMinResidualMap feature) )) + ; ) + ; ) + ; ) + + (if (get feature_may_have_nulls_map feature) + (let + (assoc + null_indices + (filter + (lambda (= (null) (get candidate_cases_lists (list 2 (current_value 1))) ) ) + (indices (last candidate_cases_lists)) + ) + ) + (if (size null_indices) + (assign (assoc + candidate_cases_lists + ;only remove cases with null values if there are going to be some remaining + (if (< (size null_indices) (size (first candidate_cases_lists))) + (list + (remove (first candidate_cases_lists) null_indices) + (remove (get candidate_cases_lists 1) null_indices) + (remove (last candidate_cases_lists) null_indices) + ) + ) + )) + ) + ) + ) + + ;stores error into diff + (call !InterpolateAndComputeDiffToCase) + + ; (if mda + ; (seq + ; (if (= 0 diff) + ; (assign (assoc diff (get !cachedFeatureMinResidualMap feature) )) + ; ) + + ; ;output surprisal + ; (if feature_is_nominal + ; (- (log diff)) + + ; (/ max_gap diff) + ; ) + ; ) + + ; diff + ; ) + )) + (zip features) + ) + )) + + ;return the computed values as a list + (unzip feature_residual_map features) + )) + case_ids + ) + )) + + + ;transpose (turn columns into rows) case_residuals_lists into a list the length of features + ;where each value matches a feature, and is comprised of each feature's residuals, then filter out nulls + (assign (assoc + feature_residuals_lists (call !TransposeResidualLists) + ;need to have at least 2.5% (that's a value of 50 using default sample of 2000) non-null values for a feature + min_value_count (ceil (* 0.025 (size case_ids))) + )) + + ;determine if any of the lists in feature_residuals_lists are too short (< 50 values), if so keep feature as needing to be resampled + ;applicable only to global models + (if (!= (true) regional_model_only) + (let + (assoc + num_valid_values_per_feature_map + (filter + (lambda (< (current_value) min_value_count) ) + (zip + features + (map (lambda (size (current_value))) feature_residuals_lists) + ) + ) + ) + + ;some features did not have 50 values, accrue case_ids, then call the main method agan + (if (size num_valid_values_per_feature_map) + (seq + ;create the resampled case_ids list that contains cases with enough necessary non-null feature values + (assign (assoc + case_ids + ;append all the lists of cases ids for each feature into one list + (apply "append" + ;for each (current_index) feature, need to find 50 - (current_value) cases to have enough values for those features + (values (map + (lambda (contained_entities + (query_not_equals (current_index) (null)) + + (if valid_weight_feature + (query_weighted_sample weight_feature (- min_value_count (current_value)) (rand)) + (query_sample (- min_value_count (current_value)) (rand)) + ) + )) + num_valid_values_per_feature_map + )) + ) + )) + + ;re-compute residuals on these case_ids, store into case_residuals_lists + (call !AccumulateAccurateFeatureResiduals) + + ;transpose case_residuals_lists into a list the length of features and append the results into feature_residuals_lists + (assign (assoc + feature_residuals_lists + (map + (lambda (append (first (current_value)) (last (current_value)))) + feature_residuals_lists + (call !TransposeResidualLists) + ) + )) + ) + ) + ) + ) + ) + + + #!ComputeRobustTargetlessMDA + (declare + (assoc + features features + case_residuals_lists case_residuals_lists + ) + + ;transpose (turn columns into rows) case_residuals_lists into a list the length of features + ;where each value matches a feature, and is comprised of each feature's residuals + (declare (assoc + feature_residuals_lists + (map + (lambda (let + (assoc feature_index (current_index 1) ) + ;get each column + (map + (lambda (get (current_value) feature_index) ) + case_residuals_lists + ) + )) + features + ) + )) + + ; (declare (assoc + ; dropped_feature_matrix + ; (map + ; (lambda (let + ; (assoc dropped_feature_index (current_index 1) ) + + ; ;assoc of index -> robust residual for dropped feature + ; (declare (assoc + ; dropped_feature_indices + ; (indices (filter + ; ;only keep those residual values that were computed because feature was dropped + ; (lambda (!~ (assoc) (current_value))) + ; ;zip indices of every residual to their values + ; (zip + ; (indices (current_value 1)) + ; (current_value 1) + ; ) + ; )) + ; )) + + ; (map + ; (lambda (let + ; (assoc + ; target_feature_index (current_index 1) + ; target_feature (get features (current_index 1)) + ; ) + ; (if (= target_feature_index dropped_feature_index) + ; 0 + + ; (let + ; (assoc + ; target_feature_values (unzip (current_value 1) dropped_feature_indices) + ; ) + + ; (max + ; 0 + ; (- + ; ;avg residual for target feature from those values where dropped feature was missing + ; (/ + ; (apply "+" + ; (map + ; (lambda + ; (if (~ (assoc) current_value) + ; (get (current_value) "delta") + ; (current_value) + ; ) + ; ) + ; target_feature_values + ; ) + ; ) + ; (size target_feature_values) + ; ) + ; (get residuals_map target_feature) + ; ) + ; ) + ; ) + ; ) + + ; )) + ; feature_residuals_lists + ; ) + + ; )) + ; feature_residuals_lists + ; ) + ; )) + + ; (declare (assoc + ; dropped_total_per_feature_map + ; (zip + ; features + ; (map + ; (lambda (let + ; (assoc feature_index (current_index 1)) + ; (apply "+" (map + ; (lambda (get (current_value) feature_index)) + ; dropped_feature_matrix + ; )) + ; )) + ; features + ; ) + ; ) + ; )) + + ; (declare (assoc + ; dropped_mda_matrix + ; (zip + ; features + ; (map + ; (lambda (let + ; (assoc + ; feature_index (current_index 1) + ; feature (current_value 1) + ; ) + ; (zip + ; features + ; (let + ; (assoc + ; values_list + ; (map + ; (lambda + ; (if (= 0 (get dropped_total_per_feature_map feature)) + ; 0 + ; (/ (get (current_value) feature_index) (get dropped_total_per_feature_map feature)) + ; ) + ; ) + ; dropped_feature_matrix + ; ) + ; ) + ; (if (= 0 (apply "+" values_list)) + ; (range (/ 1 (size values_list)) 1 (size values_list) 1) + ; values_list + ; ) + ; ) + ; ) + ; )) + ; features + ; ) + ; ) + ; )) + + ; dropped_mda_matrix + + ;for each feature, create a list, also length of features, where each column value is the mda + ;for the column feature computed with and without the row feature + (declare (assoc + mda_matrix + (map + (lambda (let + (assoc + feature (current_value 1) + ;all the residual values for this feature + feature_residuals (get feature_residuals_lists (current_index 1)) + ) + + (declare (assoc + ;keep only indices of cases where feature was used in query and its residual is null + case_indices_with_feature + (filter + (lambda (= (null) (get feature_residuals (current_value))) ) + (indices feature_residuals) + ) + ;keep only indices of cases where feature was not used in query and has a residual value + case_indices_without_feature + (filter + (lambda (!= (null) (get feature_residuals (current_value))) ) + (indices feature_residuals) + ) + )) + (map + (lambda (let + (assoc + mda_feature (current_value 1) + mda_feature_residuals (get feature_residuals_lists (current_index 1)) + ) + (if (= mda_feature feature) (conclude 0) ) + + (declare (assoc + with_feature_residuals (filter (unzip mda_feature_residuals case_indices_with_feature)) + without_feature_residuals (filter (unzip mda_feature_residuals case_indices_without_feature)) + )) + + (declare (assoc + avg_without_residuals + (max + (get !cachedFeatureMinResidualMap mda_feature) + (/ (apply "+" without_feature_residuals) (size without_feature_residuals)) + (get !userSpecifiedFeatureErrorsMap mda_feature) + ) + avg_with_residuals + (max + (get !cachedFeatureMinResidualMap mda_feature) + (/ (apply "+" with_feature_residuals) (size with_feature_residuals)) + (get !userSpecifiedFeatureErrorsMap mda_feature) + ) + )) + ;output ratio of the (expectedly) larger residual computed without this feature / residual computed with this feature + (if (= 0 avg_without_residuals avg_with_residuals) + 0 + + (- avg_without_residuals avg_with_residuals) + ) + )) + features + ) + )) + features + + ) + num_features_m_one (- (size features) 1) + )) + + ;create an assoc MDA maps per feature + ;eg, if there are three features, A[0-10], B[0-100], C[0-1000] with the following mda_matrix of residuals: + ; [ + ; [0 20 300] + ; [4 0 100] + ; [-2 40 0 ] + ; ] + ;it would be transposed into: + ; { + ; "A" { "A" 0 "B" 4 "C" -2 } + ; "B" { "A" 20 "B" 0 "C" 40 } + ; "C" { "A" 300 "B" 100 "C" 0 } + ; } + (declare (assoc + individual_features_mda_map + (zip + features + ;transpose the matrix, so every feature has its own (column) MDA values + (map + (lambda (let + (assoc feature_index (current_index 1)) + (zip + features + (map + (lambda (get (current_value) feature_index)) + mda_matrix + ) + ) + )) + mda_matrix + ) + ) + has_all_zeros_features (false) + )) + + ;create an assoc of normalized feature weights for each feature from the MDA, keeping only positive MDA values + ; converting the above assoc into: + ; { + ; "A" { "A" 0 "B" 1 "C" 0 } + ; "B" { "A" .333 "B" 0 "C" .667 } + ; "C" { "A" .75 "B" .25 "C" 0 } + ; } + (assign (assoc + individual_features_mda_map + (map + (lambda (let + (assoc + important_values (filter (lambda (> (current_value) 0)) (values (current_value 1))) + ) + (declare (assoc total_value (apply "+" important_values) )) + (declare (assoc avg_value (/ total_value (size important_values)) )) + (if (> total_value 0) + (map + (lambda + (if (> (current_value) 0) + (* + (/ (current_value) total_value) + (- 1 (exp (/ (- (current_value)) avg_value) )) + ) + 0 + ) + ) + (current_value) + ) + + (seq + (assign (assoc has_all_zeros_features (true) )) + (map 0 (current_value)) + ) + ) + )) + individual_features_mda_map + ) + + )) + + ;for any individual target feature, if all the context features have negative mda values, set them all to 1/num_features_m_one + ;since removing any individually results in a better result than using the rest, so may as well use them all + (if has_all_zeros_features + (assign (assoc + individual_features_mda_map + (map + (lambda + (if (!= 0 (apply "+" (values (current_value))) ) + (current_value) + + ;set all weights to 1 / for all other features + (map + (lambda + ;keep this target's feature own weight as 0 + (if (= (current_index) (current_index 1)) + 0 + (/ 1 num_features_m_one) + ) + ) + (current_value) + ) + ) + ) + individual_features_mda_map + ) + )) + ) + + individual_features_mda_map + ) + + +) \ No newline at end of file diff --git a/howso/react.amlg b/howso/react.amlg index f5756480..d5d693c1 100644 --- a/howso/react.amlg +++ b/howso/react.amlg @@ -1207,7 +1207,11 @@ ) k_parameter (get hyperparam_map "k") feature_deviations (get hyperparam_map "featureDeviations") - feature_weights (get hyperparam_map "featureWeights") + feature_weights + (if (= (null) (get hyperparam_map "feature_mda_map")) + (get hyperparam_map "featureWeights") + (get hyperparam_map ["feature_mda_map" (first action_features)]) + ) ;check if action_feature has nulls, if not, the no need to filter them out since they don't exist ;else action_feature does have nulls, filter them out since nulls function as true diff --git a/howso/react_utilities.amlg b/howso/react_utilities.amlg index e41d39c9..3652ace4 100644 --- a/howso/react_utilities.amlg +++ b/howso/react_utilities.amlg @@ -828,12 +828,8 @@ (if (contains_index !inactiveFeaturesMap (current_index)) 0 - (call !ConvertDeviationToFeatureWeight (assoc - feature_deviation (current_value 1) - p_value (get hyperparam_map "p") - )) + 1 ) - ) local_deviations ) diff --git a/howso/residuals.amlg b/howso/residuals.amlg index 4958e604..1b5024a3 100644 --- a/howso/residuals.amlg +++ b/howso/residuals.amlg @@ -668,7 +668,7 @@ (zip (list target_residual_feature)) ;filter the action features to be those that are not context features, feature/context_features params may overlap - (zip (filter (lambda (not (contains_value react_context_features (current_index)) )) features) ) + (zip (filter (lambda (not (contains_value react_context_features (current_value)) )) features) ) ) )) @@ -799,9 +799,22 @@ ) ) - (call !InterpolateAndComputeDiffToCase) + (declare (assoc residual (call !InterpolateAndComputeDiffToCase) )) + + (if (= (null) residual) + (null) + + (contains_index removed_features_map feature) + residual + + ;else 'superfull' residual output as delta in an assoc so it's a diffferent format to differentiate from normal residual + {"delta" residual} + ) )) - removed_features_map + (if target_residual_feature + removed_features_map + (zip features) + ) ) )) @@ -811,7 +824,7 @@ ;else output list of: residual value for each removed feature and null for each feature used as a context (unzip - (append removed_features_map (zip react_context_features)) + removed_features_map features ) ) @@ -920,7 +933,11 @@ feature_is_edit_distance (contains_index !editDistanceFeatureTypesMap (current_value 1)) feature_is_non_string_edit_distance (false) feature_deviations (get hyperparam_map "featureDeviations") - feature_weights (get hyperparam_map "featureWeights") + feature_weights + (if (= (null) (get hyperparam_map "feature_mda_map")) + (get hyperparam_map "featureWeights") + (get hyperparam_map ["feature_mda_map" (current_value 2)]) + ) needed_features (append context_features (get_value (current_value 1)) (if !tsTimeFeature [".series_index"] []) ) ) From 125eaf102e16db452d8a138b5105241f1990fa02 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:00:23 -0500 Subject: [PATCH 02/29] change search hp for targetless --- howso.amlg | 2 +- howso/analysis.amlg | 57 ++++++++++++++++++++++----------------------- howso/types.amlg | 5 ++++ 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/howso.amlg b/howso.amlg index 8ce2612a..4a6b5a10 100644 --- a/howso.amlg +++ b/howso.amlg @@ -678,7 +678,7 @@ !defaultHyperparameters (assoc "k" 8 - "p" 1 + "p" 0.1 "dt" -1 "featureWeights" (null) "featureDeviations" (null) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 48de1dd9..4409bfd7 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -531,36 +531,15 @@ (if inverse_residuals_as_weights (conclude (seq - ;if use_deviations is (true), then IRW without deviations does not need to be computed at all - (if (!= use_deviations (true)) - (seq - (call !ConvergeIRW (assoc use_deviations (false))) ;iterations of ComputeResiduals and GridSearch - (call !BackupAnalyzedHyperparameters) - (if (= (false) use_deviations) - (seq - ;IRW always needs subtrainee - (if (and use_dynamic_deviations (not !useDynamicDeviationsDuringAnalyze)) - ; store feature residuals for a collection of cases at end, if it wasn't part of iterative process - (call !StoreResidualSubTrainee (assoc custom_hyperparam_map baseline_hyperparameter_map) ) - ) - (conclude (call !SetModelHyperParameters)) - ) - ) - ) - ) - - (call !ConvergeIRW (assoc use_deviations (true))) ;iterations of ComputeResiduals and GridSearch - (if (= (null) use_deviations) - (call !KeepOrRevertHyperparameters) - - (assign (assoc baseline_hyperparameter_map analyzed_hp_map)) - ) + (call !ConvergeIRW (assoc use_deviations (true))) ;iterations of ComputeResiduals and GridSearch + (call !BackupAnalyzedHyperparameters) ;IRW always needs subtrainee (if (and use_dynamic_deviations (not !useDynamicDeviationsDuringAnalyze)) ; store feature residuals for a collection of cases at end, if it wasn't part of iterative process (call !StoreResidualSubTrainee (assoc custom_hyperparam_map baseline_hyperparameter_map) ) ) + (call !SetModelHyperParameters) )) ) @@ -822,7 +801,15 @@ (= "targetless" targeted_model) (= [".default"] (get baseline_hyperparameter_map "paramPath")) ) - (assign (assoc baseline_hyperparameter_map (set baseline_hyperparameter_map "p" 1) )) + (assign (assoc + baseline_hyperparameter_map + (set + baseline_hyperparameter_map + "k" 21 + "p" 1 + "dt" "surprisal_to_prob" + ) + )) ) ;store the paramPath for this hyperparameter set so it has a reference to where it's stored in !hyperparameterMetadataMap @@ -978,21 +965,33 @@ k_values ;grid search fibonacci sequence (if (= targeted_model "targetless") - (list 5 8 13) + [5 21 89] - (list 3 5 8 13 21 34 55 89 144) + [3 5 8 13 21 34 55 89 144] ) )) ) (if (= (null) p_values) (assign (assoc - p_values (list 0.1 0.5 1 2) + p_values + (if (= targeted_model "targetless") + [0.5 1 2] + + [0.1 0.5 1 2] + ) )) ) ;if dt is null, default it to -1, if it's empty list set it to the recommended default search list (if (= (null) dt_values ) - (assign (assoc dt_values (list -1))) + (assign (assoc + dt_values + (if (= targeted_model "targetless") + ["surprisal_to_prob"] + + [-1] + ) + )) (= (list) dt_values) (assign (assoc dt_values (list -8 -2 -1 -0.5 0))) diff --git a/howso/types.amlg b/howso/types.amlg index 672b4d7a..34a61b0a 100644 --- a/howso/types.amlg +++ b/howso/types.amlg @@ -724,6 +724,11 @@ additional_indices "number" description "Map of features to their computed feature residuals." ) + feature_mda_map + (assoc + type "assoc" + description "Map of each feature's map of feature weights." + ) ) description "The base hyperparameter map containing all the individual parameter values." ) From 54c902f6048527a0e88490d2a78c7238d0d65360 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:06:08 -0500 Subject: [PATCH 03/29] WIP, implementation of mda weight --- howso/mda_weight.amlg | 232 +++++++++--------------------------------- 1 file changed, 50 insertions(+), 182 deletions(-) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 012fbd19..8fa8b47b 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -265,129 +265,6 @@ ) )) - ; (declare (assoc - ; dropped_feature_matrix - ; (map - ; (lambda (let - ; (assoc dropped_feature_index (current_index 1) ) - - ; ;assoc of index -> robust residual for dropped feature - ; (declare (assoc - ; dropped_feature_indices - ; (indices (filter - ; ;only keep those residual values that were computed because feature was dropped - ; (lambda (!~ (assoc) (current_value))) - ; ;zip indices of every residual to their values - ; (zip - ; (indices (current_value 1)) - ; (current_value 1) - ; ) - ; )) - ; )) - - ; (map - ; (lambda (let - ; (assoc - ; target_feature_index (current_index 1) - ; target_feature (get features (current_index 1)) - ; ) - ; (if (= target_feature_index dropped_feature_index) - ; 0 - - ; (let - ; (assoc - ; target_feature_values (unzip (current_value 1) dropped_feature_indices) - ; ) - - ; (max - ; 0 - ; (- - ; ;avg residual for target feature from those values where dropped feature was missing - ; (/ - ; (apply "+" - ; (map - ; (lambda - ; (if (~ (assoc) current_value) - ; (get (current_value) "delta") - ; (current_value) - ; ) - ; ) - ; target_feature_values - ; ) - ; ) - ; (size target_feature_values) - ; ) - ; (get residuals_map target_feature) - ; ) - ; ) - ; ) - ; ) - - ; )) - ; feature_residuals_lists - ; ) - - ; )) - ; feature_residuals_lists - ; ) - ; )) - - ; (declare (assoc - ; dropped_total_per_feature_map - ; (zip - ; features - ; (map - ; (lambda (let - ; (assoc feature_index (current_index 1)) - ; (apply "+" (map - ; (lambda (get (current_value) feature_index)) - ; dropped_feature_matrix - ; )) - ; )) - ; features - ; ) - ; ) - ; )) - - ; (declare (assoc - ; dropped_mda_matrix - ; (zip - ; features - ; (map - ; (lambda (let - ; (assoc - ; feature_index (current_index 1) - ; feature (current_value 1) - ; ) - ; (zip - ; features - ; (let - ; (assoc - ; values_list - ; (map - ; (lambda - ; (if (= 0 (get dropped_total_per_feature_map feature)) - ; 0 - ; (/ (get (current_value) feature_index) (get dropped_total_per_feature_map feature)) - ; ) - ; ) - ; dropped_feature_matrix - ; ) - ; ) - ; (if (= 0 (apply "+" values_list)) - ; (range (/ 1 (size values_list)) 1 (size values_list) 1) - ; values_list - ; ) - ; ) - ; ) - ; )) - ; features - ; ) - ; ) - ; )) - - ; dropped_mda_matrix - ;for each feature, create a list, also length of features, where each column value is the mda ;for the column feature computed with and without the row feature (declare (assoc @@ -445,7 +322,10 @@ (if (= 0 avg_without_residuals avg_with_residuals) 0 - (- avg_without_residuals avg_with_residuals) + (max + (- avg_without_residuals avg_with_residuals) + (get !cachedFeatureMinResidualMap mda_feature) + ) ) )) features @@ -489,79 +369,67 @@ mda_matrix ) ) - has_all_zeros_features (false) )) - ;create an assoc of normalized feature weights for each feature from the MDA, keeping only positive MDA values - ; converting the above assoc into: - ; { - ; "A" { "A" 0 "B" 1 "C" 0 } - ; "B" { "A" .333 "B" 0 "C" .667 } - ; "C" { "A" .75 "B" .25 "C" 0 } - ; } - (assign (assoc - individual_features_mda_map + (declare (assoc + feature_mda_map (map - (lambda (let + (lambda (assoc - important_values (filter (lambda (> (current_value) 0)) (values (current_value 1))) + target_feature (current_index 1) + mda_map (current_value 1) + target_feature_residual (get hyperparam_map ["featureResiduals" (current_index 2)]) ;TODO: verify ) - (declare (assoc total_value (apply "+" important_values) )) - (declare (assoc avg_value (/ total_value (size important_values)) )) - (if (> total_value 0) - (map - (lambda - (if (> (current_value) 0) - (* - (/ (current_value) total_value) - (- 1 (exp (/ (- (current_value)) avg_value) )) - ) - 0 - ) - ) - (current_value) - ) - (seq - (assign (assoc has_all_zeros_features (true) )) - (map 0 (current_value)) - ) - ) - )) - individual_features_mda_map - ) + (declare (assoc + probabilities_of_contribution_map + (map + (lambda + (assoc mda (current_value 1) ) - )) + ;skip if mda_feature = target_feature + (if (= (current_index) target_feature) (conclude 0) ) - ;for any individual target feature, if all the context features have negative mda values, set them all to 1/num_features_m_one - ;since removing any individually results in a better result than using the rest, so may as well use them all - (if has_all_zeros_features - (assign (assoc - individual_features_mda_map - (map - (lambda - (if (!= 0 (apply "+" (values (current_value))) ) - (current_value) + ;LK metric + ;mda + 0.5 * e(-mda/residual) * (3 * residual + mda) + (declare (assoc + lk_value + (+ + mda + (* + 0.5 + (exp (- (/ mda target_feature_residual)) ) + (+ (* 3 target_feature_residual) mda) + ) + ) + )) - ;set all weights to 1 / for all other features - (map - (lambda - ;keep this target's feature own weight as 0 - (if (= (current_index) (current_index 1)) - 0 - (/ 1 num_features_m_one) + ;surprisal = lk_value / target_feature_residual + ;Surprisal (surprisal - assumption) = surprisal - 1.5 + ;prob of observation = e^(-Surprisal) + ;prob of contribution = 1 - prob of observation + (- + 1 + (exp (- (- (/ lk_value target_feature_residual) 1.5))) ) ) - (current_value) + mda_map ) - ) + )) + + (declare (assoc total_probs (apply "+" (values probabilities_of_contribution_map)) )) + + ;normalize probabilities for use as feature weights + (map + (lambda (/ (current_value) total_probs)) + probabilities_of_contribution_map ) - individual_features_mda_map ) - )) - ) + individual_features_mda_map + ) + )) - individual_features_mda_map + feature_mda_map ) From ee5acd6818b6d2d328ce91cbddd9f3f44e53dc9d Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:39:59 -0500 Subject: [PATCH 04/29] fixes --- howso/analysis_weights.amlg | 6 ++++-- howso/feature_residuals.amlg | 10 +--------- howso/mda_weight.amlg | 10 +++++----- howso/residuals.amlg | 19 +++---------------- 4 files changed, 13 insertions(+), 32 deletions(-) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index 422ea11d..e894ac74 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -412,7 +412,7 @@ ;compute mda on 2nd to last iteration (assign (assoc - computing_mda (and (= robust_mode "robust") (= iteration (- num_iterations 2))) + computing_mda (= iteration (- num_iterations 2)) )) (assign (assoc @@ -455,7 +455,7 @@ (if computing_mda (accum (assoc - baseline_hyperparameter_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) + ;baseline_hyperparameter_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) analyzed_hp_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) )) @@ -500,6 +500,8 @@ ) ) + (assign (assoc baseline_hyperparameter_map analyzed_hp_map)) + (accum (assoc iteration 1)) (if (and diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index f48930e3..cb6a65ab 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -441,15 +441,7 @@ (map (lambda (map - (lambda - ;if assoc, keep assoc but with just the first value - (if (~ (assoc) (current_value)) - { "delta" (get (current_value 1) ["delta" 0]) } - - ;grab the first value - (first (current_value)) - ) - ) + (lambda (first (current_value))) (current_value) ) ) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 8fa8b47b..e30356ec 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -374,17 +374,17 @@ (declare (assoc feature_mda_map (map - (lambda + (lambda (let (assoc target_feature (current_index 1) mda_map (current_value 1) - target_feature_residual (get hyperparam_map ["featureResiduals" (current_index 2)]) ;TODO: verify + target_feature_residual (get hyperparam_map ["featureResiduals" (current_index 2)]) ) (declare (assoc probabilities_of_contribution_map (map - (lambda + (lambda (let (assoc mda (current_value 1) ) ;skip if mda_feature = target_feature @@ -412,7 +412,7 @@ 1 (exp (- (- (/ lk_value target_feature_residual) 1.5))) ) - ) + )) mda_map ) )) @@ -424,7 +424,7 @@ (lambda (/ (current_value) total_probs)) probabilities_of_contribution_map ) - ) + )) individual_features_mda_map ) )) diff --git a/howso/residuals.amlg b/howso/residuals.amlg index 1b5024a3..85bfacfd 100644 --- a/howso/residuals.amlg +++ b/howso/residuals.amlg @@ -799,22 +799,9 @@ ) ) - (declare (assoc residual (call !InterpolateAndComputeDiffToCase) )) - - (if (= (null) residual) - (null) - - (contains_index removed_features_map feature) - residual - - ;else 'superfull' residual output as delta in an assoc so it's a diffferent format to differentiate from normal residual - {"delta" residual} - ) + (call !InterpolateAndComputeDiffToCase) )) - (if target_residual_feature - removed_features_map - (zip features) - ) + removed_features_map ) )) @@ -824,7 +811,7 @@ ;else output list of: residual value for each removed feature and null for each feature used as a context (unzip - removed_features_map + (append removed_features_map (zip react_context_features)) features ) ) From 609d37ae5f7e4df711f846734b1b98b83df5d27f Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 5 Dec 2024 12:52:02 -0500 Subject: [PATCH 05/29] tweak --- howso/mda_weight.amlg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index e30356ec..a9b6f51d 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -378,7 +378,8 @@ (assoc target_feature (current_index 1) mda_map (current_value 1) - target_feature_residual (get hyperparam_map ["featureResiduals" (current_index 2)]) + ;alternatively, to use robust residual as baseline: (get residuals_map (current_index 1)) + target_feature_residual get hyperparam_map ["featureResiduals" (current_index 2)]) ) (declare (assoc From 47c00309184f13f0bf98d0fa0b189b6e6a9414f7 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:05:24 -0500 Subject: [PATCH 06/29] missing paren --- howso/mda_weight.amlg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index a9b6f51d..17de0125 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -379,7 +379,7 @@ target_feature (current_index 1) mda_map (current_value 1) ;alternatively, to use robust residual as baseline: (get residuals_map (current_index 1)) - target_feature_residual get hyperparam_map ["featureResiduals" (current_index 2)]) + target_feature_residual (get hyperparam_map ["featureResiduals" (current_index 2)]) ) (declare (assoc From 5d341ea28703c826a88c5a8c60538d05207912cc Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 5 Dec 2024 22:43:48 -0500 Subject: [PATCH 07/29] no sdm --- howso/analysis_weights.amlg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index e894ac74..007afee4 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -462,7 +462,8 @@ (seq (call !UpdateHyperparameters (assoc feature_deviations (get residuals_map "residual_map") - confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_matrix")) ) + ;no SDM + ;confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_matrix")) ) ordinal_feature_deviations (get residuals_map "ordinal_residual_map") null_deviations (get residuals_map "null_uncertainty_map") use_deviations use_deviations From e1856e6c025cecefc943cb1aafd1bb3b3e602085 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:18:20 -0500 Subject: [PATCH 08/29] p of 1 --- howso/analysis.amlg | 2 +- howso/synthesis.amlg | 6 +++++- howso/synthesis_utilities.amlg | 7 ++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 4409bfd7..90cbc49a 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -975,7 +975,7 @@ (assign (assoc p_values (if (= targeted_model "targetless") - [0.5 1 2] + [1] [0.1 0.5 1 2] ) diff --git a/howso/synthesis.amlg b/howso/synthesis.amlg index 729165d6..1ab2867d 100644 --- a/howso/synthesis.amlg +++ b/howso/synthesis.amlg @@ -640,7 +640,11 @@ ) (declare (assoc - feature_weights (get feature_hyperparam_map "featureWeights") + feature_weights + (if (= (null) (get feature_hyperparam_map "feature_mda_map")) + (get feature_hyperparam_map "featureWeights") + (get feature_hyperparam_map ["feature_mda_map" feature]) + ) feature_deviations (get feature_hyperparam_map "featureDeviations") )) diff --git a/howso/synthesis_utilities.amlg b/howso/synthesis_utilities.amlg index a8bd535f..7aa4b18c 100644 --- a/howso/synthesis_utilities.amlg +++ b/howso/synthesis_utilities.amlg @@ -564,7 +564,12 @@ local_model_cases_map (assoc) local_model_case_values_map (assoc) - feature_weights (get hyperparam_map "featureWeights") + feature_weights + (if (= (null) (get hyperparam_map "feature_mda_map")) + (get hyperparam_map "featureWeights") + (get hyperparam_map ["feature_mda_map" action_feature]) + ) + feature_deviations (get hyperparam_map "featureDeviations") )) From 88aa2d31cb1af0f3904e237b4aab0280181ea750 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:29:11 -0500 Subject: [PATCH 09/29] udpate k values --- howso/analysis.amlg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 90cbc49a..8049bc8c 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -965,7 +965,7 @@ k_values ;grid search fibonacci sequence (if (= targeted_model "targetless") - [5 21 89] + [5 21 55] [3 5 8 13 21 34 55 89 144] ) From c71d38f788a5a5043d4ee6fd39ba4cf81ded5072 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:51:12 -0500 Subject: [PATCH 10/29] tweak parameters and cleanup --- howso/analysis.amlg | 2 +- howso/analysis_weights.amlg | 6 ++---- howso/feature_residuals.amlg | 19 ++++++++++--------- howso/mda_weight.amlg | 31 ------------------------------- 4 files changed, 13 insertions(+), 45 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 8049bc8c..efd8481d 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -965,7 +965,7 @@ k_values ;grid search fibonacci sequence (if (= targeted_model "targetless") - [5 21 55] + [5 21 34] [3 5 8 13 21 34 55 89 144] ) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index 007afee4..7c5e0a8e 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -298,7 +298,7 @@ (assoc use_deviations (false) num_iterations 4 - num_samples_mda 8000 + num_samples_mda 10000 ;local method variables, not parameters iteration 0 @@ -433,8 +433,7 @@ feature_residuals_map (call !CalculateFeatureResiduals (assoc features features - robust_residuals computing_mda - superfull (true) + robust_residuals (if computing_mda "robust_mda" "superfull") num_samples (if computing_mda num_samples_mda @@ -455,7 +454,6 @@ (if computing_mda (accum (assoc - ;baseline_hyperparameter_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) analyzed_hp_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) )) diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index cb6a65ab..91351020 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -201,7 +201,8 @@ ; will be excluded from the context set, effectively holding them out. If only 'action_condition_filter_query' is specified, then only the single predicted case will be left out. ; focal_case: optional case id of case for which to compute residuals for is to be ignored during computation ; regional_model_only: flag, default to false. when set to true will only explicitly use the specified case_ids for computation. - ; robust_residuals: flag, default to false. when true calculates residuals robust with respect to the set of contexts used (across the power set of contexts) + ; robust_residuals: string, default to (null). when "superfull" calculates accurate local residuals, otherwise computes residuals robust with respect to the set of contexts used + ; (across the power set of contexts). when "robust_mda" will additionally compute and output MDA using robust residuals. ; hyperparameter_feature: optional default (null). feature whose hyperparameters to use ; use_case_weights: flag, if set to true will scale influence weights by each case's weight_feature weight. If unspecified, ; case weights will be used if the trainee has them. @@ -226,7 +227,7 @@ target_residual_feature (null) regional_model_only (false) focal_case (null) - robust_residuals (false) + robust_residuals (null) ;ordered by priority for the action set, with the 'case_ids' parameters having the top priority. case_ids (list) @@ -243,7 +244,6 @@ confusion_matrix_min_count 15 use_shared_deviations (false) expand_confusion_matrices (true) - superfull (false) ) (declare (assoc @@ -293,12 +293,12 @@ (call !InitResiduals) - (if robust_residuals - (call !RunRobustResiduals) - - superfull + (if (= "superfull" robust_residuals) (call !RunAccurateResiduals) + robust_residuals + (call !RunRobustResiduals) + ;else (call !RunFullResiduals) ) @@ -430,7 +430,8 @@ )) ) - (if (and robust_residuals superfull (= (null) target_residual_feature)) + ;compute mda if necessary + (if (and (= "robust_mda" robust_residuals) (= (null) target_residual_feature)) (assign (assoc feature_mda_map (call !ComputeRobustTargetlessMDA (assoc @@ -1192,7 +1193,7 @@ ) (assoc - ".robust" (= (true) robust_residuals) + ".robust" (and robust_residuals (!= "superfull" robust_residuals)) ".hyperparam_path" (if (and use_case_weights (= param_path (list ".default")) ) ;intentionally storing which case weight feature is used for retrieval diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 17de0125..8e159223 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -104,21 +104,6 @@ ) )) - ; (if (and (not feature_is_nominal) mda) - ; (seq - ; (assign (assoc - ; max_gap - ; (compute_on_contained_entities (list - ; (query_in_entity_list (indices local_cases_map)) - ; (query_max_difference feature (get !cyclicFeaturesMap feature)) - ; )) - ; )) - ; (if (or (= (null) max_gap) (= .infinity max_gap) (= 0 max_gap)) - ; (assign (assoc max_gap (get !cachedFeatureMinResidualMap feature) )) - ; ) - ; ) - ; ) - (if (get feature_may_have_nulls_map feature) (let (assoc @@ -147,22 +132,6 @@ ;stores error into diff (call !InterpolateAndComputeDiffToCase) - ; (if mda - ; (seq - ; (if (= 0 diff) - ; (assign (assoc diff (get !cachedFeatureMinResidualMap feature) )) - ; ) - - ; ;output surprisal - ; (if feature_is_nominal - ; (- (log diff)) - - ; (/ max_gap diff) - ; ) - ; ) - - ; diff - ; ) )) (zip features) ) From b992e6119c08754d93e48bb7d9152f9468202303 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:19:11 -0500 Subject: [PATCH 11/29] fixes and test updates --- howso/analysis.amlg | 7 +----- howso/analysis_weights.amlg | 2 +- howso/react.amlg | 4 ++-- howso/react_utilities.amlg | 22 ------------------- howso/residuals.amlg | 13 ++++++++--- howso/synthesis.amlg | 4 ++-- howso/synthesis_utilities.amlg | 4 ++-- howso/types.amlg | 2 +- unit_tests/ut_h_dependent_features.amlg | 9 ++++---- unit_tests/ut_h_dynamic_deviations.amlg | 6 ++--- unit_tests/ut_h_null_null_react.amlg | 16 ++++++-------- unit_tests/ut_h_shared_deviations_series.amlg | 2 +- unit_tests/ut_h_unique_ids.amlg | 20 ++++------------- 13 files changed, 39 insertions(+), 72 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index efd8481d..058ea659 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -788,12 +788,7 @@ ) (assign (assoc - baseline_hyperparameter_map - (call !GetHyperparameters (assoc - feature action_feature - context_features context_features - weight_feature weight_feature - )) + baseline_hyperparameter_map (replace (retrieve_from_entity "!defaultHyperparameters")) )) ;if using default targetless hyperparameters, set p to 1 as the starting p value diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index 7c5e0a8e..c2252454 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -454,7 +454,7 @@ (if computing_mda (accum (assoc - analyzed_hp_map (assoc "feature_mda_map" (get residuals_map "feature_mda_map") ) + analyzed_hp_map (assoc "featureMdaMap" (get residuals_map "feature_mda_map") ) )) (seq diff --git a/howso/react.amlg b/howso/react.amlg index d5d693c1..ae15da66 100644 --- a/howso/react.amlg +++ b/howso/react.amlg @@ -1208,9 +1208,9 @@ k_parameter (get hyperparam_map "k") feature_deviations (get hyperparam_map "featureDeviations") feature_weights - (if (= (null) (get hyperparam_map "feature_mda_map")) + (if (= (null) (get hyperparam_map "featureMdaMap")) (get hyperparam_map "featureWeights") - (get hyperparam_map ["feature_mda_map" (first action_features)]) + (get hyperparam_map ["featureMdaMap" (first action_features)]) ) ;check if action_feature has nulls, if not, the no need to filter them out since they don't exist diff --git a/howso/react_utilities.amlg b/howso/react_utilities.amlg index 3652ace4..29999b4d 100644 --- a/howso/react_utilities.amlg +++ b/howso/react_utilities.amlg @@ -814,27 +814,5 @@ )) ) - ;if targetless, overwrite feature weights with weights derived from dynamic deviations - (if (= "targetless" (get hyperparam_map ["paramPath" 0])) - (assign (assoc - feature_weights - (append - ;original feature weights - feature_weights - ;assoc of feature weights based on local deviations - (map - (lambda - ;ensure that inactive features always maintain a feature weight of 0 - (if (contains_index !inactiveFeaturesMap (current_index)) - 0 - - 1 - ) - ) - local_deviations - ) - ) - )) - ) ) ) \ No newline at end of file diff --git a/howso/residuals.amlg b/howso/residuals.amlg index 85bfacfd..18382823 100644 --- a/howso/residuals.amlg +++ b/howso/residuals.amlg @@ -142,7 +142,14 @@ (/ (* 2 (current_value)) (+ 1 num_training_cases)) ;else set continuous lower residual bound to: gap / ln(num_cases + 1) - (/ (* 2 (current_value)) (log (+ 1 num_training_cases)) ) + (if (current_value) + (/ (* 2 (current_value)) (log (+ 1 num_training_cases)) ) + + ;if gap is 0, to prevent a 0 smallest residual, + ;Using n+1, n+2, or n+.5 are all possible considerations of Laplacian smoothing to apply a Bayesian approach for + ;estimating the probability of having no incorrect predictions. We chose .5 assuming the Jeffreys prior approach. + (/ 1 (+ 0.5 num_training_cases)) + ) ) ) feature_half_min_gap_map @@ -921,9 +928,9 @@ feature_is_non_string_edit_distance (false) feature_deviations (get hyperparam_map "featureDeviations") feature_weights - (if (= (null) (get hyperparam_map "feature_mda_map")) + (if (= (null) (get hyperparam_map "featureMdaMap")) (get hyperparam_map "featureWeights") - (get hyperparam_map ["feature_mda_map" (current_value 2)]) + (get hyperparam_map ["featureMdaMap" (current_value 2)]) ) needed_features (append context_features (get_value (current_value 1)) (if !tsTimeFeature [".series_index"] []) ) ) diff --git a/howso/synthesis.amlg b/howso/synthesis.amlg index 1ab2867d..91ee32ae 100644 --- a/howso/synthesis.amlg +++ b/howso/synthesis.amlg @@ -641,9 +641,9 @@ (declare (assoc feature_weights - (if (= (null) (get feature_hyperparam_map "feature_mda_map")) + (if (= (null) (get feature_hyperparam_map "featureMdaMap")) (get feature_hyperparam_map "featureWeights") - (get feature_hyperparam_map ["feature_mda_map" feature]) + (get feature_hyperparam_map ["featureMdaMap" feature]) ) feature_deviations (get feature_hyperparam_map "featureDeviations") )) diff --git a/howso/synthesis_utilities.amlg b/howso/synthesis_utilities.amlg index 7aa4b18c..1ee4641e 100644 --- a/howso/synthesis_utilities.amlg +++ b/howso/synthesis_utilities.amlg @@ -565,9 +565,9 @@ local_model_case_values_map (assoc) feature_weights - (if (= (null) (get hyperparam_map "feature_mda_map")) + (if (= (null) (get hyperparam_map "featureMdaMap")) (get hyperparam_map "featureWeights") - (get hyperparam_map ["feature_mda_map" action_feature]) + (get hyperparam_map ["featureMdaMap" action_feature]) ) feature_deviations (get hyperparam_map "featureDeviations") diff --git a/howso/types.amlg b/howso/types.amlg index 34a61b0a..2f175537 100644 --- a/howso/types.amlg +++ b/howso/types.amlg @@ -724,7 +724,7 @@ additional_indices "number" description "Map of features to their computed feature residuals." ) - feature_mda_map + featureMdaMap (assoc type "assoc" description "Map of each feature's map of feature weights." diff --git a/unit_tests/ut_h_dependent_features.amlg b/unit_tests/ut_h_dependent_features.amlg index 5e992836..96b373ca 100644 --- a/unit_tests/ut_h_dependent_features.amlg +++ b/unit_tests/ut_h_dependent_features.amlg @@ -1,6 +1,6 @@ (seq #unit_test (direct_assign_to_entities (assoc unit_test (load "unit_test.amlg"))) - (call (load "unit_test_howso.amlg") (assoc name "ut_h_dependent_features.amlg")) + (call (load "unit_test_howso.amlg") (assoc name "ut_h_dependent_features.amlg" retries 1)) (declare (assoc @@ -36,10 +36,12 @@ (list 19 0 1 "creatine" "mg/dL" 0.95) (list 70 180 3 (null) "ratio" 17) (list 70 190 3 (null) "ratio" 17.5) + (list 72 185 3 (null) "ratio" 17.5) (list 70 180 3 (null) "ratio" (null)) (list 80 200 1 "bmi" (null) 27) - (list 80 200 1 "bmi" (null) 32) + (list 80 225 1 "bmi" (null) 32) (list 80 230 1 "bmi" (null) (null)) + (list 80 228 1 "bmi" (null) (null)) ) )) @@ -78,8 +80,7 @@ (call_entity "howso" "analyze" (assoc context_features features targeted_model "targetless" - k_values (list 8) - use_deviations (false) + k_values [5] )) (print "Verify dependendents boundary map is properly set: \n") diff --git a/unit_tests/ut_h_dynamic_deviations.amlg b/unit_tests/ut_h_dynamic_deviations.amlg index 678acc9a..a0267f89 100644 --- a/unit_tests/ut_h_dynamic_deviations.amlg +++ b/unit_tests/ut_h_dynamic_deviations.amlg @@ -24,8 +24,8 @@ use_deviations (true) use_dynamic_deviations (true) dynamic_deviations_subtrainee_size 100 - p_values [ 2 ] - k_values [ 13 ] + p_values [ 1 ] + k_values [ 21 ] )) (declare (assoc @@ -48,7 +48,7 @@ (call assert_approximate (assoc exp 400 obs (get dd_agg_stats ["feature_residuals_full" "price"]) - thresh 300 + thresh 350 )) (call exit_if_failures (assoc msg "MAE with DD is not as expected.")) diff --git a/unit_tests/ut_h_null_null_react.amlg b/unit_tests/ut_h_null_null_react.amlg index 91ed9ec9..810d3ff7 100644 --- a/unit_tests/ut_h_null_null_react.amlg +++ b/unit_tests/ut_h_null_null_react.amlg @@ -40,9 +40,7 @@ )) (call_entity "howso" "analyze" (assoc - k_values (list 3) - p_values (list 0.5 1 2) - use_deviations (false) + k_values [3] )) (declare (assoc @@ -115,7 +113,7 @@ (call_entity "howso" "single_react" (assoc action_features act_features context_features ctx_features - context_values (list 30 (null) (null)) + context_values (list 35 (null) (null)) details (assoc "influential_cases" (true) ) )) )) @@ -125,20 +123,20 @@ obs (> (get result (list 1 "payload" "action_values" 0)) 45) )) - (print "Similar cases are not equal in influence:\n") + (print "Similar cases influences:\n") (call assert_approximate (assoc obs (get result (list 1 "payload" "influential_cases" 0 ".influence_weight")) - exp 0.42 + exp 0.35 thresh 0.1 )) (call assert_approximate (assoc obs (get result (list 1 "payload" "influential_cases" 1 ".influence_weight")) - exp 0.32 + exp 0.35 thresh 0.1 )) (call assert_approximate (assoc obs (get result (list 1 "payload" "influential_cases" 2 ".influence_weight")) - exp 0.26 + exp 0.30 thresh 0.1 )) @@ -224,7 +222,7 @@ obs result exp (assoc - missing_value_accuracy {B 0.875 C 0.75} + missing_value_accuracy {B 0.625 C 0.75} ) )) diff --git a/unit_tests/ut_h_shared_deviations_series.amlg b/unit_tests/ut_h_shared_deviations_series.amlg index 81c3042a..53b6016b 100644 --- a/unit_tests/ut_h_shared_deviations_series.amlg +++ b/unit_tests/ut_h_shared_deviations_series.amlg @@ -91,7 +91,7 @@ features features )) - (call_entity "howso" "analyze" (assoc use_deviations (true))) + (call_entity "howso" "analyze") (declare (assoc shared_feature_deviations diff --git a/unit_tests/ut_h_unique_ids.amlg b/unit_tests/ut_h_unique_ids.amlg index 5d8af3c3..6b9119d4 100644 --- a/unit_tests/ut_h_unique_ids.amlg +++ b/unit_tests/ut_h_unique_ids.amlg @@ -195,29 +195,17 @@ (lambda (keep (current_value) ["n_id" "s_id"]) ) - (keep result (list "accuracy" "mae" "mcc" "precision" "recall")) + (keep result (list "accuracy" "mae" "mcc" "precision" "recall")) ) )) - (call_entity "howso" "analyze" (assoc - use_deviations (true) - p_values (list 1) - )) + (call_entity "howso" "analyze") (assign (assoc result (call_entity "howso" "get_params") )) (call keep_result_payload) - (print "Weights for unique features are not 1: ") - (call assert_same (assoc - obs - (keep - (get result (list "hyperparameter_map" "targetless" "count.n_id.s_id.value." ".none" "featureWeights")) - (list "n_id" "s_id") - ) - ; 1 / 0.0392 = 25.5 - exp (assoc n_id 25.5 s_id 25.5) - )) + (print "Deviations for unique features are non-0, fixed to a small value: ") (call assert_approximate (assoc obs @@ -227,6 +215,6 @@ ) exp (assoc n_id 0.0392 s_id 0.0392) )) -; + (call exit_if_failures (assoc msg unit_test_name )) ) \ No newline at end of file From d6a1b067032a92d9d6cfda58ca29683e7e4282f4 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 10 Dec 2024 08:26:33 -0500 Subject: [PATCH 12/29] dd update --- howso/analysis.amlg | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 058ea659..58626c79 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -642,7 +642,12 @@ (call !InitResiduals) - (call !RunFullResiduals) + (if (= targeted_model "targetless") + (call !RunAccurateResiduals) + + ;targeted runs full residuals to match targeted flow + (call !RunFullResiduals) + ) (assign (assoc deviation_subtrainee_name From 21b5611c4b6e67181d782e412155fcad7be3cb9c Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:39:58 -0500 Subject: [PATCH 13/29] simplify code --- howso/mda_weight.amlg | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 8e159223..111e9675 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -275,17 +275,9 @@ (declare (assoc avg_without_residuals - (max - (get !cachedFeatureMinResidualMap mda_feature) - (/ (apply "+" without_feature_residuals) (size without_feature_residuals)) - (get !userSpecifiedFeatureErrorsMap mda_feature) - ) + (/ (apply "+" without_feature_residuals) (size without_feature_residuals)) avg_with_residuals - (max - (get !cachedFeatureMinResidualMap mda_feature) - (/ (apply "+" with_feature_residuals) (size with_feature_residuals)) - (get !userSpecifiedFeatureErrorsMap mda_feature) - ) + (/ (apply "+" with_feature_residuals) (size with_feature_residuals)) )) ;output ratio of the (expectedly) larger residual computed without this feature / residual computed with this feature (if (= 0 avg_without_residuals avg_with_residuals) From 6d1b319ad9bcec571507b8c1df437da576269257 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:40:08 -0500 Subject: [PATCH 14/29] update test --- unit_tests/ut_h_dynamic_deviations.amlg | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/unit_tests/ut_h_dynamic_deviations.amlg b/unit_tests/ut_h_dynamic_deviations.amlg index a0267f89..514405a6 100644 --- a/unit_tests/ut_h_dynamic_deviations.amlg +++ b/unit_tests/ut_h_dynamic_deviations.amlg @@ -1,7 +1,7 @@ (seq ;import the unit_test 'library' #unit_test (direct_assign_to_entities (assoc unit_test (load "unit_test.amlg"))) - (call (load "unit_test_howso.amlg") (assoc name "ut_h_dynamic_deviations.amlg")) + (call (load "unit_test_howso.amlg") (assoc name "ut_h_dynamic_deviations.amlg" retries 1)) (declare (assoc data (load "unit_test_data/longtail.csv") @@ -21,9 +21,8 @@ (call_entity "howso" "analyze" (assoc context_features ["log_price" "noise" "other_1" "other_2" "price"] targeted_model "targetless" - use_deviations (true) use_dynamic_deviations (true) - dynamic_deviations_subtrainee_size 100 + dynamic_deviations_subtrainee_size 1000 p_values [ 1 ] k_values [ 21 ] )) @@ -46,9 +45,9 @@ (print "Price MAE with Dynamic Deviations is as expected: ") (call assert_approximate (assoc - exp 400 + exp 325 obs (get dd_agg_stats ["feature_residuals_full" "price"]) - thresh 350 + thresh 100 )) (call exit_if_failures (assoc msg "MAE with DD is not as expected.")) From d41288cdb8c03c50bf2ca5755e35adfe6748a167 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:35:31 -0500 Subject: [PATCH 15/29] cleanup --- howso/mda_weight.amlg | 3 +-- howso/react_discriminative.amlg | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 111e9675..560868e3 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -89,8 +89,7 @@ interpolated_value 0 diff 0 output_categorical_action_probabilities (true) - categorical_action_probabilities_map (assoc)\ - max_gap (null) + categorical_action_probabilities_map (assoc) ) ;create the feature-specific candidate_cases_lists tuple for intepolation (declare (assoc diff --git a/howso/react_discriminative.amlg b/howso/react_discriminative.amlg index 2891896a..eeb7b720 100644 --- a/howso/react_discriminative.amlg +++ b/howso/react_discriminative.amlg @@ -217,7 +217,7 @@ (declare (assoc features (append action_features context_features) - ;assoc to store categorical action probabilities if requested, when output_categorical_action_probablities is set to true + ;assoc to store categorical action probabilities if requested, when output_categorical_action_probabilities is set to true ;both of these will be passed to methods inside the react flow implicitly categorical_action_probabilities_map (assoc) output_categorical_action_probabilities From fcb83096ae97a05aa67b5af9043695671c2f013a Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:12:52 -0500 Subject: [PATCH 16/29] 22395: Added confusion cap matrix --- howso/feature_residuals.amlg | 35 +++++++++++++++++++++++++++++++++-- howso/residuals.amlg | 24 ++++++++++++++---------- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index 97917996..b441c887 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -281,6 +281,7 @@ smape_map (assoc) mcc_map (assoc) confusion_matrix_map (assoc) + confusion_cap_map (assoc) nominal_performance_map (assoc) null_uncertainties_map (assoc) @@ -496,6 +497,9 @@ ;default index offset of where the actual case value is in each output react actual_offset 1 predicted_offset 2 + actual_nom_offset (if compute_all_statistics 2 1) + predicted_nom_offset (if compute_all_statistics 3 2) + cap_offset 1 )) ;if some of the features are edit distance types, r2 and spearman coefficient don't apply since there's no mean or relative sorting @@ -861,10 +865,10 @@ (append (replace (current_value 1)) (list (replace (current_value 1)))) ) ;(get (current_value) 1) refers to the actual case value in each react tuple - (map (lambda (replace (get (current_value) 1))) (current_value)) + (map (lambda (replace (get (current_value) actual_nom_offset))) (current_value)) ;and (get (current_value) 2) refers to the interpolated/predicted case value in ;each react tuple - (map (lambda (replace (get (current_value) 2))) (current_value)) + (map (lambda (replace (get (current_value) predicted_nom_offset))) (current_value)) ) ) ) @@ -875,6 +879,31 @@ ) )) + (assign (assoc + confusion_cap_map + (zip + nominal_features + (map + (lambda + (zip + (lambda + (map + (lambda + (apply "+" (filter (current_value))) + ) + (current_value 1) + (current_value) + ) + ) + (map (lambda (get (current_value) actual_nom_offset) ) (current_value)) + (map (lambda (get (current_value) cap_offset) ) (current_value)) + ) + ) + (unzip feature_residuals_lists nominal_indices) + ) + ) + )) + ;accuracy is: num correct predictions / all predictions (assign (assoc accuracy_map @@ -1147,6 +1176,7 @@ "rmse_map" rmse_map "spearman_coeff_map" spearman_coeff_map "confusion_matrix_map" confusion_matrix_map + "confusion_cap_map" confusion_cap_map ) ) ) @@ -1185,6 +1215,7 @@ "adjusted_smape" adjusted_smape_map "spearman_coeff" spearman_coeff_map "confusion_matrix" confusion_matrix_map + "confusion_cap_map" confusion_cap_map ) ) diff --git a/howso/residuals.amlg b/howso/residuals.amlg index 214c98b3..6053a12e 100644 --- a/howso/residuals.amlg +++ b/howso/residuals.amlg @@ -1127,12 +1127,8 @@ compute_all_statistics (if feature_is_nominal - ;output [diff, actual, predicted] for computing MAE, accuracy, precision, recall - [ - (call !ComputeFeatureMAEDiff) - case_feature_value - interpolated_value - ] + ;output [diff, cap, actual, predicted] for computing MAE, accuracy, precision, recall + (append (call !ComputeFeatureMAEDiff) case_feature_value interpolated_value) ;else continuous, output [diff, ordinal_diff, actual, predicted] for computing MAE, RSME, R^2 (and @@ -1184,7 +1180,12 @@ ;output the residual. if it's (null), output null for non-nominals, and 1 (or case weight) for nominals (if (= (null) diff) - (if feature_is_nominal 1) + (if feature_is_nominal + (if compute_all_statistics + [1 (filter (lambda (> (current_value) 0.01)) (get categorical_action_probabilities_map feature))] + 1 + ) + ) ;else output tuple [diff, ordinal_diff] for ordinals, and diff for continuous if applicable (contains_index ordinal_features_map feature) @@ -1200,11 +1201,15 @@ ) ;output the tuple - (list diff ordinal_diff) + [diff ordinal_diff] ) ;else output it as-is - diff + (if (and feature_is_nominal compute_all_statistics) + [diff (filter (lambda (> (current_value) 0.01)) (get categorical_action_probabilities_map feature))] + + diff + ) ) ) ) @@ -1323,7 +1328,6 @@ ) ) - (if (and !tsTimeFeature (contains_index react_context_features !tsTimeFeature) From 57c1a4738928bc9af376b392f29f9391284a9687 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 11 Dec 2024 21:37:45 -0500 Subject: [PATCH 17/29] add cap matrix --- howso/feature_residuals.amlg | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index b4495e25..7b01789b 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -1176,6 +1176,16 @@ confusion_matrix_map ) ) + confusion_cap_map + (map + (lambda (assoc + "matrix" (current_value 1) + "leftover_correct" 0 + "leftover_incorrect" 0 + "other_counts" 0 + )) + confusion_cap_map + ) )) From 183bad6a5fe0d9b02cb9ea7810ac8e1c0b1f4099 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:56:08 -0500 Subject: [PATCH 18/29] cleanup --- howso/analysis.amlg | 1 - howso/analysis_weights.amlg | 80 ++++++++++++++++++------------------- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index a40a3fc9..d45b03a0 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -532,7 +532,6 @@ (if inverse_residuals_as_weights (conclude (seq (call !ConvergeIRW (assoc use_deviations (true))) ;iterations of ComputeResiduals and GridSearch - (call !BackupAnalyzedHyperparameters) ;IRW always needs subtrainee (if (and use_dynamic_deviations (not !useDynamicDeviationsDuringAnalyze)) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index afb17d05..8d687ac8 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -298,7 +298,7 @@ (declare (assoc use_deviations (false) - num_iterations 4 + num_iterations 3 num_samples_mda 10000 ;local method variables, not parameters @@ -413,7 +413,7 @@ ;compute mda on 2nd to last iteration (assign (assoc - computing_mda (= iteration (- num_iterations 2)) + computing_mda (= iteration 2) )) (assign (assoc @@ -456,52 +456,48 @@ (if computing_mda (accum (assoc - analyzed_hp_map (assoc "featureMdaMap" (get residuals_map "feature_mda_map") ) + baseline_hyperparameter_map (assoc "featureMdaMap" (get residuals_map "feature_mda_map") ) )) - (seq - (call !UpdateHyperparameters (assoc - feature_deviations (get residuals_map "residual_map") - ;no SDM - ;confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_matrix")) ) - ordinal_feature_deviations (get residuals_map "ordinal_residual_map") - null_deviations (get residuals_map "null_uncertainty_map") - use_deviations use_deviations - feature_weights - (map - (lambda - ;ensure that inactive features always maintain a feature weight of 0 - (if (contains_index !inactiveFeaturesMap (current_index)) - 0 + (call !UpdateHyperparameters (assoc + feature_deviations (get residuals_map "residual_map") + ;no SDM + ;confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_cap_map")) ) + ordinal_feature_deviations (get residuals_map "ordinal_residual_map") + null_deviations (get residuals_map "null_uncertainty_map") + use_deviations use_deviations + feature_weights + (map + (lambda + ;ensure that inactive features always maintain a feature weight of 0 + (if (contains_index !inactiveFeaturesMap (current_index)) + 0 - 1 - ) + 1 ) - (get residuals_map "residual_map") ) - )) - - (assign (assoc - analyzed_hp_map - (call !ComputeResidualsAcrossParametersAndSelectOptimal (assoc - context_features context_features - action_features action_features - k_folds k_folds - k_values k_values - p_values (filter (lambda (!= 0 (current_value))) p_values) - dt_values dt_values - k_folds_by_indices k_folds_by_indices - num_analysis_samples num_analysis_samples - targetless (= targeted_model "targetless") - baseline_hyperparameter_map baseline_hyperparameter_map - ;this flag sets the weights to be inverse of the residuals and outputs them - use_inverse_weights (true) - )) - )) - ) + (get residuals_map "residual_map") + ) + )) ) - (assign (assoc baseline_hyperparameter_map analyzed_hp_map)) + (assign (assoc + baseline_hyperparameter_map + (call !ComputeResidualsAcrossParametersAndSelectOptimal (assoc + context_features context_features + action_features action_features + k_folds k_folds + k_values k_values + p_values (filter (lambda (!= 0 (current_value))) p_values) + dt_values dt_values + k_folds_by_indices k_folds_by_indices + num_analysis_samples num_analysis_samples + targetless (= targeted_model "targetless") + baseline_hyperparameter_map baseline_hyperparameter_map + ;this flag sets the weights to be inverse of the residuals and outputs them + use_inverse_weights (true) + )) + )) (accum (assoc iteration 1)) @@ -515,7 +511,7 @@ !useDynamicDeviationsDuringAnalyze ) ; store feature residuals for a collection of cases at end, if it wasn't part of iterative process - (call !StoreResidualSubTrainee (assoc custom_hyperparam_map analyzed_hp_map) ) + (call !StoreResidualSubTrainee (assoc custom_hyperparam_map baseline_hyperparameter_map) ) ) ) ) From 7425a1be30cd356ece0d5cd156b14fc77b38b05d Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:27:58 -0500 Subject: [PATCH 19/29] 22357: mda weights with cap sdm, WIP --- howso/analysis_weights.amlg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index 8d687ac8..5d27bc1c 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -461,8 +461,7 @@ (call !UpdateHyperparameters (assoc feature_deviations (get residuals_map "residual_map") - ;no SDM - ;confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_cap_map")) ) + confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_cap_map")) ) ordinal_feature_deviations (get residuals_map "ordinal_residual_map") null_deviations (get residuals_map "null_uncertainty_map") use_deviations use_deviations From 027f8496eaa9a3f6eb3c42bc6bc4d95faeb2ce66 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:22:52 -0500 Subject: [PATCH 20/29] converge on deviations --- howso/analysis_weights.amlg | 40 +++++++++++++++++++------------------ howso/hyperparameters.amlg | 1 + 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index 5d27bc1c..aab1631d 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -298,7 +298,7 @@ (declare (assoc use_deviations (false) - num_iterations 3 + num_iterations 4 num_samples_mda 10000 ;local method variables, not parameters @@ -413,7 +413,7 @@ ;compute mda on 2nd to last iteration (assign (assoc - computing_mda (= iteration 2) + computing_mda (= iteration 3) )) (assign (assoc @@ -480,23 +480,25 @@ )) ) - (assign (assoc - baseline_hyperparameter_map - (call !ComputeResidualsAcrossParametersAndSelectOptimal (assoc - context_features context_features - action_features action_features - k_folds k_folds - k_values k_values - p_values (filter (lambda (!= 0 (current_value))) p_values) - dt_values dt_values - k_folds_by_indices k_folds_by_indices - num_analysis_samples num_analysis_samples - targetless (= targeted_model "targetless") - baseline_hyperparameter_map baseline_hyperparameter_map - ;this flag sets the weights to be inverse of the residuals and outputs them - use_inverse_weights (true) - )) - )) + (if (>= iteration 2) + (assign (assoc + baseline_hyperparameter_map + (call !ComputeResidualsAcrossParametersAndSelectOptimal (assoc + context_features context_features + action_features action_features + k_folds k_folds + k_values k_values + p_values (filter (lambda (!= 0 (current_value))) p_values) + dt_values dt_values + k_folds_by_indices k_folds_by_indices + num_analysis_samples num_analysis_samples + targetless (= targeted_model "targetless") + baseline_hyperparameter_map baseline_hyperparameter_map + ;this flag sets the weights to be inverse of the residuals and outputs them + use_inverse_weights (true) + )) + )) + ) (accum (assoc iteration 1)) diff --git a/howso/hyperparameters.amlg b/howso/hyperparameters.amlg index 7f8a850c..6e45926d 100644 --- a/howso/hyperparameters.amlg +++ b/howso/hyperparameters.amlg @@ -758,6 +758,7 @@ "dt" -1 "featureWeights" (null) "featureDeviations" (null) + "paramPath" [".default"] ) )) (accum_to_entities (assoc !revision 1)) From 84bfbb474355825181ba1fd0db1f1c48e2b8c192 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:23:08 -0500 Subject: [PATCH 21/29] bank test cleanup --- performance_tests/bank_test.amlg | 8 -------- 1 file changed, 8 deletions(-) diff --git a/performance_tests/bank_test.amlg b/performance_tests/bank_test.amlg index 5f24e955..28495541 100644 --- a/performance_tests/bank_test.amlg +++ b/performance_tests/bank_test.amlg @@ -11,10 +11,6 @@ (declare (assoc test_start (system_time))) - (call_entity "howso" "set_params" (assoc - default_hyperparameter_map (assoc "k" 8 "p" 2 "dt" -1 ) - )) - (declare (assoc ;set to true to see full output verbose (false) @@ -34,12 +30,8 @@ features (first data) training_data (tail data) start (system_time) - k 8 - p 2 )) - (call_entity "howso" "set_random_seed" (assoc seed "abcdef")) - (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc From a6ed1ed1ef1ca812d6b469eeb51a3e008e619477 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:34:45 -0500 Subject: [PATCH 22/29] removed irw flags and references to inverse feature weighting --- howso/analysis.amlg | 93 +---------- howso/analysis_weights.amlg | 46 +++--- howso/details.amlg | 2 +- howso/feature_residuals.amlg | 156 +++++++++---------- howso/hyperparameters.amlg | 7 +- howso/react.amlg | 3 +- howso/train.amlg | 1 - howso/types.amlg | 4 +- unit_tests/ut_h_analyze_feature_weights.amlg | 1 - unit_tests/ut_h_id_features.amlg | 1 - unit_tests/ut_h_input_validation.amlg | 1 - 11 files changed, 102 insertions(+), 213 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index d45b03a0..098db0be 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -74,10 +74,6 @@ ;when true will scale influence weights by each case's weight_feature weight. if use_case_weights isn't specified, it will ; be true if auto ablation is enabled and false otherwise use_case_weights (null) - ;{type ["boolean" "null"]} - ;whether to use inverse residuals as feature weights. If unspecified, inverse residuals will be used as weights for - ; targetless params, otherwise this method will not be used. - inverse_residuals_as_weights (null) ;{type "list" values "string"} ;list of features whose values to use to rebalance case weighting of the data and to store into weight_feature ;cannot be used when auto ablation is enabled @@ -164,7 +160,6 @@ "targeted_model" targeted_model "num_analysis_samples" num_analysis_samples "analysis_sub_model_size" analysis_sub_model_size - "inverse_residuals_as_weights" inverse_residuals_as_weights "use_case_weights" use_case_weights "weight_feature" weight_feature "rebalance_features" rebalance_features @@ -255,7 +250,6 @@ k_folds_by_indices k_folds_by_indices targeted_model targeted_model num_analysis_samples num_analysis_samples - inverse_residuals_as_weights inverse_residuals_as_weights residual_num_samples (if (> num_samples 0) num_samples 200) use_case_weights use_case_weights weight_feature weight_feature @@ -482,8 +476,6 @@ ; "targetless" = analyze hyperparameters for all context features as possible action features, ignores action_features parameter ; context_features: list of context features to analyze for ; action_features: list of action features to analyze for - ; inverse_residuals_as_weights : optional, default is null, will be set to false for targeted and true for targetless - ; when true will forcibly compute and use inverse of residuals as feature weights ; num_analysis_samples: optional. number of cases to sample during analysis. only applies for k_folds = 1 ; residual_num_samples: optional. initial number of samples to use for computing residuals ; derived_auto_analyzed: optional, if true, will set the 'derivedAutoAnalyzed' parameter to true in the hyperparameter set, @@ -529,9 +521,9 @@ use_deviations (false) )) - (if inverse_residuals_as_weights + (if (= "targetless" targeted_model) (conclude (seq - (call !ConvergeIRW (assoc use_deviations (true))) ;iterations of ComputeResiduals and GridSearch + (call !ConvergeTargetless (assoc use_deviations (true))) ;IRW always needs subtrainee (if (and use_dynamic_deviations (not !useDynamicDeviationsDuringAnalyze)) @@ -785,11 +777,6 @@ ) ) - ;if inverse_residuals_as_weights isn't defined, default it to true for targetless, false for targeted - (if (= (null) inverse_residuals_as_weights) - (assign (assoc inverse_residuals_as_weights (= targeted_model "targetless") )) - ) - (assign (assoc baseline_hyperparameter_map (replace (retrieve_from_entity "!defaultHyperparameters")) )) @@ -923,7 +910,6 @@ ; targetless : optional flag, if true will randomly select a context feature as an action feature for each case during grid search ; num_analysis_samples : optional. number of cases to sample during analysis. only applies for k_folds = 1 ; baseline_hyperparameter_map : the base hyperparameters to use for computation - ; use_inverse_weights: if true, will compute inverse residual weights (IRW) during the grid search for use in error computations #!ComputeResidualsAcrossParametersAndSelectOptimal (declare (assoc @@ -942,7 +928,6 @@ targetless (false) num_analysis_samples (null) baseline_hyperparameter_map (assoc) - use_inverse_weights (false) ) ;can't do analysis if there is a max of one context feature provided and no different action features @@ -1198,31 +1183,7 @@ (append baseline_hyperparameter_map - ;overwrite with the autotuned k and p values - (if (not use_inverse_weights) - output_map - - ;else compute and output the corresponding IRW with the parameters - (append - output_map - (assoc - "featureWeights" - ;set inverted residuals to be 1 / (residual^p), unless it's 0 (or within floating point error of 0) - (map - (lambda - ;ensure that inactive features always maintain a feature weight of 0 - (if (contains_index !inactiveFeaturesMap (current_index)) - 0 - - 1 - ) - - ) - (get baseline_hyperparameter_map "featureResiduals") - ) - ) - ) - ) + output_map ) ) @@ -1345,36 +1306,7 @@ #!AccumulateErrorsViaGridSearch (map (lambda (let - (assoc - p_value (current_value 1) - inverted_residuals_map (null) - ) - - ;compute IRW if residuals for IRW map was provided - (if use_inverse_weights - (assign (assoc - ;set inverted residuals to be 1 / (residual^p), unless it's 0 (or within floating point error of 0) - ;in which case set it to 1 as to not affect the feature since it's already accurate - ;each feature's residual value will be on the same scale as the feature itself, e.g., for large feature values - ;like billions, a residual of a few percent will be in the tens of millions, for tiny feature values, the - ;residual values will also be tiny. Thus deviding each feature by its residual scales the large values down - ;and small values up. If the residual is within an order or two of magnitude, this weighing still - ;effectively normalizes the data. Relatively large residuals also scale the values smaller, - ;decreasing the effect of features that are noisy and hard to predict. - inverted_residuals_map - (map - (lambda - ;ensure that inactive features always maintain a feature weight of 0 - (if (contains_index !inactiveFeaturesMap (current_index)) - 0 - - 1 - ) - ) - (get baseline_hyperparameter_map "featureResiduals") - ) - )) - ) + (assoc p_value (current_value 1)) (map (lambda (let @@ -1386,22 +1318,9 @@ mae_hyperparam_map baseline_hyperparameter_map ) - ;overwrite the k/p/dt and inverse weights and deviations if appropriate + ;overwrite the k/p/dt (accum (assoc - mae_hyperparam_map - (append - (assoc "p" p_value "k" k_value "dt" dt_value) - - ;if using inverse weights, overwrite existing weights in baseline_hyperparameter_map - (if use_inverse_weights - (assoc - "featureWeights" inverted_residuals_map - "featureDeviations" (get baseline_hyperparameter_map "featureDeviations") - ) - - (assoc) - ) - ) + mae_hyperparam_map (assoc "p" p_value "k" k_value "dt" dt_value) )) ;accumulate the accuracy distance for each set of parameters diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index aab1631d..ef3c2428 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -213,7 +213,7 @@ weights_map ) - ;helper function for the residuals computing methods used within Analysis (ConvergeResiduals and ConvergeIRW) + ;helper function for the residuals computing methods used within Analysis (ConvergeResiduals and ConvergeTargetless) ;estimates the number of samples necessary to allow for the correct amount of samples for a feature to be predicted ;across all of its features that share deviations ;parameters: @@ -293,11 +293,10 @@ ) ;updates analyzed_hp_map - ;run multiple iterations of grid search and residuals to either use with the inverse_residuals_as_weights flow - #!ConvergeIRW + ;run multiple iterations of grid search and residuals + #!ConvergeTargetless (declare (assoc - use_deviations (false) num_iterations 4 num_samples_mda 10000 @@ -308,7 +307,7 @@ (set_rand_seed sampling_random_seed) - (call !UpdateHyperparameters (assoc use_deviations use_deviations)) + (call !UpdateHyperparameters (assoc use_deviations (true) )) (declare (assoc features @@ -405,6 +404,8 @@ )) ) + (declare (assoc in_analyze (true) )) + ;First pass uses inverse feature gap/2 values as weights to compute hyperparameters that should provide decent results. ;Second pass uses these decent residuals to finds even better hyperparmeters and improved residuals. ;Last pass finds usable hyperparams using the improved residuals and then calculates usable residuals for weights. @@ -443,7 +444,7 @@ ) custom_hyperparam_map baseline_hyperparameter_map ;must compute confusion matrix to use sparse deviation matrix - compute_all_statistics use_deviations + compute_all_statistics (true) ;don't sparsify the confusion matrix so that SDM can be computed using full counts confusion_matrix_min_count 0 use_shared_deviations (true) @@ -461,21 +462,22 @@ (call !UpdateHyperparameters (assoc feature_deviations (get residuals_map "residual_map") - confusion_matrix_map (if use_deviations (get residuals_map (list "prediction_stats" "confusion_cap_map")) ) + confusion_matrix_map (get residuals_map ["prediction_stats" "confusion_matrix"]) ordinal_feature_deviations (get residuals_map "ordinal_residual_map") null_deviations (get residuals_map "null_uncertainty_map") - use_deviations use_deviations + use_deviations (true) feature_weights - (map - (lambda - ;ensure that inactive features always maintain a feature weight of 0 - (if (contains_index !inactiveFeaturesMap (current_index)) - 0 - - 1 + (if (size !inactiveFeaturesMap) + (map + (lambda + ;ensure that inactive features always maintain a feature weight of 0 + (if (contains_index !inactiveFeaturesMap (current_index)) + 0 + 1 + ) ) + (get residuals_map "residual_map") ) - (get residuals_map "residual_map") ) )) ) @@ -494,23 +496,13 @@ num_analysis_samples num_analysis_samples targetless (= targeted_model "targetless") baseline_hyperparameter_map baseline_hyperparameter_map - ;this flag sets the weights to be inverse of the residuals and outputs them - use_inverse_weights (true) )) )) ) (accum (assoc iteration 1)) - (if (and - use_dynamic_deviations - ;should store if needed for deviations or IRW feature weights - (or - use_deviations - (= targeted_model "targetless") - ) - !useDynamicDeviationsDuringAnalyze - ) + (if (and use_dynamic_deviations !useDynamicDeviationsDuringAnalyze) ; store feature residuals for a collection of cases at end, if it wasn't part of iterative process (call !StoreResidualSubTrainee (assoc custom_hyperparam_map baseline_hyperparameter_map) ) ) diff --git a/howso/details.amlg b/howso/details.amlg index 43d19dec..bd10f20c 100644 --- a/howso/details.amlg +++ b/howso/details.amlg @@ -159,7 +159,7 @@ feature_deviations (zip (append context_features action_features) 0) ) - "use_irw" (= "targetless" (get hyperparam_map (list "paramPath" 0))) + "targetless" (= "targetless" (get hyperparam_map (list "paramPath" 0))) "k" k_parameter "p" p_parameter "distance_transform" dt_parameter diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index 7b01789b..371dc32d 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -282,7 +282,6 @@ smape_map (assoc) mcc_map (assoc) confusion_matrix_map (assoc) - confusion_cap_map (assoc) nominal_performance_map (assoc) null_uncertainties_map (assoc) @@ -847,91 +846,92 @@ (assign (assoc confusion_matrix_map - (zip - nominal_features - ;iterate over nominal features, where (current_value) is a list of react tuples for each feature - (map - (lambda - (map - (lambda - (zip - ;all collisions will increase the count value - ;e.g. zipping ["small" "medium" "small"] will create { small 1 medium 1 } then accumulate a 1 - ;for the second "small" in the list since it's a key collision, resulting in { small 2 medium 1 } - (lambda (+ 1 (current_value 1)) ) - ;ensure to zip a list of classes instead of single value by converting it to a list if it isn't a list already - (if (~ [] (current_value)) - (current_value) - [(current_value 1)] - ) - ;zip all classes with a value of 1 so that all clases in the predicted list are assigned a value of 1 - (range - (lambda 1) - 1 - ;if there is only one entry for a value, then it's predicted values: (current_value) - ;will be singular (not a list, size=0), so we bound the list of 1's to be at least size=1 - (max 1 (size (current_value))) - 1 - ) - ) - ) - ;keep only those class values that had non-null counts - (filter - (lambda (size (current_value))) - ;nominal prediction tuples are in the format of [diff, actual, predicted] - ;thus this iterates over the two corresponding lists of [actual] and [predicted] values - ;accruing into an assoc of lists of predicted classes for every provided actual class, e.g., - ;given actual ["a" "a" "x" "x" "x"] and predicted ["a" "a" "x" "a" "x"] results in: - ; { - ; "a" [ "a" a" ] - ; "x" [ "x "a" "x"] - ; } - (map - ;filter out the nulls out of the list of predicted values - (lambda (filter (replace (current_value)))) - (zip - ;iterate over pairs of (actual value, predicted value), this lambda is run when there is a - ;key collision (actual value). In that case append the new predicted value to get an assoc of - ; actual values to the list of values they are predicted as + ;analyze uses confusion matrix to compute SDM, use Categorical Action Probabilities for higher fidelity of nominal residuals + (if in_analyze + (zip + nominal_features + (map + (lambda + (zip + (lambda + (map (lambda - (append (replace (current_value 1)) (list (replace (current_value 1)))) + (apply "+" (filter (current_value))) ) - ;(get (current_value) 1) refers to the actual case value in each react tuple - (map (lambda (replace (get (current_value) actual_nom_offset))) (current_value)) - ;and (get (current_value) 2) refers to the interpolated/predicted case value in - ;each react tuple - (map (lambda (replace (get (current_value) predicted_nom_offset))) (current_value)) + (current_value 1) + (current_value) ) ) + (map (lambda (get (current_value) actual_nom_offset) ) (current_value)) + (map (lambda (get (current_value) cap_offset) ) (current_value)) ) ) + (unzip feature_residuals_lists nominal_indices) ) - (unzip feature_residuals_lists nominal_indices) ) - ) - )) - (assign (assoc - confusion_cap_map - (zip - nominal_features - (map - (lambda - (zip - (lambda - (map - (lambda - (apply "+" (filter (current_value))) + ;else compute confusion matrix using counts of correct guesses for each class + (zip + nominal_features + ;iterate over nominal features, where (current_value) is a list of react tuples for each feature + (map + (lambda + (map + (lambda + (zip + ;all collisions will increase the count value + ;e.g. zipping ["small" "medium" "small"] will create { small 1 medium 1 } then accumulate a 1 + ;for the second "small" in the list since it's a key collision, resulting in { small 2 medium 1 } + (lambda (+ 1 (current_value 1)) ) + ;ensure to zip a list of classes instead of single value by converting it to a list if it isn't a list already + (if (~ [] (current_value)) + (current_value) + [(current_value 1)] + ) + ;zip all classes with a value of 1 so that all clases in the predicted list are assigned a value of 1 + (range + (lambda 1) + 1 + ;if there is only one entry for a value, then it's predicted values: (current_value) + ;will be singular (not a list, size=0), so we bound the list of 1's to be at least size=1 + (max 1 (size (current_value))) + 1 + ) + ) + ) + ;keep only those class values that had non-null counts + (filter + (lambda (size (current_value))) + ;nominal prediction tuples are in the format of [diff, actual, predicted] + ;thus this iterates over the two corresponding lists of [actual] and [predicted] values + ;accruing into an assoc of lists of predicted classes for every provided actual class, e.g., + ;given actual ["a" "a" "x" "x" "x"] and predicted ["a" "a" "x" "a" "x"] results in: + ; { + ; "a" [ "a" a" ] + ; "x" [ "x "a" "x"] + ; } + (map + ;filter out the nulls out of the list of predicted values + (lambda (filter (replace (current_value)))) + (zip + ;iterate over pairs of (actual value, predicted value), this lambda is run when there is a + ;key collision (actual value). In that case append the new predicted value to get an assoc of + ; actual values to the list of values they are predicted as + (lambda + (append (replace (current_value 1)) (list (replace (current_value 1)))) + ) + ;(get (current_value) 1) refers to the actual case value in each react tuple + (map (lambda (replace (get (current_value) actual_nom_offset))) (current_value)) + ;and (get (current_value) 2) refers to the interpolated/predicted case value in + ;each react tuple + (map (lambda (replace (get (current_value) predicted_nom_offset))) (current_value)) + ) ) - (current_value 1) - (current_value) ) ) - (map (lambda (get (current_value) actual_nom_offset) ) (current_value)) - (map (lambda (get (current_value) cap_offset) ) (current_value)) ) + (unzip feature_residuals_lists nominal_indices) ) - (unzip feature_residuals_lists nominal_indices) ) ) )) @@ -1176,16 +1176,6 @@ confusion_matrix_map ) ) - confusion_cap_map - (map - (lambda (assoc - "matrix" (current_value 1) - "leftover_correct" 0 - "leftover_incorrect" 0 - "other_counts" 0 - )) - confusion_cap_map - ) )) @@ -1218,7 +1208,6 @@ "rmse_map" rmse_map "spearman_coeff_map" spearman_coeff_map "confusion_matrix_map" confusion_matrix_map - "confusion_cap_map" confusion_cap_map ) ) ) @@ -1257,7 +1246,6 @@ "adjusted_smape" adjusted_smape_map "spearman_coeff" spearman_coeff_map "confusion_matrix" confusion_matrix_map - "confusion_cap_map" confusion_cap_map ) ) diff --git a/howso/hyperparameters.amlg b/howso/hyperparameters.amlg index 6e45926d..e620e2a0 100644 --- a/howso/hyperparameters.amlg +++ b/howso/hyperparameters.amlg @@ -97,7 +97,7 @@ ;if clearing weights, set them to null (if (= (false) use_weights) ;if there are inactive features, we have to reset active features to 1s and inactives to 0s - (if !inactiveFeaturesMap + (if (size !inactiveFeaturesMap) (assoc "featureWeights" (append (zip @@ -834,10 +834,6 @@ ;when true will scale influence weights by each case's weight_feature weight. if use_case_weights isn't specified, it will ; be true if auto ablation is enabled and false otherwise use_case_weights (null) - ;{type ["boolean" "null"]} - ;whether to use inverse residuals as feature weights. If unspecified, inverse residuals will be used as weights for - ; targetless params, otherwise this method will not be used. - inverse_residuals_as_weights (null) ;{type "list" values "string"} ;list of features whose values to use to rebalance case weighting of the data and to store into weight_feature ;cannot be used when auto ablation is enabled @@ -909,7 +905,6 @@ "targeted_model" targeted_model "num_analysis_samples" num_analysis_samples "analysis_sub_model_size" analysis_sub_model_size - "inverse_residuals_as_weights" inverse_residuals_as_weights "weight_feature" weight_feature "use_case_weights" use_case_weights "rebalance_features" rebalance_features diff --git a/howso/react.amlg b/howso/react.amlg index ae15da66..b9711031 100644 --- a/howso/react.amlg +++ b/howso/react.amlg @@ -86,8 +86,7 @@ ; Not applicable to generative reacts. ; ; "derivation_parameters" true or false. If true outputs the parameters used during the react. These parameters - ; include k, p, distance_transform, feature_weights, feature_deviations, nominal_class_counts, and flags to indicate - ; if deviations or inverse residual weighting were used. + ; include k, p, distance_transform, feature_weights, feature_deviations. ; ; "hypothetical_values" assoc context feature -> values. If specified, shows how a prediction could change ; in a what-if scenario where the influential cases' context feature values are replaced with diff --git a/howso/train.amlg b/howso/train.amlg index 0b7df8ac..ea34a423 100644 --- a/howso/train.amlg +++ b/howso/train.amlg @@ -1020,7 +1020,6 @@ ".none" ) use_case_weights (and !autoAblationEnabled !autoAblationWeightFeature) - inverse_residuals_as_weights (true) k_folds 1 )) ) diff --git a/howso/types.amlg b/howso/types.amlg index d200308a..d7533dbf 100644 --- a/howso/types.amlg +++ b/howso/types.amlg @@ -1680,9 +1680,9 @@ ref "FeatureMetricIndex" description "A map of nominal feature names to the number of unique classes for each feature." } - "use_irw" + "targetless" { - type "boolean" description "Flag indicating if feature weights are derived using inverse-residual weighting." + type "boolean" description "Flag indicating if parameters were derived using a targetless approach." } } ) diff --git a/unit_tests/ut_h_analyze_feature_weights.amlg b/unit_tests/ut_h_analyze_feature_weights.amlg index cda56935..c00494a3 100644 --- a/unit_tests/ut_h_analyze_feature_weights.amlg +++ b/unit_tests/ut_h_analyze_feature_weights.amlg @@ -127,7 +127,6 @@ k_values (list 5 8 13) k_folds 1 num_analysis_samples 1000 - inverse_residuals_as_weights (true) )) (declare (assoc diff --git a/unit_tests/ut_h_id_features.amlg b/unit_tests/ut_h_id_features.amlg index d0046acf..e151413b 100644 --- a/unit_tests/ut_h_id_features.amlg +++ b/unit_tests/ut_h_id_features.amlg @@ -157,7 +157,6 @@ (call_entity "howso" "analyze" (assoc context_features features targeted_model "targetless" - inverse_residuals_as_weights (true) use_case_weights (true) )) (assign (assoc diff --git a/unit_tests/ut_h_input_validation.amlg b/unit_tests/ut_h_input_validation.amlg index f5fbfde2..5e2588cc 100644 --- a/unit_tests/ut_h_input_validation.amlg +++ b/unit_tests/ut_h_input_validation.amlg @@ -134,7 +134,6 @@ (call_entity "howso" "analyze" (assoc context_features (list "x" "y" "z") targeted_model "targetless" - inverse_residuals_as_weights (true) )) From 4f61b84093231c34106ec34d286e56f354b285c5 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:48:18 -0500 Subject: [PATCH 23/29] renames --- howso/analysis.amlg | 4 ++-- howso/analysis_weights.amlg | 2 +- howso/feature_residuals.amlg | 8 ++++---- howso/mda_weight.amlg | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 098db0be..9ce4cebf 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -523,7 +523,7 @@ (if (= "targetless" targeted_model) (conclude (seq - (call !ConvergeTargetless (assoc use_deviations (true))) + (call !ConvergeTargetless) ;IRW always needs subtrainee (if (and use_dynamic_deviations (not !useDynamicDeviationsDuringAnalyze)) @@ -634,7 +634,7 @@ (call !InitResiduals) (if (= targeted_model "targetless") - (call !RunAccurateResiduals) + (call !ComputeDeviations) ;targeted runs full residuals to match targeted flow (call !RunFullResiduals) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index ef3c2428..a84d0262 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -436,7 +436,7 @@ feature_residuals_map (call !CalculateFeatureResiduals (assoc features features - robust_residuals (if computing_mda "robust_mda" "superfull") + robust_residuals (if computing_mda "robust_mda" "deviations") num_samples (if computing_mda num_samples_mda diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index 371dc32d..db67f7ee 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -207,7 +207,7 @@ ; will be excluded from the context set, effectively holding them out. If only 'action_condition_filter_query' is specified, then only the single predicted case will be left out. ; focal_case: optional case id of case for which to compute residuals for is to be ignored during computation ; regional_model_only: flag, default to false. when set to true will only explicitly use the specified case_ids for computation. - ; robust_residuals: string, default to (null). when "superfull" calculates accurate local residuals, otherwise computes residuals robust with respect to the set of contexts used + ; robust_residuals: string, default to (null). when "deviations" calculates accurate local residuals, otherwise computes residuals robust with respect to the set of contexts used ; (across the power set of contexts). when "robust_mda" will additionally compute and output MDA using robust residuals. ; hyperparameter_feature: optional default (null). feature whose hyperparameters to use ; use_case_weights: flag, if set to true will scale influence weights by each case's weight_feature weight. If unspecified, @@ -299,8 +299,8 @@ (call !InitResiduals) - (if (= "superfull" robust_residuals) - (call !RunAccurateResiduals) + (if (= "deviations" robust_residuals) + (call !ComputeDeviations) robust_residuals (call !RunRobustResiduals) @@ -1228,7 +1228,7 @@ ) (assoc - ".robust" (and robust_residuals (!= "superfull" robust_residuals)) + ".robust" (and robust_residuals (!= "deviations" robust_residuals)) ".hyperparam_path" (if (and use_case_weights (= param_path (list ".default")) ) ;intentionally storing which case weight feature is used for retrieval diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 560868e3..b2af2752 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -1,7 +1,7 @@ (null - ;helper method for CalculateFeatureResiduals to calculate 'superfull' accurate residuals - #!RunAccurateResiduals + ;helper method for CalculateFeatureResiduals to calculate 'superfull' local accurate residuals + #!ComputeDeviations (seq ;keep a copy of all originally specified features (assign (assoc case_features features)) @@ -28,7 +28,7 @@ )) ;iterate over each case and accumulate residuals for all the feature(s) - #!AccumulateAccurateFeatureResiduals + #!AccumulateLocalFeatureDeviations (assign (assoc case_residuals_lists ||(map @@ -191,7 +191,7 @@ )) ;re-compute residuals on these case_ids, store into case_residuals_lists - (call !AccumulateAccurateFeatureResiduals) + (call !AccumulateLocalFeatureDeviations) ;transpose case_residuals_lists into a list the length of features and append the results into feature_residuals_lists (assign (assoc From 412426af46ef5f741027e6b64dcdb15d3eb8a83c Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 17 Dec 2024 22:02:12 -0500 Subject: [PATCH 24/29] fixes for shared deviations --- howso/analysis.amlg | 1 - howso/analysis_weights.amlg | 4 ++-- howso/feature_residuals.amlg | 20 +++----------------- howso/mda_weight.amlg | 12 ++++++++---- howso/shared_deviations.amlg | 2 +- unit_tests/ut_h_analyze.amlg | 4 ++-- unit_tests/ut_h_boundary_values.amlg | 2 +- 7 files changed, 17 insertions(+), 28 deletions(-) diff --git a/howso/analysis.amlg b/howso/analysis.amlg index 9ce4cebf..00983368 100644 --- a/howso/analysis.amlg +++ b/howso/analysis.amlg @@ -1182,7 +1182,6 @@ ;output updated hyperparam map (append baseline_hyperparameter_map - output_map ) ) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index a84d0262..38ccb07f 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -443,8 +443,8 @@ residual_num_samples ) custom_hyperparam_map baseline_hyperparameter_map - ;must compute confusion matrix to use sparse deviation matrix - compute_all_statistics (true) + ;must compute confusion matrix to use sparse deviation matrix, but not when computing mda + compute_all_statistics (not computing_mda) ;don't sparsify the confusion matrix so that SDM can be computed using full counts confusion_matrix_min_count 0 use_shared_deviations (true) diff --git a/howso/feature_residuals.amlg b/howso/feature_residuals.amlg index db67f7ee..8dbfe9a8 100644 --- a/howso/feature_residuals.amlg +++ b/howso/feature_residuals.amlg @@ -295,6 +295,7 @@ ts_feature_lag_amount_map (null) max_lag_index_value (null) feature_mda_map (null) + original_features (replace features) )) (call !InitResiduals) @@ -441,23 +442,8 @@ (assign (assoc feature_mda_map (call !ComputeRobustTargetlessMDA (assoc - features features - case_residuals_lists - (if compute_all_statistics - ;grab just the mae from the output for all stats - (map - (lambda - (map - (lambda (first (current_value))) - (current_value) - ) - ) - case_residuals_lists - ) - - ;else use as-is - case_residuals_lists - ) + features original_features + case_residuals_lists case_residuals_lists )) )) ) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index b2af2752..7ce80131 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -380,10 +380,14 @@ (declare (assoc total_probs (apply "+" (values probabilities_of_contribution_map)) )) - ;normalize probabilities for use as feature weights - (map - (lambda (/ (current_value) total_probs)) - probabilities_of_contribution_map + (if total_probs + ;normalize probabilities for use as feature weights + (map + (lambda (/ (current_value) total_probs)) + probabilities_of_contribution_map + ) + ;else set them all to 1 because this feature had no affect on others + (map 1 probabilities_of_contribution_map) ) )) individual_features_mda_map diff --git a/howso/shared_deviations.amlg b/howso/shared_deviations.amlg index 8074dfa7..c6636006 100644 --- a/howso/shared_deviations.amlg +++ b/howso/shared_deviations.amlg @@ -24,7 +24,7 @@ ;parameters: ; compressed_values: an assoc or list of values that need to expanded. If a list, adds back in the features from shared deviations groups that were removed ; for calculations If an assoc, adds back in removed features as keys and copies the value of the primary key for each shared deviation group as its value. - ; e.g. {"a": 2.5, "b": 0} -> {"a": 2.5 "b":9 "c": 2.5 "d": 2.5} if features a, c, d were in a shared deviations group. + ; e.g. {"a": 2.5, "b": 9} -> {"a": 2.5 "b": 9 "c": 2.5 "d": 2.5} if features a, c, d were in a shared deviations group. #!ExpandForSharedDeviations (declare (assoc compressed_values (null)) diff --git a/unit_tests/ut_h_analyze.amlg b/unit_tests/ut_h_analyze.amlg index a485ad43..03c1f5da 100644 --- a/unit_tests/ut_h_analyze.amlg +++ b/unit_tests/ut_h_analyze.amlg @@ -105,14 +105,14 @@ result (call_entity "howso" "debug_label" (assoc label "!hyperparameterMetadataMap")) )) - (print "targetless analyzed with weights: ") + (print "targetless analyzed without weights: ") (call assert_true (assoc obs (and (!= (null) (get result ["targetless" "fruit.height.length.size.sweet.tart.weight.width." ".none" "k"])) (!= (null) (get result ["targetless" "fruit.height.length.size.sweet.tart.weight.width." ".none" "p"])) (!= (null) (get result ["targetless" "fruit.height.length.size.sweet.tart.weight.width." ".none" "dt"])) - (> (size (get result ["targetless" "fruit.height.length.size.sweet.tart.weight.width." ".none" "featureWeights"])) 0) + (= (size (get result ["targetless" "fruit.height.length.size.sweet.tart.weight.width." ".none" "featureWeights"])) 0) ) )) diff --git a/unit_tests/ut_h_boundary_values.amlg b/unit_tests/ut_h_boundary_values.amlg index 0c0b1a1d..13ddd7c6 100644 --- a/unit_tests/ut_h_boundary_values.amlg +++ b/unit_tests/ut_h_boundary_values.amlg @@ -1,6 +1,6 @@ (seq #unit_test (direct_assign_to_entities (assoc unit_test (load "unit_test.amlg"))) - (call (load "unit_test_howso.amlg") (assoc name "ut_h_boundary_cases.amlg")) + (call (load "unit_test_howso.amlg") (assoc name "ut_h_boundary_values.amlg")) (call_entity "howso" "set_feature_attributes" (assoc feature_attributes From 66916dae04bb77d46c6d010fa5667a04c0ea10b1 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 17 Dec 2024 23:09:37 -0500 Subject: [PATCH 25/29] update unit tests --- unit_tests/ut_h_null_null_react.amlg | 17 ++++- unit_tests/ut_h_stats.amlg | 100 +++++++++++++-------------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/unit_tests/ut_h_null_null_react.amlg b/unit_tests/ut_h_null_null_react.amlg index 810d3ff7..809c687c 100644 --- a/unit_tests/ut_h_null_null_react.amlg +++ b/unit_tests/ut_h_null_null_react.amlg @@ -61,8 +61,23 @@ )) (call assert_approximate (assoc obs (get result "C") - exp (list (null) 1 0.5) + exp + (list + [ + { + A [ {A 0.032} 0.99 ] + B [ {B 0.131} 0.97 ] + C [ {C 0.124} 0.97 ] + D [ {D 0.027} 0.99 ] + E [ {D 0.683 E 0.397 } 0.98 ] + } + 0.72 + ] + 1 + 0.5 + ) thresh .3 + unordered (true) )) (call_entity "howso" "set_params" (assoc diff --git a/unit_tests/ut_h_stats.amlg b/unit_tests/ut_h_stats.amlg index 01cb44d0..3c9d007f 100644 --- a/unit_tests/ut_h_stats.amlg +++ b/unit_tests/ut_h_stats.amlg @@ -334,11 +334,11 @@ { accuracy { fruit 0.9 - size 0.7 + size 0.8 } precision { fruit 0.9 - size 0.7 + size 0.8 } recall { fruit 0.9 @@ -346,31 +346,31 @@ } mcc { fruit 0.89 - size 0.73 + size 0.77 } mae { fruit 0.19 height 0.6 - length 0.73 + length 0.65 size 0.29 sweet 0.06 tart 0.16 weight 1.1 - width 0.8 + width 0.7 } adjusted_smape { height 20 length 20 sweet 10 - tart 60 + tart 63 weight 20 - width 26 + width 28 } rmse { - height 0.85 + height 0.9 length 0.9 sweet 0.08 - tart 0.21 + tart 0.25 weight 1.8 width 1 } @@ -378,7 +378,7 @@ height 20 length 20 sweet 10 - tart 60 + tart 63 weight 20 width 26 } @@ -391,7 +391,7 @@ width 0.8 } } - percent 0.45 + percent 0.4 )) (assign (assoc @@ -415,7 +415,7 @@ height 0.86 length 0.80 sweet 0.58 - tart 0.3 + tart 0.25 weight 0.9 width 0.8 } @@ -437,8 +437,7 @@ (call_entity "howso" "analyze" (assoc targeted_model "targetless" - p_values (list 0.5 1) - k_values (list 8 13) + k_values [8] )) (assign (assoc @@ -459,45 +458,45 @@ exp { accuracy { - fruit 0.7 + fruit 0.85 size 0.87 } precision { - fruit 0.7 + fruit 0.85 size 0.61 } recall { - fruit 0.3 + fruit 0.8 size 0.63 } mae { - fruit 0.59 - height 0.85 - length 1.09 - size 0.34 - sweet 0.09 - tart 0.20 - weight 1.44 - width 1.27 + fruit 0.26 + height 0.7 + length 0.5 + size 0.3 + sweet 0.06 + tart 0.03 + weight 0.85 + width 0.6 } r2 { - height 0.77 - length 0.73 - sweet -3.47 - tart -15.33 - weight 0.86 - width 0.56 + height 0.8 + length 0.9 + sweet -0.7 + tart 0.4 + weight 0.9 + width 0.9 } adjusted_smape { - height 22 - length 24 - sweet 10 - tart 70 - weight 19 - width 30 + height 16 + length 12 + sweet 7 + tart 14 + weight 10 + width 18 } } - percent 0.5 + percent 0.4 )) (assign (assoc @@ -516,7 +515,7 @@ ; holding out all of one fruit type should result in not being able to predict that fruit correctly (print "Output conditioned prediction stats for fruits with disjoint datasets correctly: ") (call assert_same (assoc - obs + obs [ (get result ["accuracy" "fruit"]) (get result ["precision" "fruit"]) @@ -543,9 +542,9 @@ (call assert_approximate (assoc obs (map (lambda (get (current_value) "fruit")) result) exp (assoc - accuracy 0.42 + accuracy 0.43 recall 0.75 - precision 0.55 + precision 0.44 confusion_matrix (assoc matrix @@ -553,7 +552,7 @@ apple (assoc apple 1) banana (assoc banana 1) peach (assoc peach 1) - strawberry (assoc apple 4) + strawberry (assoc apple 3 peach 1) ) leftover_correct 0 leftover_incorrect 0 @@ -583,7 +582,7 @@ exp (assoc accuracy {fruit 0.42 size 0} recall {fruit 0.75 size 0} - precision {fruit 0.55 size 0} + precision {fruit 0.44 size 0} confusion_matrix { fruit { @@ -629,7 +628,7 @@ (call assert_approximate (assoc obs (get result ["accuracy" "size"]) - exp 0.83 + exp 0.8 thresh 0.06 ) ) @@ -653,15 +652,10 @@ ; with 'features' returns prediction stats. Also tests whether having ; an 'action_feature' that overlaps with 'features' returns correct ; stats. Accuracy should be one since should only predict one case correctly. - (call assert_exact - (assoc - obs result - exp (assoc - fruit (assoc accuracy 1) - size (assoc accuracy 1) - ) - ) - ) + (call assert_same (assoc + obs result + exp { accuracy { fruit 1 size 1}} + )) (call exit_if_failures (assoc msg unit_test_name )) ) \ No newline at end of file From 232c5cb4243487a25c3a56cb05b3a3b7fd372717 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:17:22 -0500 Subject: [PATCH 26/29] merge main --- howso/mda_weight.amlg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/howso/mda_weight.amlg b/howso/mda_weight.amlg index 7ce80131..f36e097a 100644 --- a/howso/mda_weight.amlg +++ b/howso/mda_weight.amlg @@ -181,8 +181,8 @@ (query_not_equals (current_index) (null)) (if valid_weight_feature - (query_weighted_sample weight_feature (- min_value_count (current_value)) (rand)) - (query_sample (- min_value_count (current_value)) (rand)) + (query_sample (- min_value_count (current_value)) weight_feature (rand)) + (query_sample (- min_value_count (current_value)) (null) (rand)) ) )) num_valid_values_per_feature_map From 1aaa32b7d53a5f1a1b501172478dfb064d150cb9 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:29:43 -0500 Subject: [PATCH 27/29] fix unit test --- howso/analysis_weights.amlg | 2 +- unit_tests/ut_h_boundary_values.amlg | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/howso/analysis_weights.amlg b/howso/analysis_weights.amlg index b2b14b1e..4440197a 100644 --- a/howso/analysis_weights.amlg +++ b/howso/analysis_weights.amlg @@ -409,7 +409,7 @@ ;At this point the hyperparameters and residuals and weights are stable enough for use. (while (< iteration num_iterations) - ;compute mda on 2nd to last iteration + ;compute mda on last iteration (assign (assoc computing_mda (= iteration 3) )) diff --git a/unit_tests/ut_h_boundary_values.amlg b/unit_tests/ut_h_boundary_values.amlg index 13ddd7c6..6ec89ee1 100644 --- a/unit_tests/ut_h_boundary_values.amlg +++ b/unit_tests/ut_h_boundary_values.amlg @@ -43,8 +43,6 @@ (call_entity "howso" "analyze" (assoc k_values [3] - p_values [0.1] - use_deviations (false) )) ;use f1 to predict f2 @@ -179,7 +177,7 @@ result (call_entity "howso" "react" (assoc context_features ["f1"] - context_values [[6]] + context_values [[5]] action_features ["f2"] details { From b4b396effff274eb0c57210c0b156116b18a3b8f Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:20:11 -0500 Subject: [PATCH 28/29] use 'surprisal' for conviction when dt is 'surprisal_to_prob' --- howso/conviction.amlg | 38 ++++++++++++++++++++++++-------------- howso/distances.amlg | 4 +++- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/howso/conviction.amlg b/howso/conviction.amlg index 9072419a..4046d681 100644 --- a/howso/conviction.amlg +++ b/howso/conviction.amlg @@ -160,7 +160,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -231,7 +231,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -296,7 +296,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -427,7 +427,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -461,7 +461,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -496,7 +496,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -530,7 +530,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -570,7 +570,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -614,7 +614,7 @@ query_feature_attributes_map feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if valid_weight_feature weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -722,7 +722,7 @@ (get hyperparam_map "featureDomainAttributes") feature_deviations (get hyperparam_map "p") - (get hyperparam_map "dt") + (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) (if use_case_weights weight_feature (null)) ;use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed") "fixed rand seed" @@ -797,7 +797,7 @@ feature_weights (get hyperparam_map "featureWeights") feature_deviations (get hyperparam_map "featureDeviations") model_size (call !GetNumTrainingCases) - dt_parameter (get hyperparam_map "dt") + dt_parameter (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) p_parameter (get hyperparam_map "p") query_closest_k (get hyperparam_map "k") query_feature_attributes_map (get hyperparam_map "featureDomainAttributes") @@ -836,7 +836,8 @@ query_feature_attributes_map feature_deviations p_parameter - 1 ;dt is since we just want actual distances + ;pull actual distance or surprisal, not influence + (if (= "surprisal" dt_parameter) "surprisal" 1) (if use_case_weights weight_feature (null)) (rand) (null) ;radius @@ -868,7 +869,7 @@ ) ;if the distance weight exponent is not the default value of -1, apply the negative value of it to all the distances - (if (!= -1 dt_parameter) + (if (and (!= -1 dt_parameter) (!= "surprisal" dt_parameter)) (assign (assoc non_zero_distances (map (lambda (pow (current_value) (- dt_parameter))) non_zero_distances) @@ -893,7 +894,16 @@ )) ;if all the neighbors have zero distance, return a 0 - (if (= 0 (size non_zero_distances)) + (if (= "surprisal" dt_parameter) + (let + (assoc + probabilities (map (lambda (exp (- (current_value)))) (values closest_cases_distances_map)) + ) + ;normalize by total probabilities + (/ (entropy probabilities) (apply "+" probabilities ) ) + ) + + (= 0 (size non_zero_distances)) 0 ;else scale the harmonic mean by corresponding count of zero distance neighbors diff --git a/howso/distances.amlg b/howso/distances.amlg index 3f200eef..81688d48 100644 --- a/howso/distances.amlg +++ b/howso/distances.amlg @@ -295,6 +295,7 @@ feature_deviations (get hyperparam_map "featureDeviations") p_value (get hyperparam_map "p") query_feature_attributes_map (get hyperparam_map "featureDomainAttributes") + in_surprisal_space (= "surprisal_to_prob" (get hyperparam_map "dt")) )) ;iterate over all the unique session ids and create a map of session id -> session indices map @@ -397,6 +398,7 @@ to_case_values (null) features + in_surprisal_space ) )) (range 0 last_index) @@ -628,7 +630,7 @@ query_feature_attributes_map feature_deviations p_value - 1 ;dt + (if (= "surprisal_to_prob" dt_value) "surprisal" 1) (if valid_weight_feature weight_feature (null)) "fixed rand seed" (null) ;radius From f475050ad9eb12a58de0948fe3293257986e3c9e Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:23:17 -0500 Subject: [PATCH 29/29] use surprisal math for unique check --- howso/conviction.amlg | 10 +++++----- howso/distances.amlg | 20 ++++++++++++-------- howso/synthesis_validation.amlg | 11 +++++++---- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/howso/conviction.amlg b/howso/conviction.amlg index 4046d681..d61f23c5 100644 --- a/howso/conviction.amlg +++ b/howso/conviction.amlg @@ -797,7 +797,7 @@ feature_weights (get hyperparam_map "featureWeights") feature_deviations (get hyperparam_map "featureDeviations") model_size (call !GetNumTrainingCases) - dt_parameter (if (= (get hyperparam_map "dt") "surprisal_to_prob") "surprisal" (get hyperparam_map "dt") ) + dt_parameter (get hyperparam_map "dt") p_parameter (get hyperparam_map "p") query_closest_k (get hyperparam_map "k") query_feature_attributes_map (get hyperparam_map "featureDomainAttributes") @@ -837,7 +837,7 @@ feature_deviations p_parameter ;pull actual distance or surprisal, not influence - (if (= "surprisal" dt_parameter) "surprisal" 1) + (if (= "surprisal_to_prob" dt_parameter) "surprisal" 1) (if use_case_weights weight_feature (null)) (rand) (null) ;radius @@ -847,7 +847,7 @@ )) (assign (assoc - non_zero_distances (filter (lambda (!= (current_value) 0)) (values closest_cases_distances_map)) + non_zero_distances (filter (lambda (> (current_value) 1e-13)) (values closest_cases_distances_map)) )) (if (or (> (size non_zero_distances) 0) (= query_closest_k model_size)) @@ -869,7 +869,7 @@ ) ;if the distance weight exponent is not the default value of -1, apply the negative value of it to all the distances - (if (and (!= -1 dt_parameter) (!= "surprisal" dt_parameter)) + (if (and (!= -1 dt_parameter) (!= "surprisal_to_prob" dt_parameter)) (assign (assoc non_zero_distances (map (lambda (pow (current_value) (- dt_parameter))) non_zero_distances) @@ -894,7 +894,7 @@ )) ;if all the neighbors have zero distance, return a 0 - (if (= "surprisal" dt_parameter) + (if (= "surprisal_to_prob" dt_parameter) (let (assoc probabilities (map (lambda (exp (- (current_value)))) (values closest_cases_distances_map)) diff --git a/howso/distances.amlg b/howso/distances.amlg index 81688d48..58a24ec8 100644 --- a/howso/distances.amlg +++ b/howso/distances.amlg @@ -25,9 +25,10 @@ query_feature_attributes_map ;Feature deviations are not used in order to ensure that we are measuring privacy ;assuming it has been maximally preserved. Deviations make cases look farther away than they are. - (null) ;feature_deviations + (if (= "surprisal_to_prob" dt_parameter) feature_deviations (null) ) p_parameter - 1 ;dt of 1 queries distance in ascending order + ;dt of 1 queries distance in ascending order + (if (= "surprisal_to_prob" dt_parameter) "surprisal" 1) (null) ;Weight_feature is set to null so the computation done here matches the rejection criteria ;in generate.amlg. ;Use a fixed random seed to guarantee deterministic behavior for reacts (named "fixed rand seed"). @@ -718,9 +719,10 @@ feature_weights !queryDistanceTypeMap query_feature_attributes_map - (if use_feature_deviations (get hyperparam_map "featureDeviations") (get hyperparam_map "nullUncertainties")) + (if (or use_feature_deviations (= "surprisal_to_prob" dt_parameter)) (get hyperparam_map "featureDeviations") (get hyperparam_map "nullUncertainties")) p_parameter - 1 ;dt = 1 means return computed distance to the case + ;dt = 1 means return computed distance to the case + (if (= "surprisal_to_prob" dt_parameter) "surprisal" 1) (null) ;weight_feature (rand) (null) ;radius @@ -752,9 +754,10 @@ query_feature_attributes_map ;Feature deviations are not used in order to ensure that privacy is maximally preserved. ;If feature deviations are used, duplicate cases may be deemed private. - (null) ;feature_deviations + (if (= "surprisal_to_prob" dt_parameter) feature_deviations (null) ) p_parameter - 1 ;dt = 1 means return computed distance to each case + ;dt = 1 means return computed distance to each case + (if (= "surprisal_to_prob" dt_parameter) "surprisal" 1) (null) ;weight (rand) (null) ;radius @@ -776,9 +779,10 @@ feature_weights !queryDistanceTypeMap query_feature_attributes_map - (if use_feature_deviations (get hyperparam_map "featureDeviations") (get hyperparam_map "nullUncertainties")) + (if (or use_feature_deviations (= "surprisal_to_prob" dt_parameter)) (get hyperparam_map "featureDeviations") (get hyperparam_map "nullUncertainties")) p_parameter - 1 ;dt = 1 means return computed distance to the case + ;dt = 1 means return computed distance to the case + (if (= "surprisal_to_prob" dt_parameter) "surprisal" 1) (null) ;weight_feature (rand) (null) ;radius diff --git a/howso/synthesis_validation.amlg b/howso/synthesis_validation.amlg index fb9f0e96..d60a0275 100644 --- a/howso/synthesis_validation.amlg +++ b/howso/synthesis_validation.amlg @@ -46,6 +46,7 @@ feature_deviations (get hyperparam_map "featureDeviations") p_parameter (get hyperparam_map "p") query_feature_attributes_map (get hyperparam_map "featureDomainAttributes") + dt_parameter (get hyperparam_map "dt") ;override global residuals with calculated residuals per feature threshold_feature_residuals_map @@ -180,11 +181,12 @@ feature_weights !queryDistanceTypeMap query_feature_attributes_map - ;Feature deviations are not used in order to ensure that privacy is maximally preserved. + ;Feature deviations are not used in order to ensure that privacy is maximally preserved when using distance ;If feature deviations are used, duplicate cases may be deemed private. - (null) ;feature_deviations + (if (= "surprisal_to_prob" dt_parameter) feature_deviations (null) ) p_parameter - 1 ;dt = 1 means return computed distance to each case + ;dt = 1 means return computed distance to each case + (if (= "surprisal_to_prob" dt_parameter) "surprisal" 1) (null) ;weight (rand) (null) ;radius @@ -208,7 +210,7 @@ ;only test for uniqueness if the generated case is not a perfect match because has_dupes begins with true, ;skipping this block will indicate that the case is a duplicate - (if (!= dist_to_closest_case 0) + (if (> dist_to_closest_case 1e-13) (seq (assign (assoc closest_case_values (retrieve_from_entity closest_case (if has_novel_substitions non_novel_context_features context_features)) @@ -410,6 +412,7 @@ query_feature_attributes_map (get hyperparam_map "featureDomainAttributes") non_novel_context_features (null) has_novel_substitions (and exclude_novel_nominals_from_uniqueness_check (size !novelSubstitionFeatureSet)) + dt_parameter (get hyperparam_map "dt") )) ;find the closest cases using the same code as generate case, set generate_attempt to 2 so that it