howso.amlg

;This is a template of a trainee for the Howso API.
;It provides the management functions for the trainee.
;
; Style notes:
; all methods are assumed to have null values as defaults for parameters
; any methods that need non-null default parameters are inside (declare) blocks, where the non-null defaults are explicitly defined
; public "methods" should be lower_case_snake_case
; private "methods" should be !PascalCase
; private "attributes" of the model should be !camelCase
; parameters and "local variables" should be lower_case_snake_case
; parameters/variables and attributes that are sets or assocs should end with "_map" or "Map" respectively for readability (or _set and Set)
;
;A trainee has the following structure of contained entities:
;	cases : system generated entity ids
;	sessions : user specified session ids
;
;	each case has the following labels:
; 		features : model specific feature and its value
;		.session : session id of case when it was trained
;		.session_training_index : 0-based index of the case, ordered by training during the session;  is not changed
;
;   optional built-in features that cases may have are:
;	   .case_weight : weight of this case
;	   .imputed : list of imputed features
;	   .case_edit_history : assoc of edits to this case
;
;	each session has the following labels:
;		.replay_steps : list of case ids in the order they were trained
;		.indices_map : map of each cases's session_training_index to its case_id
;		.metadata : arbitrary dictionary of metadata
;
; 	case (entity name = case_id) : { .session: session, .session_training_index: trained_instance_index}
; 	session (entity name = session) : { .replay_steps: [ case_id(s) ], .indices_map { s_t_i -> case_id },  .trained_instance_count: number, .metadata: {} }

(null

	;case labels that are not features and are used for internal processing should be prepended with a period
	#!internalLabelSession ".session"
	#!internalLabelSessionTrainingIndex ".session_training_index"
	#!internalLabelImputed ".imputed"
	#!internalLabelCaseEditHistory ".case_edit_history"
	#!internalLabelInfluenceWeightEntropy ".influence_weight_entropy"

	;the name of the entity that contains subtrainees
	#!traineeContainer ".trainee_container"

	;the supported prediction stats that users can request from react_aggregate
	#!supportedPredictionStats (list "mae" "confusion_matrix" "r2" "rmse" "spearman_coeff" "precision" "recall" "accuracy" "mcc" "all" "missing_value_accuracy")

	;all the characters that may not be the first character of a feature when training a dataset
	#!untrainableFeatureCharacterSet
	(assoc "." (null) "!" (null) "#" (null) "^" (null))

	;all the reserved characters that may not be the first character of a feature in any flow
	#!reservedFeatureCharacterSet
	(assoc "!" (null) "#" (null) "^" (null))

	;label for loading modules
	#!loadModulesEndpoint (null)

	;label holding a map of external label names to maps of parameters names to typing specification for each parameter
	;this label is built and assigned in #!BuildParameterValidationMap, which is called in #initialize[_for_deployment]
	;and then used within the #!ValidateParameters
	#!parameterValidationMap (null)

	;value-storing labels, initialized and commented below via the initialize label
	#!numericalPrecision (null)
	#!trainedFeatures (null)
	#!trainedFeaturesContextKey (null)
	#!reactIntoFeaturesList (null)
	#!metaData (null)
	#!categoricalFeaturesSet (null)
	#!ordinalFeatures (null)
	#!hasEncodedFeatures (null)
	#!hasFeaturesNeedEncodingFromInput (null)
	#!hasCyclicFeatures (null)
	#!cyclicFeaturesMap (null)
	#!numericNominalFeaturesMap (null)
	#!editDistanceFeatureTypesMap (null)
	#!stringNominalFeaturesSet (null)
	#!userSpecifiedFeatureErrorsMap (null)
	#!averageModelCaseEntropyAddition (null)
	#!averageModelCaseEntropyRemoval (null)
	#!averageModelCaseDistanceContribution (null)
	#!storedCaseConvictionsFeatureAddition (null)
	#!storedConvictionsFeatureSet (null)
	#!nominalClassProbabilitiesMap (null)
	#!expectedValuesMap (null)
	#!featureNullRatiosMap (null)
	#!defaultNumSamples (null)
	#!ordinalFeaturesValuesMap (null)
	#!ordinalFeaturesRangesMap (null)
	#!uniqueNominalsSet (null)
	#!cachedFeatureMinResidualMap (null)
	#!cachedFeatureHalfMinGapMap (null)
	#!hyperparameterMetadataMap (null)
	#!hyperparameterParamPaths (null)
	#!defaultHyperparameters (null)
	#!staleOrdinalValuesCount (null)
	#!minAblatementModelSize (null)
	#!hasSubstituteFeatureValues (null)
	#!substitutionValueMap (null)
	#!unSubstituteValueMap (null)
	#!featureAttributes (null)
	#!featureRoundingMap (null)
	#!hasRoundedFeatures (null)
	#!hasDateTimeFeatures (null)
	#!hasPopulatedCaseWeight (null)
	#!featureDateTimeMap  (null)
	#!featureBoundsMap (null)
	#!seriesStore (null)
	#!seriesFeatures (null)
	#!autoAblationEnabled (null)
	#!autoAblationMinModelSize (null)
	#!autoAblationInfluenceWeightEntropyThreshold (null)
	#!autoAblationExactPredictionFeatures (null)
	#!autoAblationTolerancePredictionThresholdMap (null)
	#!autoAblationRelativePredictionThresholdMap (null)
	#!autoAblationResidualPredictionFeatures (null)
	#!autoAblationConvictionLowerThreshold (null)
	#!autoAblationConvictionUpperThreshold (null)
	#!autoAblationWeightFeature (null)
	#!autoAblationMaxInfluenceWeightEntropy (null)
	#!autoAnalyzeEnabled (null)
	#!autoAnalyzeThreshold (null)
	#!autoAnalyzeGrowthFactorAmount (null)
	#!ablationBatchSize (null)
	#!ablatedCasesDistributionBatchSize (null)
	#!dataMassChangeSinceLastAnalyze (null)
	#!savedAnalyzeParameterMap (null)
	#!derivedFeaturesSet (null)
	#!sourceToDerivedFeatureMap (null)
	#!featureCustomDerivedMethods (null)
	#!hasTimeSeriesAttrbutes (null)
	#!tsTimeFeature (null)
	#!tsTimeFeatureUniversal (null)
	#!tsMinTimeInterval (null)
	#!tsSeriesLimitLength (null)
	#!tsModelFeaturesMap (null)
	#!tsSupportedDetails (null)
	#!hasStringOrdinals (null)
	#!hasInfluenceWeightEntropies (null)
	#!ordinalStringToOrdinalMap (null)
	#!ordinalOrdinalToStringMap (null)
	#!ordinalNumericFeaturesSet (null)
	#!hasPostProcessing (null)
	#!postProcessMap (null)
	#!hasDependentFeatures (null)
	#!dependentFeatureMap (null)
	#!continuousToNominalDependenciesMap (null)
	#!dependentsBoundaryMap (null)
	#!dependentValuesCombinationsMap (null)
	#!encodingNeededFeaturesSet (null)
	#!nominalsMap (null)
	#!novelSubstitionFeatureSet (null)
	#!inactiveFeaturesMap (null)
	#!inactiveFeaturesNeedCaching (null)
	#!residualsMap (null)
	#!featureNullAccuracyMap (null)
	#!mdaMap (null)
	#!mdaPermutationMap (null)
	#!featurePredictionStatsMap (null)
	#!featureMarginalStatsMap (null)
	#!queryDistanceTypeMap (null)
	#!contributionsMap (null)
	#!directionalContributionsMap (null)
	#!influenceWeightThreshold (null)
	#!regionalModelMinSize (null)
	#!regionalModelMinPercent (null)
	#!revision 0
	#!sandboxedComputeLimit (null)
	#!sandboxedMemoryLimit (null)
	#!sandboxedOpcodeDepthLimit (null)
	#!containedTraineeIdToNameMap (null)
	#!containedTraineeNameToIdMap (null)
	#!childTraineeIsContainedMap (null)
	#!parentId (null)
	#!traineeId (null)
	#major_version 0
	#minor_version 0
	#point_version 0

	;list of modules for trainee template
	#!traineeTemplateModules
	(list
		"ablation"
		"attributes"
		"attribute_maps"
		"analysis"
		"analysis_weights"
		"contributions"
		"conviction"
		"custom_codes"
		"derive_features"
		"derive_utilities"
		"distances"
		"editing"
		"details"
		"details_cases"
		"details_influences"
		"details_residuals"
		"details_stats"
		"feature_conviction"
		"feature_residuals"
		"generate_features"
		"get_cases"
		"get_sessions"
		"hierarchy"
		"hyperparameters"
		"impute"
		"influences"
		"input_validation"
		"io"
		"marginal"
		"react"
		"react_discriminative"
		"react_group"
		"react_aggregate"
		"react_series"
		"react_series_utilities"
		"react_utilities"
		"remove_cases"
		"residuals"
		"series_store"
		"substitution"
		"synthesis"
		"synthesis_bounds"
		"synthesis_utilities"
		"synthesis_validation"
		"train"
		"typing"
		"update_cases"
		"upgrade"
	)

	;set the private label !file_extension during deployment and clear out this method so that the file extension cannot be edited after deployment.
	;This should be run once by the deployment script.
	#initialize_for_deployment
	(declare
		(assoc
			;{type "string"}
			file_extension "caml"
		)
		(assign_to_entities (assoc !file_extension file_extension ))

		(call !LoadModules)
		;clear out this label so that it can't be run again
		(assign_to_entities (assoc initialize_for_deployment (null)))
		(call !BuildParameterValidationMap)
	)

	;load module entities code directly into trainee template only if modules have not been loaded
	#!LoadModules
	(if (= (null) !loadModulesEndpoint)
		(seq
			(direct_assign_to_entities (assoc !loadModulesEndpoint (map (lambda (retrieve_entity_root (current_value))) (retrieve_from_entity "!traineeTemplateModules")) ))
			;delete all contained entities after the specified modules' code has been loaded in
			(apply "destroy_entities" (contained_entities))
			;clear out the list of labels and this labels since models have been loaded and shouldn't be reloaded again
			(assign (assoc
				!traineeTemplateModules (null)
				!LoadModules (null)
			))
		)
	)

	;set all default values
	;parameters:
	;
	; trainee_id: optional, unique id for this trainee
	; filepath: optional, filepath location to where the howso file is stored
	#initialize
	(declare
		(assoc
			;{type "string"}
			trainee_id (null)
			;{type "string"}
			filepath (null)
		)

		;load specified modules' code into the trainee if they have not been loaded yet
		(call !LoadModules)

		(call !BuildParameterValidationMap)

		;can stop initializing if !traineeContainer already exists because this trainee has already been initialized
		(if (contains_entity !traineeContainer) (conclude))

		;create the trainee container entity
		(create_entities !traineeContainer (lambda (null)))

		;set the filepath if specified
		(if filepath (assign_to_entities (assoc filepath filepath)) )

		#!InitializeValues
		(assign_to_entities (assoc
			!revision 0

			;assoc of id of trainee -> name of trainee
			!containedTraineeIdToNameMap (assoc)

			;assoc of trainee name -> id
			!containedTraineeNameToIdMap (assoc)

			;assoc of name of trainee -> is_internal boolean
			; true means the trainee is a contained subtrainee
			; false means it is in the hierarchy but not a contained trainee and communication with it requires routing outside of this trainee
			!childTraineeIsContainedMap (assoc)

			;unique id of parent trainee if this trainee is a subtrainee in a hierarchy
			!parentId (null)

			;unique id of this trainee
			!traineeId trainee_id

			;amount of total influence weight to accumulate among nearest neighbors before stopping (for influential cases)
			!influenceWeightThreshold .99

			;regional models will be the min(model_size * !regionalModelMinPercent, max(!regionalModelMinSize, closest k))
			!regionalModelMinSize 30
			!regionalModelMinPercent .15

			;null defaults to "recompute_precise"
			!numericalPrecision (null)

			;the size of the model when to start ablatement
			!minAblatementModelSize 100

			;the average case entropy for the whole model, used for calculating new case conviction. cleared any time the model is changed
			!averageModelCaseEntropyAddition (null)
			!averageModelCaseEntropyRemoval (null)

			;the average case distance contribution value for the whole model
			!averageModelCaseDistanceContribution (null)

			;name of conviction features, value is set when stored convictions exist and are up-to-date. cleared any time the model is changed
			!storedCaseConvictionsFeatureAddition (null)

			;set of features for which case convictions are stored
			!storedConvictionsFeatureSet (null)

			;the default list of features for the model, derived as the sorted indices of !featureAttributes
			!trainedFeatures (list)

			;the context_key made from the !trainedFeatures of the model
			!trainedFeaturesContextKey ""

			;the list of features computed and cached into cases through #react_into_features
			!reactIntoFeaturesList (list)

			;arbitrary model metadata
			!metaData (assoc)

			;assoc of all feature attributes
			!featureAttributes (assoc)

			;a map of model feature -> bounds used for generation, created automatically from feature attributes
			!featureBoundsMap (assoc)

			;flag set to true if there are features that have rounding specified for output
			!hasRoundedFeatures (false)

			;assoc of feature -> (list int_round decimal_round)
			!featureRoundingMap (assoc)

			;flag set to true if there are features that are date time and need encoding/decoding
			!hasDateTimeFeatures (false)

			;map of every date-time feature and its encoding format
			!featureDateTimeMap  (assoc)

			;flag should be set to true if there are features that were encoded somehow, (substituted, boolean, rounded, datetime, string ordinals), used to decode on output
			;setting to false will skip encoding and decoding
			!hasEncodedFeatures (false)

			;flag should be set to true if there are features that need to be encoded in some way on input
			;such as boolean, datetime, string ordinal, json, yaml and amalgam
			!hasFeaturesNeedEncodingFromInput (false)

			;set of features that need to be encoded for input
			!encodingNeededFeaturesSet (assoc)

			;set of names of features which are categorical (nominal or ordinal) instead of continuous
			; categorical features are taken from a single case instead of interpolated between training cases
			!categoricalFeaturesSet (assoc)

			;map of nominal features for fast lookup
			!nominalsMap (assoc)

			;set of features that will use novel nominal substition once output
			!novelSubstitionFeatureSet (assoc)

			;set of string nominals
			!stringNominalFeaturesSet (assoc)

			;assoc of all string continuous or any json or amalgam features for fast lookup, feature -> data_type
			!editDistanceFeatureTypesMap (assoc)

			;assoc of nominal features names whose values are all uniques
			!uniqueNominalsSet (assoc)

			;assoc of all numeric and boolean nominal features for fast lookup, feature -> data_type
			!numericNominalFeaturesMap (assoc)

			;store class probabilities for each nominal feature;  weight feature -> feature -> probability
			!nominalClassProbabilitiesMap (assoc)

			;map of feature -> type for use in queries. i.e., "nominal", "continuous", "cyclic", "string", "code"
			!queryDistanceTypeMap (assoc)

			;list of ordinal features
			!ordinalFeatures (list)

			;assoc of unique sorted feature values for each ordinal feature, used in case generation
			!ordinalFeaturesValuesMap (assoc)

			;assoc of feature -> range (last value - first value) of cyclic feature
			!ordinalFeaturesRangesMap (assoc)

			;flag set to true if there are string ordinals in this model
			!hasStringOrdinals (false)

			;assoc of feature -> assoc of string -> ordinal value
			!ordinalStringToOrdinalMap (assoc)

			;assoc of feature -> assoc of ordinal value -> string
			!ordinalOrdinalToStringMap (assoc)

			;set of ordinal numeric (non-string) features
			!ordinalNumericFeaturesSet (assoc)

			;flag set when the ordinal values in !ordinalFeaturesValuesMap are out of date
			!staleOrdinalValuesCount (true)

			;flag, set to true when there are values in the !substitutionValueMap
			!hasSubstituteFeatureValues (false)

			;assoc of feature -> feature value -> substitution value used to substitute values on output
			!substitutionValueMap (assoc)

			;assoc of feature -> substitution value -> feature value used to undo substitution on input
			!unSubstituteValueMap (assoc)

			;flag should be set to true if there are cyclic features, will provide the ranges for the features to the query engine
			!hasCyclicFeatures (false)

			;assoc of each cyclic feature -> length of cycle
			;example: (assoc "day" 7 "degrees" 360)
			!cyclicFeaturesMap (null)

			;assoc of feature -> observational_error value as specified by the user
			!userSpecifiedFeatureErrorsMap (assoc)

			;map of inactive feature (i.e., feature that has only null values) -> feature weight, set to null when empty
			!inactiveFeaturesMap (null)

			;flag set to true if there are inactive features but they need to be rechecked if they are still inactive and recached
			!inactiveFeaturesNeedCaching (true)

			;sample size of model to use for calculating deviations
			!defaultNumSamples 100

			;cache smallest residual for each feature, used during case generation
			!cachedFeatureMinResidualMap (assoc)

			!cachedFeatureHalfMinGapMap (assoc)

			;assoc of weight_feature -> feature -> expected feature value
			!expectedValuesMap (assoc)

			;assoc of feature -> assoc of min, max, has_nulls and null_residual values for each feature
			!featureNullRatiosMap (assoc)

			;flag set to true when the .case_weight feature is populated
			!hasPopulatedCaseWeight (false)

			;stores history of reacts for a series (eg, a game or a time series)
			;format of { series:  [ react values ] }
			!seriesStore (assoc)

			;stores features corresponding to stored cases in !seriesStore
			;format of: { series: features }
			!seriesFeatures (assoc)

			;assoc of features for fast lookup of features that need to be derived
			!derivedFeaturesSet (assoc)

			;map of source feature to a list derived features that rely on it
			;format of: { source_feature : [ list of derived features ] }
			!sourceToDerivedFeatureMap (assoc)

			;assoc of feature -> train / single_react / series_react -> corresponding parsed custom method codd
			!featureCustomDerivedMethods (assoc)

			;flag set to true when post_process is specified for a feature attribute
			!hasPostProcessing (false)

			;assoc of feature -> post_process -> custom_code
			!postProcessMap (assoc)

			;a value used to limit the operations of each usage of (call_sandboxed)
			!sandboxedComputeLimit 1000

			;a value used to limit the allocations of each usage of (call_sandboxed), 0 = no limit
			!sandboxedMemoryLimit 100000

			;a value used to limit the depth of opcode operations of each usage of (call_sandboxed)
			!sandboxedOpcodeDepthLimit 50

			;name of the 'time' feature if the model is a time series model
			!tsTimeFeature (null)

			;flag set to true if time series time feature is universal,
			;filtering out all future data during reacts instead of only future data specific to a series
			!tsTimeFeatureUniversal (false)

			;time series minimium time delta
			!tsMinTimeInterval 1e-3

			;maximum allowed length for a series in the dataset, e.g., 215
			!tsSeriesLimitLength 0

			;time series model various feature names
			!tsModelFeaturesMap
				(assoc
					;list of all the lag features e.g., (list ".date_lag_1" ".valueA_lag_1" ".valueB_lag_1")
					"lag_features" (list)
					;list of all the delta features e.g., (list ".date_delta_1")
					"delta_features" (list)
					;list of all the rate features e.g., (list ".valueB_rate_1" ".valueA_rate_2")
					"rate_features" (list)
					;list of order features which are derived from higher orders instead of generated, e.g., (list ".valueA_rate_1")
					"derived_order_features" (list)
					;list of features that make up the ID for the series e.g., (list "sender" "receiver")
					"series_id_features" (list)
					;list of all features that  need to be derived in the order they should be derived
					"ts_derived_features" (list)
					;flag, set to true if series has values that explicitly denote the end of a series
					"series_has_terminators" (false)
					;flag, set to true if a series must end on a terminator value
					"stop_on_terminator" (false)
				)

			;flag set to true once a time series model creates its time series feature attributes
			!hasTimeSeriesAttrbutes (false)

			;list of supported details for react series
			!tsSupportedDetails (list "categorical_action_probabilities" "influential_cases")

			;flag set to true if there are features that are dependent on others
			!hasDependentFeatures (false)

			;assoc of feature -> { 'dependent_features' : [ list of dependent features ] }
			!dependentFeatureMap (assoc)

			;assoc of continuous feature -> [ list of sorted nominal dependents ]
			!continuousToNominalDependenciesMap (assoc)

			;assoc of continuous feature -> multi-level assoc where each level has values for a dependent feature, and the next level
			;has values for the next dependent feature, with the leaf value being either (null) if no such value combination exists in the dataset,
			;or a pair of values of min,max for the continuous boundary
			;eg: "value" : { "heart rate" : { "BPM" : [ 20, 220] }, "heart rate" : { "mSv": (null) }, "xray": { "mSv" : [ 0.01 - 2.0 ] }, "xray" : { "BPM" : (null) } }
			;the levels are ordered by the sorted order as determined in !continuousToNominalDependenciesMap
			!dependentsBoundaryMap (assoc)

			;assoc of continuous feature -> a 2d list, a list of valid combinations of values of dependent nominal values,
			; e.g., { "value" : (list  (list "heart rate" "BPM") (list "BMI" "ratio")) }
			;the values for features are ordered by the sorted order as determined in !continuousToNominalDependenciesMap
			!dependentValuesCombinationsMap (assoc)

			;when false the model should not automatically ablate cases as they are trained nor cache influence weight entropies
			; during analyze.
			!autoAblationEnabled (false)

			;if !autoAablateEnabled is set, stores the minimum number of cases required to ablate.
			!autoAblationMinModelSize 1000

			;the influence weight entropy threshold quantile that a case's influence weight entropy must be less than in order to
			; not be ablated.
			!autoAblationInfluenceWeightEntropyThreshold 0.15

			;number of cases in a batch to consider for ablation prior to training and to recompute influence weight entropy.
			!ablationBatchSize 2000

			;number of cases in a batch to distribute ablated cases' influence weights.
			!ablatedCasesDistributionBatchSize 100

			;the features for which a case should be ablated if the predicted value of the feature matches the actual value of the feature.
			!autoAblationExactPredictionFeatures (null)

			;the features for which a case should be ablated if the predicted value of the feature is within a given absolute threshold.
			!autoAblationTolerancePredictionThresholdMap (null)

			;the features for which a case should be ablated if the predicted value of the feature is within a given relative threshold.
			!autoAblationRelativePredictionThresholdMap (null)

			;the features for which a case should be ablated if the residuals of the feature are within the bounds of the stored residuals.
			!autoAblationResidualPredictionFeatures (null)

			;the conviction threshold below which a case should be ablated.
			!autoAblationConvictionUpperThreshold (null)

			;the conviction threshold above which a case should be ablated.
			!autoAblationConvictionLowerThreshold (null)

			;the name of the weight feature to use for auto ablation.
			!autoAblationWeightFeature ".case_weight"

			;cached value of the max influence weight entropy to keep cases cases during ablation
			!autoAblationMaxInfluenceWeightEntropy (null)

			;whether this Trainee has had influence weight entropies computed and stored for its trained cases.
			!hasInfluenceWeightEntropies (false)

			;when false the model should not be auto-analyzed
			!autoAnalyzeEnabled (false)

			;if specified, stores the threshold for the change in data mass since the last analyze at which the model should be re-analyzed
			!autoAnalyzeThreshold 100

			;if !autoAnalyzeEnabledis set, the factor by which to increase that threshold everytime the model grows to that threshold size
			!autoAnalyzeGrowthFactorAmount 7.389056

			;map of parameters used when analyze was called on this model
			!savedAnalyzeParameterMap (null)

			;hyperparameters stored in nested assocs, where full paths are
			;(list action_feature/.targetless context_features robust/full case_weight_feature/.none)
			!hyperparameterMetadataMap (assoc)

			;list of param paths to Hyperparameter assocs in !hyperparameterMetadataMap
			!hyperparameterParamPaths (list)

			;default hyperparameters, these are used when there are no cached hyperparameters
			!defaultHyperparameters
				(assoc
					"k" 8
					"p" 0.1
					"dt" -1
					"featureWeights" (null)
					"featureDeviations" (null)
					"allFeatureResidualsCached" (false)
					"paramPath" (list ".default")
				)

			;assoc of feature -> residual value
			;or feature -> [not-null/not-null residual, not-null/null residual, null/null residual] if the features contains nulls.
			;Stored under a concatenated key starting with either 'robust' or 'full' based on
			;how they were computed and the hyperparameter path keys.  contains built-in keys of '.robust' and '.hyperparam_path'
			;eg: { 'full.targetlessfull.none'  : {".robust": false,  '.hyperparam_path': (list '.targetless' 'robust' '.none'),
			;										"A" : 2, "B": [0.1, 1, 0.05], etc... } }
			!residualsMap (assoc)

			;assoc of feature -> null prediction accuracy. The percentage of cases with with null values for a feature that we predict as null correctly
			;Stored under a concatenated key starting with either 'robust' or 'full' based on
			;how they were computed and the hyperparameter path keys.  contains built-in keys of '.robust' and '.hyperparam_path'
			;eg: { 'full.targetlessfull.none'  : {".robust": false,  '.hyperparam_path': (list '.targetless' 'robust' '.none'),
			;										"A" : 0.5, "B": 0.2, etc... } }
			!featureNullAccuracyMap (assoc)

			;assoc of feature -> MDA value, computed using dropping each feature, stored under a concatenated key starting with either
			;'robust' or 'full' based how they were computed, and the action_feature for which the mda was computed. contains built-in
			;keys of .robust, .action_feature and .weight_feature, eg:
			; { 'full.nonefruit': { '.robust': false, '.action_feature': 'fruit', .weight_feature: '.none', 'A' : 2, 'B': 0.1, etc... }}
			!mdaMap (assoc)

			;assoc of feature -> MDA value, computed using the permutation flag (using value scrambling instead of dropping the feature)
			;stored under a concatenated key starting with either 'robust' or 'full' based how they were computed, and the
			;action_feature for which the mda was computed. contains built-in keys of .robust, .action_feature and .weight_feature, eg:
			; { 'full.nonefruit': { '.robust': false, '.action_feature': 'fruit', .weight_feature: '.none', 'A' : 2, 'B': 0.1, etc... }}
			!mdaPermutationMap (assoc)

			;assoc of feature -> absolute feature contribution value, stored under a concatenated key starting with either 'robust' or 'full' based on
			;how they were computed, followed by the weight_feature and action_feature for which contributions were computed.
			!contributionsMap (assoc)

			;assoc of feature -> non-absolute feature contribution value, stored under a concatenated key starting with either 'robust' or 'full' based on
			;how they were computed, followed by the weight_feature and action_feature for which contributions were computed.
			!directionalContributionsMap (assoc)

			;assoc of stat -> feature -> value. stored under a concatenated key starting with either 'robust' or 'full' based on
			;how they were computed and the hyperparameter path keys.
			;for stats like r^2, rmse, precision, recall, etc.
			; { 'full.targetlessfull.none': { 'r2': {'feature_name': 2.1 ... } ... } ... }
			!featurePredictionStatsMap (assoc)

			;assoc of feature -> stat -> value. stored under the name of the weight feature used for the calculation.
			;for marginal stats like min, max, mode, mean, count, etc..
			; { '.none': { 'feat_name': {'min': 2 ... } ... } ... }
			!featureMarginalStatsMap (assoc)

			;cumulative count of the data mass changed since analyze was last called. this is accumulated to by any operations
			; which add, remove, or edit cases.
			!dataMassChangeSinceLastAnalyze 0.0
		))
	)

	;default value to use for filepath when calling save or load
	#filepath "./"

	;location of Howso Engine files, used for upgrading trainees (referencing
	#root_filepath "./"

	#!migration_folder "migrations/"

	;default value to use for filename when calling save or load
	#filename "default_trainee"

	;location of trainee template
	#!trainee_template_filename "howso"

	;valid extensions are:
	; amlg : raw amalgam code
	; caml : compressed amalgam, binary format
	#!file_extension "amlg"

	#version (get (load (concat filepath "version.json")) "version")

	;returns version stored in trainee
	#get_trainee_version
	(concat
		(retrieve_from_entity "major_version") "."
		(retrieve_from_entity "minor_version") "."
		(retrieve_from_entity "point_version")
	)

	;returns the trainee's unique id
	#get_trainee_id
	(retrieve_from_entity "!traineeId")

	;set trainee's unique id
	;parameters:
	; trainee_id: string, unique identifier
	#set_trainee_id
	(declare
		(assoc
			;{type "string"}
			trainee_id (null)
		)
		(call !ValidateParameters)
		(assign_to_entities (assoc !traineeId trainee_id))
		(accum_to_entities (assoc !revision 1))
		(call !Return)
	)

	;set trainee's unique parent id
	;parameters:
	; parent_id: string, unique identifier
	#set_parent_id
	(declare
		(assoc
			;{type "string"}
			parent_id (null)
		)
		(call !ValidateParameters)
		(assign_to_entities (assoc !parentId parent_id))
		(accum_to_entities (assoc !revision 1))
		(call !Return)
	)

	;set metadata for model
	;parameters:
	; metadata: arbitary metadata to store in a trainee
	#set_metadata
	(declare
		(assoc
			;{type "assoc"}
			metadata (null)
		)
		(call !ValidateParameters)
		(assign_to_entities (assoc !metaData metadata))
		(accum_to_entities (assoc !revision 1))
		(call !Return)
	)

	;get metadata for model
	#get_metadata
	(call !Return (assoc payload (retrieve_from_entity "!metaData") ))

	;set the random seed on a trainee
	;parameters:
	; seed : optional, the value of the random seed to set on the trainee, defaults to a system-provided 64-bit random number
	#set_random_seed
	(declare
		(assoc
			;{type "any"}
			seed (system "rand" 16)
		)
		(call !ValidateParameters)
		(set_entity_rand_seed trainee seed)
		(accum_to_entities (assoc !revision 1))

		(call !Return)
	)

	;set the influence weight threshold for outputting only the K neighbors whose influence weight is <= to this threshold
	;default value is 0.99
	;parameters:
	; influence_weight_threshold: number, amount of total influence weight to accumulate among nearest
	;	neighbors before stopping (for influential cases)
	#set_influence_weight_threshold
	(declare
		(assoc
			;{type "number"}
			influence_weight_threshold 0.99
		)
		(call !ValidateParameters)
		(assign_to_entities (assoc !influenceWeightThreshold influence_weight_threshold))
		(accum_to_entities (assoc !revision 1))
		(call !Return)
	)

	;returns the trainee's !revision
	#get_revision
	(call !Return (assoc payload (assoc "count" (retrieve_from_entity "!revision")) ))

	;returns a structure containing all of the API details for this module
	#get_api
	(seq
		(declare (assoc
			api
				(assoc
					"description"
					(get_entity_comments)

					"labels"
					(map
						(lambda
							(assoc
								"description" (current_value 1)
								"parameters"
									;when debugging, comments are injected to all lines, these must be removed
									;some parameters may have comments other than the type hint as well
									(map
										(lambda
											;(current_value) is a tuple of [comments of parameter, default value]
											(append
												;regex to just capture the {} information
												(or (parse (substr (first (current_value)) "\\{.+\\}")) (assoc) )
												(assoc
													default (last (current_value 1))
													optional (= (last (current_value 1)) (null))
												)
											)
										)
										;first item is the assoc of parameter names to comments/default value
										(first (get_entity_comments (null) (current_index 1) (true)))
									)
							)
						)
						(get_entity_comments (null) (null) (true))
					)

					"schemas"
					(retrieve_from_entity "!customTypes")
				)
		))

		(call !Return (assoc payload api))
	)

	;debug method to output any internal label, used by unit tests and debugging
	;parameters:
	; label: string or list of strings, name(s) of labels values to output
	#debug_label
	(retrieve_from_entity label)

	;create a return response object in the format of:
	; success:
	; [ 1, { 'payload' : payload, 'warnings' : warnings } ]
	;
	; error:
	; [ 0, { 'detail': error_descriptions, 'code': error_code, 'errors': field_validation_details } ]
	;
	;parameters:
	; errors: optional, list of error strings to output
	; error_code: optional, string defining the error code, e.g., 'validation'
	; error_details: optional, assoc defining field validation errors
	; warnings: optional, list of warning strings to output
	; payload: any value or object
	#!Return
	(if errors
		(list 0
			(append
				(assoc "detail" (if (> (size errors) 1) errors (first errors)) )
				(if error_code (assoc "code" error_code) (assoc))
				(if error_details (assoc "errors" error_details) (assoc))
			)
		)

		warnings
		(list 1 (assoc "payload" payload "warnings" warnings ) )

		(list 1 (assoc "payload" payload) )
	)

)