diff --git a/wres-config/nonsrc/schema.yml b/wres-config/nonsrc/schema.yml index 302a0e855..3df710bf1 100644 --- a/wres-config/nonsrc/schema.yml +++ b/wres-config/nonsrc/schema.yml @@ -1568,12 +1568,19 @@ definitions: enum: - contingency table - threat score + - threat score difference - equitable threat score + - equitable threat score difference - frequency bias + - frequency bias difference - probability of detection + - probability of detection difference - probability of false detection + - probability of false detection difference - false alarm ratio + - false alarm ratio difference - peirce skill score + - peirce skill score difference SingleValuedMetricEnum: title: Single-valued metrics. diff --git a/wres-config/src/wres/config/MetricConstants.java b/wres-config/src/wres/config/MetricConstants.java index 32288e4a9..e2d7b4175 100644 --- a/wres-config/src/wres/config/MetricConstants.java +++ b/wres-config/src/wres/config/MetricConstants.java @@ -86,6 +86,10 @@ public enum MetricConstants FALSE_ALARM_RATIO( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 0 ) ), + /** Difference in the False alarm ratio. */ + FALSE_ALARM_RATIO_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, true, + new Limits( -1, 1, Double.NaN ) ), + /** Pearson's product-moment correlation coefficient. */ PEARSON_CORRELATION_COEFFICIENT( SampleDataGroup.SINGLE_VALUED, StatisticType.DOUBLE_SCORE, new Limits( -1, 1, 1 ) ), @@ -95,17 +99,33 @@ public enum MetricConstants new Limits( -2, 2, Double.NaN ) ), /** Threat Score. */ - THREAT_SCORE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, - new Limits( 0, 1, 1 ) ), + THREAT_SCORE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 1 ) ), + + /** Difference in Threat Score. */ + THREAT_SCORE_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, + StatisticType.DOUBLE_SCORE, + true, + new Limits( -1, 1, Double.NaN ) ), /** Equitable Threat Score. */ EQUITABLE_THREAT_SCORE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, true, new Limits( -1.0 / 3, Double.POSITIVE_INFINITY, 1 ) ), + /** Difference in Equitable Threat Score. */ + EQUITABLE_THREAT_SCORE_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, true, + new Limits( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN ) ), + /** Frequency Bias. */ - FREQUENCY_BIAS( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, + FREQUENCY_BIAS( SampleDataGroup.DICHOTOMOUS, + StatisticType.DOUBLE_SCORE, new Limits( 0, Double.POSITIVE_INFINITY, 1 ) ), + /** Difference in Frequency Bias. */ + FREQUENCY_BIAS_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, + StatisticType.DOUBLE_SCORE, + true, + new Limits( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN ) ), + /** Index of Agreement. */ INDEX_OF_AGREEMENT( SampleDataGroup.SINGLE_VALUED, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 1 ) ), @@ -171,13 +191,28 @@ public enum MetricConstants StatisticType.DOUBLE_SCORE, true, new Limits( -1, 1, 1 ) ), + /** Difference in Peirce Skill Score. */ + PEIRCE_SKILL_SCORE_DIFFERENCE( new SampleDataGroup[] { SampleDataGroup.DICHOTOMOUS, SampleDataGroup.MULTICATEGORY }, + StatisticType.DOUBLE_SCORE, true, + new Limits( -2, 2, Double.NaN ) ), + /** Probability Of Detection. */ - PROBABILITY_OF_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, - new Limits( 0, 1, 1 ) ), + PROBABILITY_OF_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 1 ) ), + + /** Difference in Probability Of Detection. */ + PROBABILITY_OF_DETECTION_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, + StatisticType.DOUBLE_SCORE, + true, + new Limits( -1, 1, Double.NaN ) ), /** Probability Of False Detection.*/ - PROBABILITY_OF_FALSE_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, - new Limits( 0, 1, 0 ) ), + PROBABILITY_OF_FALSE_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 0 ) ), + + /** Difference in Probability Of False Detection.*/ + PROBABILITY_OF_FALSE_DETECTION_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, + StatisticType.DOUBLE_SCORE, + true, + new Limits( -1, 1, Double.NaN ) ), /** Quantile-quantile diagram. */ QUANTILE_QUANTILE_DIAGRAM( SampleDataGroup.SINGLE_VALUED, StatisticType.DIAGRAM, diff --git a/wres-metrics/src/wres/metrics/MetricFactory.java b/wres-metrics/src/wres/metrics/MetricFactory.java index 8a868a9ef..b1f60e8a2 100644 --- a/wres-metrics/src/wres/metrics/MetricFactory.java +++ b/wres-metrics/src/wres/metrics/MetricFactory.java @@ -75,16 +75,10 @@ public final class MetricFactory { - /** - * String used in several error messages to denote an unrecognized metric. - */ - + /** String used in several error messages to denote an unrecognized metric. */ private static final String UNRECOGNIZED_METRIC_ERROR = "Unrecognized metric for identifier."; - /** - * Test seed system property name. - */ - + /** Test seed system property name. */ private static final String TEST_SEED_PROPERTY = "wres.systemTestSeed"; /** @@ -388,6 +382,10 @@ public final class MetricFactory ( Collectable>, DoubleScoreStatisticOuter, DoubleScoreStatisticOuter> ) m; builder.addCollectableMetric( c ); } + else + { + builder.addMetric( m ); + } } builder.setExecutorService( executor ); return builder.build(); @@ -649,13 +647,21 @@ public final class MetricFactory return switch ( metric ) { case THREAT_SCORE -> ThreatScore.of(); + case THREAT_SCORE_DIFFERENCE -> DoubleScoreDifference.of( ThreatScore.of() ); case EQUITABLE_THREAT_SCORE -> EquitableThreatScore.of(); + case EQUITABLE_THREAT_SCORE_DIFFERENCE -> DoubleScoreDifference.of( EquitableThreatScore.of() ); case PEIRCE_SKILL_SCORE -> PeirceSkillScore.of(); + case PEIRCE_SKILL_SCORE_DIFFERENCE -> DoubleScoreDifference.of( PeirceSkillScore.of() ); case PROBABILITY_OF_DETECTION -> ProbabilityOfDetection.of(); + case PROBABILITY_OF_DETECTION_DIFFERENCE -> DoubleScoreDifference.of( ProbabilityOfDetection.of() ); case PROBABILITY_OF_FALSE_DETECTION -> ProbabilityOfFalseDetection.of(); + case PROBABILITY_OF_FALSE_DETECTION_DIFFERENCE -> + DoubleScoreDifference.of( ProbabilityOfFalseDetection.of() ); case FREQUENCY_BIAS -> FrequencyBias.of(); + case FREQUENCY_BIAS_DIFFERENCE -> DoubleScoreDifference.of( FrequencyBias.of() ); case CONTINGENCY_TABLE -> ContingencyTable.of(); case FALSE_ALARM_RATIO -> FalseAlarmRatio.of(); + case FALSE_ALARM_RATIO_DIFFERENCE -> DoubleScoreDifference.of( FalseAlarmRatio.of() ); default -> throw new IllegalArgumentException( UNRECOGNIZED_METRIC_ERROR + " '" + metric + "'." ); }; } @@ -846,6 +852,12 @@ private static boolean isCollectable( MetricConstants metric ) { Objects.requireNonNull( metric, "Specify a non-null metric to test." ); + // DoubleScoreDifference does not implement Collectable + if ( metric.isDifferenceMetric() ) + { + return false; + } + boolean singleValued = metric == MetricConstants.COEFFICIENT_OF_DETERMINATION || metric == MetricConstants.PEARSON_CORRELATION_COEFFICIENT || metric == MetricConstants.SUM_OF_SQUARE_ERROR; diff --git a/wres-statistics/nonsrc/wresproto/metric_name.proto b/wres-statistics/nonsrc/wresproto/metric_name.proto index c28c36199..571e51410 100644 --- a/wres-statistics/nonsrc/wresproto/metric_name.proto +++ b/wres-statistics/nonsrc/wresproto/metric_name.proto @@ -103,4 +103,11 @@ enum MetricName CONTINUOUS_RANKED_PROBABILITY_SCORE_DIFFERENCE = 88; BRIER_SCORE_DIFFERENCE = 89; RELATIVE_OPERATING_CHARACTERISTIC_SCORE_DIFFERENCE = 90; + THREAT_SCORE_DIFFERENCE = 91; + EQUITABLE_THREAT_SCORE_DIFFERENCE = 92; + FREQUENCY_BIAS_DIFFERENCE = 93; + PEIRCE_SKILL_SCORE_DIFFERENCE = 94; + PROBABILITY_OF_DETECTION_DIFFERENCE = 95; + PROBABILITY_OF_FALSE_DETECTION_DIFFERENCE = 96; + FALSE_ALARM_RATIO_DIFFERENCE = 97; } \ No newline at end of file