Support score differences for single-valued scores of dichotomous var…

…iables, #47.
NOAA-OWP · Aug 2, 2024 · 2d528fd · 2d528fd
1 parent b24657c
commit 2d528fd
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 15 deletions.
diff --git a/wres-config/nonsrc/schema.yml b/wres-config/nonsrc/schema.yml
@@ -1568,12 +1568,19 @@ definitions:
     enum:
       - contingency table
       - threat score
+      - threat score difference
       - equitable threat score
+      - equitable threat score difference
       - frequency bias
+      - frequency bias difference
       - probability of detection
+      - probability of detection difference
       - probability of false detection
+      - probability of false detection difference
       - false alarm ratio
+      - false alarm ratio difference
       - peirce skill score
+      - peirce skill score difference
 
   SingleValuedMetricEnum:
     title: Single-valued metrics.

diff --git a/wres-config/src/wres/config/MetricConstants.java b/wres-config/src/wres/config/MetricConstants.java
@@ -86,6 +86,10 @@ public enum MetricConstants
     FALSE_ALARM_RATIO( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE,
                        new Limits( 0, 1, 0 ) ),
 
+    /** Difference in the False alarm ratio. */
+    FALSE_ALARM_RATIO_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, true,
+                                  new Limits( -1, 1, Double.NaN ) ),
+
     /** Pearson's product-moment correlation coefficient. */
     PEARSON_CORRELATION_COEFFICIENT( SampleDataGroup.SINGLE_VALUED, StatisticType.DOUBLE_SCORE,
                                      new Limits( -1, 1, 1 ) ),
@@ -95,17 +99,33 @@ public enum MetricConstants
                                                 new Limits( -2, 2, Double.NaN ) ),
 
     /** Threat Score. */
-    THREAT_SCORE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE,
-                  new Limits( 0, 1, 1 ) ),
+    THREAT_SCORE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 1 ) ),
+
+    /** Difference in Threat Score. */
+    THREAT_SCORE_DIFFERENCE( SampleDataGroup.DICHOTOMOUS,
+                             StatisticType.DOUBLE_SCORE,
+                             true,
+                             new Limits( -1, 1, Double.NaN ) ),
 
     /** Equitable Threat Score. */
     EQUITABLE_THREAT_SCORE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, true,
                             new Limits( -1.0 / 3, Double.POSITIVE_INFINITY, 1 ) ),
 
+    /** Difference in Equitable Threat Score. */
+    EQUITABLE_THREAT_SCORE_DIFFERENCE( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, true,
+                                       new Limits( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN ) ),
+
     /** Frequency Bias. */
-    FREQUENCY_BIAS( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE,
+    FREQUENCY_BIAS( SampleDataGroup.DICHOTOMOUS,
+                    StatisticType.DOUBLE_SCORE,
                     new Limits( 0, Double.POSITIVE_INFINITY, 1 ) ),
 
+    /** Difference in Frequency Bias. */
+    FREQUENCY_BIAS_DIFFERENCE( SampleDataGroup.DICHOTOMOUS,
+                               StatisticType.DOUBLE_SCORE,
+                               true,
+                               new Limits( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN ) ),
+
     /** Index of Agreement. */
     INDEX_OF_AGREEMENT( SampleDataGroup.SINGLE_VALUED, StatisticType.DOUBLE_SCORE,
                         new Limits( 0, 1, 1 ) ),
@@ -171,13 +191,28 @@ public enum MetricConstants
                         StatisticType.DOUBLE_SCORE, true,
                         new Limits( -1, 1, 1 ) ),
 
+    /** Difference in Peirce Skill Score. */
+    PEIRCE_SKILL_SCORE_DIFFERENCE( new SampleDataGroup[] { SampleDataGroup.DICHOTOMOUS, SampleDataGroup.MULTICATEGORY },
+                                   StatisticType.DOUBLE_SCORE, true,
+                                   new Limits( -2, 2, Double.NaN ) ),
+
     /** Probability Of Detection. */
-    PROBABILITY_OF_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE,
-                              new Limits( 0, 1, 1 ) ),
+    PROBABILITY_OF_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 1 ) ),
+
+    /** Difference in Probability Of Detection. */
+    PROBABILITY_OF_DETECTION_DIFFERENCE( SampleDataGroup.DICHOTOMOUS,
+                                         StatisticType.DOUBLE_SCORE,
+                                         true,
+                                         new Limits( -1, 1, Double.NaN ) ),
 
     /** Probability Of False Detection.*/
-    PROBABILITY_OF_FALSE_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE,
-                                    new Limits( 0, 1, 0 ) ),
+    PROBABILITY_OF_FALSE_DETECTION( SampleDataGroup.DICHOTOMOUS, StatisticType.DOUBLE_SCORE, new Limits( 0, 1, 0 ) ),
+
+    /** Difference in Probability Of False Detection.*/
+    PROBABILITY_OF_FALSE_DETECTION_DIFFERENCE( SampleDataGroup.DICHOTOMOUS,
+                                               StatisticType.DOUBLE_SCORE,
+                                               true,
+                                               new Limits( -1, 1, Double.NaN ) ),
 
     /** Quantile-quantile diagram. */
     QUANTILE_QUANTILE_DIAGRAM( SampleDataGroup.SINGLE_VALUED, StatisticType.DIAGRAM,

diff --git a/wres-metrics/src/wres/metrics/MetricFactory.java b/wres-metrics/src/wres/metrics/MetricFactory.java
@@ -75,16 +75,10 @@
 
 public final class MetricFactory
 {
-    /**
-     * String used in several error messages to denote an unrecognized metric.
-     */
-
+    /** String used in several error messages to denote an unrecognized metric. */
     private static final String UNRECOGNIZED_METRIC_ERROR = "Unrecognized metric for identifier.";
 
-    /**
-     * Test seed system property name.
-     */
-
+    /** Test seed system property name. */
     private static final String TEST_SEED_PROPERTY = "wres.systemTestSeed";
 
     /**
@@ -388,6 +382,10 @@ public final class MetricFactory
                         ( Collectable<Pool<Pair<Boolean, Boolean>>, DoubleScoreStatisticOuter, DoubleScoreStatisticOuter> ) m;
                 builder.addCollectableMetric( c );
             }
+            else
+            {
+                builder.addMetric( m );
+            }
         }
         builder.setExecutorService( executor );
         return builder.build();
@@ -649,13 +647,21 @@ public final class MetricFactory
         return switch ( metric )
         {
             case THREAT_SCORE -> ThreatScore.of();
+            case THREAT_SCORE_DIFFERENCE -> DoubleScoreDifference.of( ThreatScore.of() );
             case EQUITABLE_THREAT_SCORE -> EquitableThreatScore.of();
+            case EQUITABLE_THREAT_SCORE_DIFFERENCE -> DoubleScoreDifference.of( EquitableThreatScore.of() );
             case PEIRCE_SKILL_SCORE -> PeirceSkillScore.of();
+            case PEIRCE_SKILL_SCORE_DIFFERENCE -> DoubleScoreDifference.of( PeirceSkillScore.of() );
             case PROBABILITY_OF_DETECTION -> ProbabilityOfDetection.of();
+            case PROBABILITY_OF_DETECTION_DIFFERENCE -> DoubleScoreDifference.of( ProbabilityOfDetection.of() );
             case PROBABILITY_OF_FALSE_DETECTION -> ProbabilityOfFalseDetection.of();
+            case PROBABILITY_OF_FALSE_DETECTION_DIFFERENCE ->
+                    DoubleScoreDifference.of( ProbabilityOfFalseDetection.of() );
             case FREQUENCY_BIAS -> FrequencyBias.of();
+            case FREQUENCY_BIAS_DIFFERENCE -> DoubleScoreDifference.of( FrequencyBias.of() );
             case CONTINGENCY_TABLE -> ContingencyTable.of();
             case FALSE_ALARM_RATIO -> FalseAlarmRatio.of();
+            case FALSE_ALARM_RATIO_DIFFERENCE -> DoubleScoreDifference.of( FalseAlarmRatio.of() );
             default -> throw new IllegalArgumentException( UNRECOGNIZED_METRIC_ERROR + " '" + metric + "'." );
         };
     }
@@ -846,6 +852,12 @@ private static boolean isCollectable( MetricConstants metric )
     {
         Objects.requireNonNull( metric, "Specify a non-null metric to test." );
 
+        // DoubleScoreDifference does not implement Collectable
+        if ( metric.isDifferenceMetric() )
+        {
+            return false;
+        }
+
         boolean singleValued = metric == MetricConstants.COEFFICIENT_OF_DETERMINATION
                                || metric == MetricConstants.PEARSON_CORRELATION_COEFFICIENT
                                || metric == MetricConstants.SUM_OF_SQUARE_ERROR;

diff --git a/wres-statistics/nonsrc/wresproto/metric_name.proto b/wres-statistics/nonsrc/wresproto/metric_name.proto
@@ -103,4 +103,11 @@ enum MetricName
     CONTINUOUS_RANKED_PROBABILITY_SCORE_DIFFERENCE = 88;
     BRIER_SCORE_DIFFERENCE = 89;
     RELATIVE_OPERATING_CHARACTERISTIC_SCORE_DIFFERENCE = 90;
+    THREAT_SCORE_DIFFERENCE = 91;
+    EQUITABLE_THREAT_SCORE_DIFFERENCE = 92;
+    FREQUENCY_BIAS_DIFFERENCE = 93;
+    PEIRCE_SKILL_SCORE_DIFFERENCE = 94;
+    PROBABILITY_OF_DETECTION_DIFFERENCE = 95;
+    PROBABILITY_OF_FALSE_DETECTION_DIFFERENCE = 96;
+    FALSE_ALARM_RATIO_DIFFERENCE = 97;
 }