Merge branch 'master' into github_296

README.md

@@ -35,7 +35,7 @@ value_date,variable_name,location,measurement_unit,value
 1985-06-01T14:00:00Z,streamflow,myLocation,CMS,22.0
 ```
 
-* Create a file `observation.csv` with the following content:
+* Create a file `observations.csv` with the following content:
 
 ```
 value_date,variable_name,location,measurement_unit,value
@@ -73,4 +73,4 @@ cd build/install/wres/
 * Execute your project
 ```
 bin/wres myEvaluation.yml
-```
+```

dist/lib/conf/logback.xml

@@ -1,5 +1,4 @@
 <configuration>
-    <!-- Some prefer timestamps, thread name, level, class name, etc. -->
     <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
 
         <!-- Stop multiple JVMs from stepping on one another, see #52867 -->
@@ -23,13 +22,13 @@
           <!-- Always write the log files in UTF-8. -->
           <charset>UTF-8</charset>
 
-          <!-- %exception{full}: full stacktrace in log file.
-               %X{pid}: pid disambiguates multiple wres processes running. -->
-          <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} %X{pid} [%thread] %level %logger - %msg%n%exception{full}</pattern>
+          <!-- %exception{full}: full stacktrace in log file. -->
+          <!-- %X{pid}: pid disambiguates multiple wres processes running. -->
+          <!-- %logger{0}: show only rightmost part of logger name (class) -->
+          <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} %X{pid} [%thread] %level %logger{0} - %msg%n%exception{full}</pattern>
         </encoder>
     </appender>
 
-    <!-- some prefer the message to be printed as formatted in the code -->
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 
         <!-- To allow DEBUG in FILE but only INFO on STDOUT, uncomment the
@@ -48,7 +47,7 @@
           <!-- %exception{short}: just the first line for each exception -->
           <!-- %exception{full}: full stacktrace for each exception -->
           <!-- %logger{0}: show only rightmost part of logger name (class) -->
-          <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} %level %logger{0} %msg%n%exception{full}</pattern>
+          <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} %X{pid} [%thread] %level %logger{0} - %msg%n%exception{full}</pattern>
         </encoder>
     </appender>
 

wres-config/src/wres/config/yaml/DeclarationValidator.java

@@ -122,7 +122,9 @@ public class DeclarationValidator
     private static final String THE_EVALUATION_REQUESTED_THE_SAMPLING_UNCERTAINTY =
             "The evaluation requested the 'sampling_uncertainty' ";
     /** Re-used string. */
-    private static final String AND_TRY_AGAIN = "and try again.";
+    private static final String TRY_AGAIN = "try again.";
+    /** Re-used string. */
+    private static final String AND_TRY_AGAIN = "and " + TRY_AGAIN;
 
     /**
      * Performs validation against the schema, followed by "business-logic" validation if there are no schema
@@ -1246,7 +1248,7 @@ private static List<EvaluationStatusEvent> covariateFiltersAreValid( EvaluationD
                                                              + ( variables.size() - named.size() )
                                                              + ". Please fix the 'minimum' and/or "
                                                              + "'maximum' value associated with these covariates and "
-                                                             + "try again." )
+                                                             + TRY_AGAIN )
                                            .build();
             events.add( event );
         }
@@ -1542,7 +1544,7 @@ else if ( Objects.isNull( dataset.rescaleFunction() )
                                                                           .getFunction()
                                                              + "'. If this is incorrect, please add the correct "
                                                              + "'rescale_function' for the covariate dataset and "
-                                                             + "try again." )
+                                                             + TRY_AGAIN )
                                            .build();
             events.add( event );
         }
@@ -2365,9 +2367,15 @@ private static List<EvaluationStatusEvent> thresholdsAreValid( EvaluationDeclara
         List<EvaluationStatusEvent> events =
                 new ArrayList<>( DeclarationValidator.thresholdSourcesAreValid( declaration ) );
 
+        // Check that value thresholds include units
         List<EvaluationStatusEvent> valueThresholds = DeclarationValidator.valueThresholdsIncludeUnits( declaration );
         events.addAll( valueThresholds );
 
+        // Check that the feature orientation is consistent with other declaration
+        List<EvaluationStatusEvent> featureThresholds =
+                DeclarationValidator.thresholdFeatureNameFromIsValid( declaration );
+        events.addAll( featureThresholds );
+
         return Collections.unmodifiableList( events );
     }
 
@@ -2416,6 +2424,36 @@ private static List<EvaluationStatusEvent> valueThresholdsIncludeUnits( Evaluati
         return Collections.unmodifiableList( events );
     }
 
+    /**
+     * Checks that the {@code feature_name_from} declaration associated with thresholds is valid, specifically that a
+     * baseline dataset exists when a baseline orientation is declared.
+     * @param declaration the declaration
+     * @return the validation events encountered
+     */
+    private static List<EvaluationStatusEvent> thresholdFeatureNameFromIsValid( EvaluationDeclaration declaration )
+    {
+        List<EvaluationStatusEvent> events = new ArrayList<>();
+
+        Set<Threshold> thresholds = DeclarationUtilities.getInbandThresholds( declaration );
+        if ( thresholds.stream()
+                       .anyMatch( t -> t.featureNameFrom() == DatasetOrientation.BASELINE )
+             && !DeclarationUtilities.hasBaseline( declaration ) )
+        {
+            EvaluationStatusEvent event
+                    = EvaluationStatusEvent.newBuilder()
+                                           .setStatusLevel( StatusLevel.ERROR )
+                                           .setEventMessage( "Discovered one or more thresholds with a "
+                                                             + "'feature_name_from' of 'baseline', but the evaluation "
+                                                             + "does not declare a 'baseline' dataset. Please fix the "
+                                                             + "'feature_name_from' or declare a 'baseline' dataset and "
+                                                             + TRY_AGAIN )
+                                           .build();
+            events.add( event );
+        }
+
+        return Collections.unmodifiableList( events );
+    }
+
     /**
      * Checks that the sampling uncertainty declaration is valid.
      * @param declaration the declaration
@@ -2690,13 +2728,13 @@ private static List<EvaluationStatusEvent> thresholdSourceIsValid( ThresholdSour
             EvaluationStatusEvent event
                     = EvaluationStatusEvent.newBuilder()
                                            .setStatusLevel( StatusLevel.ERROR )
-                                           .setEventMessage( "The 'threshold_service' declaration requested that "
+                                           .setEventMessage( "The 'threshold_sources' declaration requested that "
                                                              + "feature names with an orientation of '"
                                                              + DatasetOrientation.BASELINE
                                                              + "' are used to correlate features with thresholds, but "
                                                              + "no 'baseline' dataset was discovered. Please add a "
                                                              + "'baseline' dataset or fix the 'feature_name_from' "
-                                                             + "in the 'threshold_service' declaration." )
+                                                             + "in the 'threshold_sources' declaration." )
                                            .build();
 
             events.add( event );
@@ -2714,7 +2752,7 @@ private static List<EvaluationStatusEvent> thresholdSourceIsValid( ThresholdSour
             EvaluationStatusEvent event
                     = EvaluationStatusEvent.newBuilder()
                                            .setStatusLevel( StatusLevel.ERROR )
-                                           .setEventMessage( "Discovered declaration for a 'threshold_service', which "
+                                           .setEventMessage( "Discovered declaration of 'threshold_sources', which "
                                                              + "requests thresholds whose feature names have an "
                                                              + "orientation of '"
                                                              + DatasetOrientation.BASELINE
@@ -2723,7 +2761,7 @@ private static List<EvaluationStatusEvent> thresholdSourceIsValid( ThresholdSour
                                                              + " feature(s) were discovered with a missing '"
                                                              + DatasetOrientation.BASELINE
                                                              + "' feature name. Please fix the 'feature_name_from' in "
-                                                             + "the 'threshold_service' declaration or supply fully "
+                                                             + "the 'threshold_sources' declaration or supply fully "
                                                              + "composed feature tuples with an appropriate feature "
                                                              + "for the '"
                                                              + DatasetOrientation.BASELINE

wres-config/test/wres/config/yaml/DeclarationValidatorTest.java

@@ -1614,6 +1614,36 @@ void testFeaturefulThresholdsNotCorrelatedWithFeaturesToEvaluateProducesErrors()
         );
     }
 
+    @Test
+    void testFeaturefulThresholdsWithFeatureNameFromBaselineAndNoBaselineProducesError()
+    {
+        Geometry featureFoo = Geometry.newBuilder()
+                                      .setName( "foo" )
+                                      .build();
+        Threshold one = Threshold.newBuilder()
+                                 .setLeftThresholdValue( 1.0 )
+                                 .build();
+        wres.config.yaml.components.Threshold wrappedOne = ThresholdBuilder.builder()
+                                                                           .threshold( one )
+                                                                           .feature( featureFoo )
+                                                                           .featureNameFrom( DatasetOrientation.BASELINE )
+                                                                           .type( ThresholdType.VALUE )
+                                                                           .build();
+        EvaluationDeclaration declaration =
+                EvaluationDeclarationBuilder.builder()
+                                            .left( this.defaultDataset )
+                                            .right( this.defaultDataset )
+                                            .thresholds( Set.of( wrappedOne ) )
+                                            .build();
+
+        List<EvaluationStatusEvent> events = DeclarationValidator.validate( declaration );
+
+        assertTrue( DeclarationValidatorTest.contains( events,
+                                                       "Please fix the 'feature_name_from' or declare a "
+                                                       + "'baseline' dataset",
+                                                       StatusLevel.ERROR ) );
+    }
+
     @Test
     void testFeaturefulThresholdsAndNoFeaturesProducesWarning()
     {

wres-eventsbroker/dist/lib/conf/logback.xml

@@ -2,8 +2,9 @@
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
 
-            <!-- %exception{full}: full stacktrace in log file. -->
-            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} [%thread] %level %logger - %msg%n%exception{full}</pattern>
+            <!-- %exception{full}: full stacktrace for each exception -->
+            <!-- %logger{0}: show only rightmost part of logger name (class) -->
+            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} %X{pid} [%thread] %level %logger{0} - %msg%n%exception{full}</pattern>
         </encoder>
     </appender>
 

wres-reading/src/wres/reading/wrds/ahps/WrdsAhpsReader.java

@@ -586,9 +586,11 @@ private Map<String, String> createWrdsAhpsUrlParameters( Pair<Instant, Instant>
         }
 
         urlParameters.put( timeTag,
-                           "[" + dateRange.getLeft().toString()
+                           "[" + dateRange.getLeft()
+                                          .toString()
                            + ","
-                           + dateRange.getRight().toString()
+                           + dateRange.getRight()
+                                      .toString()
                            + "]" );
 
         return Collections.unmodifiableMap( urlParameters );

wres-reading/src/wres/reading/wrds/nwm/DateTimeDeserializer.java

@@ -0,0 +1,46 @@
+package wres.reading.wrds.nwm;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import wres.reading.ReaderUtilities;
+
+/**
+ * Custom deserializer for a datetime string in the ISO8601 "basic" format with optional minutes and seconds. For
+ * example: 20240830T12Z, 20240830T1200Z and 20240830T120000Z are all acceptable.
+ *
+ * @author James Brown
+ */
+public class DateTimeDeserializer extends JsonDeserializer<Instant>
+{
+    @Override
+    public Instant deserialize( JsonParser jp, DeserializationContext context )
+            throws IOException
+    {
+        JsonNode node = jp.getCodec()
+                          .readTree( jp );
+
+        String time;
+
+        // Parse the instant.
+        if ( node.isTextual() )
+        {
+            time = node.asText();
+        }
+        else
+        {
+            throw new IOException( "Could not find a datetime field in the document, which is not allowed." );
+        }
+
+        // Lenient formatting in the "basic" ISO8601 format, hours and seconds are optional
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern( "yyyyMMdd'T'HH[mm[ss]]'Z'" )
+                                                       .withZone( ReaderUtilities.UTC );
+        return formatter.parse( time, Instant::from );
+    }
+}

wres-reading/src/wres/reading/wrds/nwm/NwmDataPointDeserializer.java

@@ -3,8 +3,6 @@
 import java.io.IOException;
 import java.io.Serial;
 import java.time.Instant;
-import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeFormatterBuilder;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.DeserializationContext;
@@ -15,13 +13,17 @@
 
 /**
  * Custom deserializer to allow for handling a null value in a NWM data point.
- * @author Hank.Herr
+ * @author Hank Herr
+ * @author James Brown
  */
 public class NwmDataPointDeserializer extends StdDeserializer<NwmDataPoint>
 {
     @Serial
     private static final long serialVersionUID = 5616289115474402095L;
 
+    /** Deserializer for a datetime {@link Instant}. **/
+    private static final DateTimeDeserializer INSTANT_DESERIALIZER = new DateTimeDeserializer();
+
     /**
      * Creates an instance.
      *
@@ -43,21 +45,25 @@ public NwmDataPointDeserializer()
     @Override
     public NwmDataPoint deserialize( JsonParser jp, DeserializationContext ctxt ) throws IOException
     {
-        JsonNode node = jp.getCodec().readTree( jp );
+        JsonNode node = jp.getCodec()
+                          .readTree( jp );
+
+        JsonNode timeNode = node.get( "time" );
+        JsonParser parser = timeNode.traverse();
+        parser.setCodec( jp.getCodec() );
 
-        //Parse the instant.
-        DateTimeFormatter formatter = new DateTimeFormatterBuilder()
-                .appendPattern( "uuuuMMdd'T'HHX" )
-                .toFormatter();
-        Instant instant = formatter.parse( node.get( "time" ).asText(), Instant::from );
+        // Parse the instant.
+        Instant instant = INSTANT_DESERIALIZER.deserialize( parser, ctxt );
 
-        //Parse the value.  Note that if the value is null, the node will not be
-        //null.  Rather, isNull will return true.  So there is not need to check
-        //explicitly for null.
+        // Parse the value.  Note that if the value is null, the node will not be
+        // null.  Rather, isNull will return true.  So there is no need to check
+        // explicitly for null.
         double value = MissingValues.DOUBLE;
-        if ( !node.get( "value" ).isNull() )
+        if ( !node.get( "value" )
+                  .isNull() )
         {
-            value = Double.parseDouble( node.get( "value" ).asText() );
+            value = node.get( "value" )
+                        .asDouble();
         }
 
         return new NwmDataPoint( instant, value );

wres-reading/src/wres/reading/wrds/nwm/NwmForecast.java

@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 
 /**
@@ -28,6 +29,7 @@ public class NwmForecast
     public NwmForecast( @JsonProperty( "reference_time" )
                         @JsonFormat( shape = JsonFormat.Shape.STRING,
                                      pattern = "uuuuMMdd'T'HHX" )
+                        @JsonDeserialize( using = DateTimeDeserializer.class )
                         Instant referenceDatetime,
                         @JsonProperty( "features" )
                         List<NwmFeature> nwmFeatures )

wres-reading/src/wres/reading/wrds/nwm/WrdsNwmJsonReader.java

@@ -158,7 +158,7 @@ private Supplier<TimeSeriesTuple> getTimeSeriesSupplier( DataSource dataSource,
         // Create a supplier that returns a time-series once complete
         return () -> {
 
-            // Read all of the time-series eagerly on first use: this will still delay any read until a terminal stream
+            // Read the time-series eagerly on first use: this will still delay any read until a terminal stream
             // operation pulls from the supplier (which is why we use a reference holder and do not request the 
             // time-series outside of this lambda), but it will then acquire all the time-series eagerly, i.e., now
             if ( Objects.isNull( timeSeriesTuples.get() ) )