Merge branch 'master' into monitoring

NOAA-OWP · Aug 30, 2024 · 71700cc · 71700cc
2 parents 0fb8b3c + bc99397
commit 71700cc
Show file tree

Hide file tree

Showing 40 changed files with 824 additions and 213 deletions.
diff --git a/build.gradle b/build.gradle
@@ -14,11 +14,11 @@ import java.time.ZoneId
 /* Used by gradle plugins outside the official gradle plugins repository */
 buildscript {
 
-    // Work around for missing transitive dependency for plugin org.kordamp.gradle.markdown
+    // Workaround for missing transitive dependency for plugin org.kordamp.gradle.markdown
     // https://github.com/kordamp/markdown-gradle-plugin/issues/36
     configurations.all {
         resolutionStrategy.dependencySubstitution {
-            substitute module("com.overzealous:remark:1.1.0") using module( 'com.wavefront:remark:2023-07.07' ) because "not available on maven central anymore"
+            substitute module( 'com.overzealous:remark:1.1.0' ) using module( 'com.wavefront:remark:2023-07.07' ) because "not available on maven central anymore"
         }
     }
     repositories {

diff --git a/compose-entry.yml b/compose-entry.yml
@@ -16,7 +16,7 @@ volumes:
             device: "${NFS_HOME_DIR_DEVICE}"
 services:
     persister:
-        image: "${DOCKER_REGISTRY}/wres/wres-redis:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-redis:20240821-1d5b305"
         restart: always
         volumes:
          # For the job data
@@ -33,7 +33,7 @@ services:
     tasker:
         ports:
          - "443:8443"
-        image: "${DOCKER_REGISTRY}/wres/wres-tasker:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-tasker:20240821-1d5b305"
         restart: always
         depends_on:
             broker:
@@ -68,7 +68,7 @@ services:
         ports:
          - "5671:5671"
          - "15671:15671"
-        image: "${DOCKER_REGISTRY}/wres/wres-broker:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-broker:20240821-1d5b305"
         restart: always
         volumes:
          # For certificates and keys that correspond to them:
@@ -87,7 +87,7 @@ services:
         healthcheck:
             start_period: 5m
     worker:
-        image: "${DOCKER_REGISTRY}/wres/wres-worker:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-worker:20240821-1d5b305"
         restart: always
         depends_on:
             broker:
@@ -137,7 +137,7 @@ services:
          - 5673
         ports:
          - 15673:15673
-        image: "${DOCKER_REGISTRY}/wres/wres-eventsbroker:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-eventsbroker:20240821-1d5b305"
         restart: always
         volumes:
          # For heap-dump files
@@ -159,7 +159,7 @@ services:
         depends_on:
             eventsbroker:
                 condition: service_started
-        image: "${DOCKER_REGISTRY}/wres/wres-graphics:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-graphics:20240821-1d5b305"
         restart: always
         volumes:
          # To write graphics outputs
@@ -181,7 +181,7 @@ services:
         depends_on:
             eventsbroker:
                 condition: service_started
-        image: "${DOCKER_REGISTRY}/wres/wres-writing:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-writing:20240821-1d5b305"
         restart: always
         volumes:
          # To write numeric outputs

diff --git a/compose-workers.yml b/compose-workers.yml
@@ -16,7 +16,7 @@ volumes:
             device: "${NFS_HOME_DIR_DEVICE}"
 services:
     worker:
-        image: "${DOCKER_REGISTRY}/wres/wres-worker:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-worker:20240821-1d5b305"
         restart: always
         depends_on:
             eventsbroker:
@@ -64,7 +64,7 @@ services:
          - 5673
         ports:
          - 15673:15673
-        image: "${DOCKER_REGISTRY}/wres/wres-eventsbroker:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-eventsbroker:20240821-1d5b305"
         restart: always
         volumes:
          # For heap-dump files
@@ -86,7 +86,7 @@ services:
         depends_on:
             eventsbroker:
                 condition: service_started
-        image: "${DOCKER_REGISTRY}/wres/wres-graphics:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-graphics:20240821-1d5b305"
         restart: always
         volumes:
          # To write graphics outputs
@@ -108,7 +108,7 @@ services:
         depends_on:
             eventsbroker:
                 condition: service_started
-        image: "${DOCKER_REGISTRY}/wres/wres-writing:20240701-5b07a6e"
+        image: "${DOCKER_REGISTRY}/wres/wres-writing:20240821-1d5b305"
         restart: always
         volumes:
          # To write numeric outputs

diff --git a/src/wres/pipeline/pooling/PoolFactory.java b/src/wres/pipeline/pooling/PoolFactory.java
diff --git a/test/wres/pipeline/EvaluationUtilitiesTest.java b/test/wres/pipeline/EvaluationUtilitiesTest.java
@@ -19,6 +19,7 @@
 import wres.config.yaml.components.EvaluationDeclaration;
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.LeadTimeInterval;
 import wres.config.yaml.components.ThresholdBuilder;
 import wres.config.yaml.components.ThresholdType;
@@ -268,7 +269,9 @@ void testGetSummaryStatisticsCalculatorsWithTwoTimeWindowsAcrossFeatureGroups()
 
         Set<GeometryGroup> geometryGroups = Set.of( firstGroup, secondGroup );
 
-        FeatureGroups featureGroups = new FeatureGroups( geometryGroups );
+        FeatureGroups featureGroups = FeatureGroupsBuilder.builder()
+                                                          .geometryGroups( geometryGroups )
+                                                          .build();
 
         EvaluationDeclaration evaluation = EvaluationDeclarationBuilder.builder()
                                                                        .leadTimes( leadTimeInterval )

diff --git a/testoutput/wres/configcontrol/config/project/verificationProjectTest/.gitignore b/testoutput/wres/configcontrol/config/project/verificationProjectTest/.gitignore
diff --git a/wres-config/nonsrc/schema.yml b/wres-config/nonsrc/schema.yml
@@ -566,9 +566,9 @@ definitions:
     cross-pairing chosen."
     anyOf:
       - "$ref": "#/definitions/CrossPairEnum"
-      - "$ref": "#/definitions/CrossPairFull"
+      - "$ref": "#/definitions/CrossPairWithParameters"
 
-  CrossPairFull:
+  CrossPairWithParameters:
     type: object
     additionalProperties: false
     properties:
@@ -665,25 +665,25 @@ definitions:
     properties:
       observed:
         anyOf:
-          - "$ref": "#/definitions/FeatureTupleFull"
+          - "$ref": "#/definitions/FeatureWithParameters"
           - type: string
             minLength: 1
             maxLength: 32
       predicted:
         anyOf:
-          - "$ref": "#/definitions/FeatureTupleFull"
+          - "$ref": "#/definitions/FeatureWithParameters"
           - type: string
             minLength: 1
             maxLength: 32
       baseline:
         anyOf:
-          - "$ref": "#/definitions/FeatureTupleFull"
+          - "$ref": "#/definitions/FeatureWithParameters"
           - type: string
             minLength: 1
             maxLength: 32
 
-  FeatureTupleFull:
-    title: A feature tuple with additional properties
+  FeatureWithParameters:
+    title: A feature with additional properties
     type: object
     additionalProperties: false
     properties:
@@ -695,6 +695,8 @@ definitions:
         type: string
         minLength: 1
         maxLength: 2083
+      offset:
+        type: number
     required:
       - name
 
@@ -1487,13 +1489,16 @@ definitions:
     is present, evaluates common events by reference time and valid time. If 
     absent, retains all events of each type. With fuzzy matching, each time-
     series is matched with its nearest, corresponding, time-series according to 
-    the total duration between all reference times of a corresponding type. In 
-    other words, if there is an exact match, that will be used, else the time-
-    series whose reference times are nearest overall. Once a time-series has 
-    been matched, it cannot be re-used. Always uses exact matching for valid 
-    times. The scope of the cross-pairing always includes the predicted and 
-    baseline datasets, where defined, and may be further controlled using the 
-    'scope' parameter."
+    the total duration between all reference times across the candidate time-
+    series. In other words, if there is an exact match, that will be used, else 
+    the time-series whose reference times are nearest overall. With exact 
+    matching, both the type of reference time and the time itself must 
+    correspond. With fuzzy matching, all types of reference time are considered 
+    equal and hence used to calculate the total duration between all reference 
+    times of the candidate series. Once a time-series has been matched, it 
+    cannot be re-used. Always uses exact matching for valid times. The scope of 
+    the cross-pairing always includes the predicted and baseline datasets, where 
+    defined, and may be further controlled using the 'scope' parameter."
     type: string
     enum:
       - exact

diff --git a/wres-config/src/wres/config/yaml/DeclarationFactory.java b/wres-config/src/wres/config/yaml/DeclarationFactory.java
@@ -165,11 +165,11 @@ public class DeclarationFactory
                                      .enable( DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY )
                                      .enable( DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY )
                                      .enable( JsonParser.Feature.STRICT_DUPLICATE_DETECTION )
+                                     .disable( DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES )
                                      .build()
                                      .registerModule( new ProtobufModule() )
                                      .registerModule( new JavaTimeModule() )
-                                     .configure( DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES,
-                                                 true );
+                                     .enable( DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES );
 
     /** Mapper for serialization. */
     private static final ObjectMapper SERIALIZER =

diff --git a/wres-config/src/wres/config/yaml/DeclarationInterpolator.java b/wres-config/src/wres/config/yaml/DeclarationInterpolator.java
@@ -34,6 +34,7 @@
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureAuthority;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.Features;
 import wres.config.yaml.components.FeaturesBuilder;
 import wres.config.yaml.components.Formats;
@@ -486,7 +487,8 @@ private static void interpolateSparseFeatures( EvaluationDeclarationBuilder buil
                                                      .geometries();
                 Set<GeometryTuple> denseFeatures = DeclarationInterpolator.interpolateSparseFeatures( features,
                                                                                                       hasBaseline );
-                Features adjustedFeatures = new Features( denseFeatures );
+                Features adjustedFeatures = new Features( denseFeatures, builder.features()
+                                                                                .offsets() );
                 builder.features( adjustedFeatures );
             }
 
@@ -512,7 +514,9 @@ private static void interpolateSparseFeatures( EvaluationDeclarationBuilder buil
                                      .build();
                     adjustedGeoGroups.add( nextAdjustedGroup );
                 }
-                FeatureGroups adjustedFeatureGroups = new FeatureGroups( adjustedGeoGroups );
+                FeatureGroups adjustedFeatureGroups = FeatureGroupsBuilder.builder()
+                                                                          .geometryGroups( adjustedGeoGroups )
+                                                                          .build();
                 builder.featureGroups( adjustedFeatureGroups );
             }
         }

diff --git a/wres-config/src/wres/config/yaml/DeclarationMigrator.java b/wres-config/src/wres/config/yaml/DeclarationMigrator.java
@@ -82,10 +82,12 @@
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureAuthority;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.FeatureService;
 import wres.config.yaml.components.FeatureServiceBuilder;
 import wres.config.yaml.components.FeatureServiceGroup;
 import wres.config.yaml.components.Features;
+import wres.config.yaml.components.FeaturesBuilder;
 import wres.config.yaml.components.Format;
 import wres.config.yaml.components.Formats;
 import wres.config.yaml.components.GeneratedBaseline;
@@ -293,7 +295,9 @@ private static void migrateFeatures( List<NamedFeature> features, EvaluationDecl
         if ( !features.isEmpty() )
         {
             Set<GeometryTuple> geometries = DeclarationMigrator.migrateFeatures( features );
-            Features wrappedFeatures = new Features( geometries );
+            Features wrappedFeatures = FeaturesBuilder.builder()
+                                                      .geometries( geometries )
+                                                      .build();
             builder.features( wrappedFeatures );
         }
     }
@@ -312,7 +316,9 @@ private static void migrateFeatureGroups( List<FeaturePool> featureGroups, Evalu
                     featureGroups.stream()
                                  .map( DeclarationMigrator::migrateFeatureGroup )
                                  .collect( Collectors.toSet() );
-            FeatureGroups wrappedGroups = new FeatureGroups( geometryGroups );
+            FeatureGroups wrappedGroups = FeatureGroupsBuilder.builder()
+                                                              .geometryGroups( geometryGroups )
+                                                              .build();
             builder.featureGroups( wrappedGroups );
         }
     }

diff --git a/wres-config/src/wres/config/yaml/DeclarationUtilities.java b/wres-config/src/wres/config/yaml/DeclarationUtilities.java
@@ -49,6 +49,7 @@
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureAuthority;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.FeatureServiceGroup;
 import wres.config.yaml.components.Features;
 import wres.config.yaml.components.LeadTimeInterval;
@@ -2243,7 +2244,8 @@ private static EvaluationDeclaration removeFeaturesWithoutThresholds( Evaluation
                                                   .collect( Collectors.toSet() );
 
             // Set the new features
-            Features filteredFeatures = new Features( filtered );
+            Features filteredFeatures = new Features( filtered, declaration.features()
+                                                                           .offsets() );
             builder.features( filteredFeatures );
 
             if ( LOGGER.isWarnEnabled()
@@ -2305,7 +2307,9 @@ private static EvaluationDeclaration removeFeaturesWithoutThresholds( Evaluation
                                                   .toList() );
             }
 
-            FeatureGroups finalFeatureGroups = new FeatureGroups( adjustedGroups );
+            FeatureGroups finalFeatureGroups = FeatureGroupsBuilder.builder()
+                                                                   .geometryGroups( adjustedGroups )
+                                                                   .build();
             builder.featureGroups( finalFeatureGroups );
         }
 

diff --git a/wres-config/src/wres/config/yaml/DeclarationValidator.java b/wres-config/src/wres/config/yaml/DeclarationValidator.java
@@ -510,7 +510,6 @@ private static List<EvaluationStatusEvent> validateDatasets( EvaluationDeclarati
         // Data types are valid
         List<EvaluationStatusEvent> typesValid = DeclarationValidator.typesAreValid( declaration );
         events.addAll( typesValid );
-
         // Ensembles cannot be present on both left and right sides
         List<EvaluationStatusEvent> ensembles = DeclarationValidator.ensembleOnOneSideOnly( declaration );
         events.addAll( ensembles );

diff --git a/wres-config/src/wres/config/yaml/components/FeatureGroups.java b/wres-config/src/wres/config/yaml/components/FeatureGroups.java
@@ -1,7 +1,9 @@
 package wres.config.yaml.components;
 
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
@@ -14,15 +16,18 @@
 import wres.config.yaml.deserializers.FeatureGroupsDeserializer;
 import wres.config.yaml.serializers.FeatureGroupsSerializer;
 import wres.statistics.generated.GeometryGroup;
+import wres.statistics.generated.GeometryTuple;
 
 /**
- * Geographic feature groups.
+ * Geographic feature groups and associated offset values (e.g., datum offsets). Absence of an offset for a given
+ * feature with a group implies no/zero offset.
  * @param geometryGroups the feature groups
+ * @param offsets the offset values associated with features in the group, such as a datum offset, if any
  */
 @RecordBuilder
 @JsonSerialize( using = FeatureGroupsSerializer.class )
 @JsonDeserialize( using = FeatureGroupsDeserializer.class )
-public record FeatureGroups( Set<GeometryGroup> geometryGroups )
+public record FeatureGroups( Set<GeometryGroup> geometryGroups, Map<GeometryTuple, Offset> offsets )
 {
     /**
      * Sets the default values.
@@ -40,6 +45,16 @@ public record FeatureGroups( Set<GeometryGroup> geometryGroups )
             // Immutable copy, preserving insertion order
             geometryGroups = Collections.unmodifiableSet( new LinkedHashSet<>( geometryGroups ) );
         }
+
+        if ( Objects.isNull( offsets ) )
+        {
+            offsets = Collections.emptyMap();
+        }
+        else
+        {
+            // Immutable copy, preserving insertion order
+            offsets = Collections.unmodifiableMap( new LinkedHashMap<>( offsets ) );
+        }
     }
 
     @Override