Correct index mapping types (#666)

* Correct index mapping types Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add PR number Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Correct numeric types Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Specify default Property variant Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add test for icu_collation_keyword Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Rename test folder Signed-off-by: Thomas Farr <tsfarr@amazon.com> --------- Signed-off-by: Thomas Farr <tsfarr@amazon.com>
opensearch-project · Nov 13, 2024 · 6065144 · 6065144
1 parent 0871d02
commit 6065144
Show file tree

Hide file tree

Showing 9 changed files with 163 additions and 145 deletions.
diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
@@ -44,6 +44,8 @@ jobs:
             tests: plugins/query_insights
           - version: 2.18.0
             tests: plugins/workload-management
+          - version: 2.18.0
+            tests: plugins/analysis
           - version: 2.19.0
             hub: opensearchstaging
             ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,9 +11,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `_search` with `sort: direction` ([#658](https://github.com/opensearch-project/opensearch-api-specification/pull/658))
 - Added `_common.mapping:FlatObjectProperty` ([#661](https://github.com/opensearch-project/opensearch-api-specification/pull/661)) 
 - Added `HEAD /{index}/_doc/{id}` returning `404` ([#670](https://github.com/opensearch-project/opensearch-api-specification/pull/670))
+- Added `_common.mapping:IcuCollationKeywordProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
+- Removed unsupported `_common.mapping:DenseVectorProperty`, `_common.mapping:SparseVectorProperty`, and `_common.mapping:FlattenedProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666))
+- Removed unsupported time series dimension properties and types ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666))
+- Removed unsupported runtime field properties (`script`/`on_script_error`) on `NumberPropertyBase` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) 
 
 ### Fixed
 - Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))

diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml
@@ -2045,7 +2045,7 @@ components:
                   The index template with the highest priority is chosen.
                   If no priority is specified the template is treated as though it is of priority 0 (lowest priority).
                   This number is not automatically generated by OpenSearch.
-                type: number
+                type: integer
               version:
                 $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber'
               _meta:
@@ -2225,7 +2225,7 @@ components:
                   The index template with the highest priority is chosen.
                   If no priority is specified the template is treated as though it is of priority 0 (lowest priority).
                   This number is not automatically generated by OpenSearch.
-                type: number
+                type: integer
               version:
                 $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber'
               _meta:

diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml
@@ -109,6 +109,7 @@ components:
       type: object
       discriminator:
         propertyName: type
+        x-default: object
       oneOf:
         - $ref: '#/components/schemas/BinaryProperty'
         - $ref: '#/components/schemas/BooleanProperty'
@@ -125,9 +126,6 @@ components:
         - $ref: '#/components/schemas/DateNanosProperty'
         - $ref: '#/components/schemas/DateProperty'
         - $ref: '#/components/schemas/AggregateMetricDoubleProperty'
-        - $ref: '#/components/schemas/DenseVectorProperty'
-        - $ref: '#/components/schemas/SparseVectorProperty'
-        - $ref: '#/components/schemas/FlattenedProperty'
         - $ref: '#/components/schemas/FlatObjectProperty'
         - $ref: '#/components/schemas/NestedProperty'
         - $ref: '#/components/schemas/ObjectProperty'
@@ -158,6 +156,7 @@ components:
         - $ref: '#/components/schemas/IpRangeProperty'
         - $ref: '#/components/schemas/LongRangeProperty'
         - $ref: '#/components/schemas/KnnVectorProperty'
+        - $ref: '#/components/schemas/IcuCollationKeywordProperty'
     BinaryProperty:
       allOf:
         - $ref: '#/components/schemas/DocValuesPropertyBase'
@@ -227,19 +226,6 @@ components:
                 - boolean
           required:
             - type
-    OnScriptError:
-      type: string
-      enum:
-        - continue
-        - fail
-    TimeSeriesMetricType:
-      type: string
-      enum:
-        - counter
-        - gauge
-        - histogram
-        - position
-        - summary
     IndexOptions:
       type: string
       enum:
@@ -310,9 +296,6 @@ components:
               type: string
             split_queries_on_whitespace:
               type: boolean
-            time_series_dimension:
-              description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false.
-              type: boolean
             type:
               type: string
               enum:
@@ -535,86 +518,10 @@ components:
               type: array
               items:
                 type: string
-            time_series_metric:
-              $ref: '#/components/schemas/TimeSeriesMetricType'
           required:
             - default_metric
             - metrics
             - type
-    DenseVectorProperty:
-      allOf:
-        - $ref: '#/components/schemas/PropertyBase'
-        - type: object
-          properties:
-            type:
-              type: string
-              enum:
-                - dense_vector
-            dims:
-              type: number
-            similarity:
-              type: string
-            index:
-              type: boolean
-            index_options:
-              $ref: '#/components/schemas/DenseVectorIndexOptions'
-          required:
-            - dims
-            - type
-    DenseVectorIndexOptions:
-      type: object
-      properties:
-        type:
-          type: string
-        m:
-          type: number
-        ef_construction:
-          type: number
-      required:
-        - ef_construction
-        - m
-        - type
-    SparseVectorProperty:
-      allOf:
-        - $ref: '#/components/schemas/PropertyBase'
-        - type: object
-          properties:
-            type:
-              type: string
-              enum:
-                - sparse_vector
-          required:
-            - type
-    FlattenedProperty:
-      allOf:
-        - $ref: '#/components/schemas/PropertyBase'
-        - type: object
-          properties:
-            boost:
-              type: number
-              format: double
-            depth_limit:
-              type: number
-            doc_values:
-              type: boolean
-            eager_global_ordinals:
-              type: boolean
-            index:
-              type: boolean
-            index_options:
-              $ref: '#/components/schemas/IndexOptions'
-            null_value:
-              type: string
-            similarity:
-              type: string
-            split_queries_on_whitespace:
-              type: boolean
-            type:
-              type: string
-              enum:
-                - flattened
-          required:
-            - type
     FlatObjectProperty:
       allOf:
         - $ref: '#/components/schemas/PropertyBase'
@@ -751,13 +658,6 @@ components:
               type: boolean
             null_value:
               type: string
-            on_script_error:
-              $ref: '#/components/schemas/OnScriptError'
-            script:
-              $ref: '_common.yaml#/components/schemas/Script'
-            time_series_dimension:
-              description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false.
-              type: boolean
             type:
               type: string
               enum:
@@ -843,20 +743,22 @@ components:
             - type
     GeoOrientation:
       oneOf:
+        # eslint-disable yml/sort-sequence-values
         - title: left
           type: string
           enum:
+            - left
             - LEFT
             - clockwise
             - cw
-            - left
         - title: right
           type: string
           enum:
+            - right
             - RIGHT
-            - ccw
             - counterclockwise
-            - right
+            - ccw
+        # eslint-enable yml/sort-sequence-values
     GeoStrategy:
       type: string
       enum:
@@ -927,15 +829,6 @@ components:
               type: boolean
             index:
               type: boolean
-            on_script_error:
-              $ref: '#/components/schemas/OnScriptError'
-            script:
-              $ref: '_common.yaml#/components/schemas/Script'
-            time_series_metric:
-              $ref: '#/components/schemas/TimeSeriesMetricType'
-            time_series_dimension:
-              description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false.
-              type: boolean
     DoubleNumberProperty:
       allOf:
         - $ref: '#/components/schemas/NumberPropertyBase'
@@ -1168,6 +1061,43 @@ components:
           required:
             - dimension
             - type
+    IcuCollationKeywordProperty:
+      allOf:
+        - $ref: '#/components/schemas/DocValuesPropertyBase'
+        - type: object
+          properties:
+            type:
+              type: string
+              enum:
+                - icu_collation_keyword
+            index:
+              type: boolean
+            null_value:
+              type: string
+            alternate:
+              $ref: '_common.analysis.yaml#/components/schemas/IcuCollationAlternate'
+            case_level:
+              type: boolean
+            case_first:
+              $ref: '_common.analysis.yaml#/components/schemas/IcuCollationCaseFirst'
+            decomposition:
+              $ref: '_common.analysis.yaml#/components/schemas/IcuCollationDecomposition'
+            hiragana_quaternary_mode:
+              type: boolean
+            numeric:
+              type: boolean
+            strength:
+              $ref: '_common.analysis.yaml#/components/schemas/IcuCollationStrength'
+            variable_top:
+              type: string
+            country:
+              type: string
+            language:
+              type: string
+            variant:
+              type: string
+          required:
+            - type
     MatchType:
       type: string
       enum:

diff --git a/spec/schemas/_core.field_caps.yaml b/spec/schemas/_core.field_caps.yaml
@@ -28,25 +28,6 @@ components:
         metadata_field:
           description: Whether this field is registered as a metadata field.
           type: boolean
-        time_series_dimension:
-          description: Whether this field is used as a time series dimension.
-          type: boolean
-        time_series_metric:
-          $ref: '_common.mapping.yaml#/components/schemas/TimeSeriesMetricType'
-        non_dimension_indices:
-          description: |-
-            If this list is present in the response, then indexes not contained in the list have the
-            field marked as a dimension. Any indexes contained in the list are not marked as a dimension.
-          type: array
-          items:
-            $ref: '_common.yaml#/components/schemas/IndexName'
-        metric_conflicts_indices:
-          description: |-
-            The list of indexes in which this field is present if the indexes
-            don't have the same `time_series_metric` value for the field.
-          type: array
-          items:
-            $ref: '_common.yaml#/components/schemas/IndexName'
       required:
         - aggregatable
         - searchable

diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml
@@ -148,8 +148,6 @@ components:
           $ref: '#/components/schemas/IndexSettingsAnalysis'
         settings:
           $ref: '#/components/schemas/IndexSettings'
-        time_series:
-          $ref: '#/components/schemas/IndexSettingsTimeSeries'
         queries:
           $ref: '#/components/schemas/IndexSettingsQueries'
         similarity:
@@ -457,13 +455,6 @@ components:
           type: object
           additionalProperties:
             $ref: '_common.analysis.yaml#/components/schemas/Tokenizer'
-    IndexSettingsTimeSeries:
-      type: object
-      properties:
-        end_time:
-          $ref: '_common.yaml#/components/schemas/DateTime'
-        start_time:
-          $ref: '_common.yaml#/components/schemas/DateTime'
     IndexSettingsQueries:
       type: object
       properties:
@@ -919,7 +910,8 @@ components:
             The index template with the highest priority is chosen.
             If no priority is specified the template is treated as though it is of priority 0 (lowest priority).
             This number is not automatically generated by OpenSearch.
-          type: number
+          type: integer
+          format: int64
         _meta:
           $ref: '_common.yaml#/components/schemas/Metadata'
         allow_auto_create:

diff --git a/tests/plugins/analysis/Dockerfile b/tests/plugins/analysis/Dockerfile
@@ -0,0 +1,6 @@
+ARG OPENSEARCH_DOCKER_HUB_PROJECT
+ARG OPENSEARCH_VERSION
+ARG OPENSEARCH_DOCKER_REF
+
+FROM ${OPENSEARCH_DOCKER_HUB_PROJECT}/opensearch:${OPENSEARCH_VERSION}${OPENSEARCH_DOCKER_REF}
+RUN ./bin/opensearch-plugin install -b analysis-icu
diff --git a/tests/plugins/analysis/docker-compose.yml b/tests/plugins/analysis/docker-compose.yml
@@ -0,0 +1,17 @@
+version: '3'
+
+services:
+  opensearch-cluster:
+    build:
+      context: .
+      args:
+        - OPENSEARCH_DOCKER_HUB_PROJECT=${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}
+        - OPENSEARCH_DOCKER_REF=${OPENSEARCH_DOCKER_REF}
+        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
+    ports:
+      - 9200:9200
+      - 9600:9600
+    environment:
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - discovery.type=single-node