From 52b95a158a891afc1866b9c1ce1ac2838a2b1919 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 13 Nov 2024 12:42:37 +1300 Subject: [PATCH 1/6] Correct index mapping types Signed-off-by: Thomas Farr --- CHANGELOG.md | 4 + spec/schemas/_common.mapping.yaml | 151 +++++++---------------------- spec/schemas/_core.field_caps.yaml | 19 ---- spec/schemas/indices._common.yaml | 9 -- 4 files changed, 41 insertions(+), 142 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 886986786..46b2ae2b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `_search` with `sort: direction` ([#658](https://github.com/opensearch-project/opensearch-api-specification/pull/658)) - Added `_common.mapping:FlatObjectProperty` ([#661](https://github.com/opensearch-project/opensearch-api-specification/pull/661)) - Added `HEAD /{index}/_doc/{id}` returning `404` ([#670](https://github.com/opensearch-project/opensearch-api-specification/pull/670)) +- Added `_common.mapping:IcuCollationKeywordProperty` ([#]()) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) +- Removed unsupported `_common.mapping:DenseVectorProperty`, `_common.mapping:SparseVectorProperty`, and `_common.mapping:FlattenedProperty` ([#]()) +- Removed unsupported time series dimension properties and types ([#]()) +- Removed unsupported runtime field properties (`script`/`on_script_error`) on `NumberPropertyBase` ([#]()) ### Fixed - Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 9e2bc2c5f..334076cea 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -125,9 +125,6 @@ components: - $ref: '#/components/schemas/DateNanosProperty' - $ref: '#/components/schemas/DateProperty' - $ref: '#/components/schemas/AggregateMetricDoubleProperty' - - $ref: '#/components/schemas/DenseVectorProperty' - - $ref: '#/components/schemas/SparseVectorProperty' - - $ref: '#/components/schemas/FlattenedProperty' - $ref: '#/components/schemas/FlatObjectProperty' - $ref: '#/components/schemas/NestedProperty' - $ref: '#/components/schemas/ObjectProperty' @@ -158,6 +155,7 @@ components: - $ref: '#/components/schemas/IpRangeProperty' - $ref: '#/components/schemas/LongRangeProperty' - $ref: '#/components/schemas/KnnVectorProperty' + - $ref: '#/components/schemas/IcuCollationKeywordProperty' BinaryProperty: allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' @@ -227,19 +225,6 @@ components: - boolean required: - type - OnScriptError: - type: string - enum: - - continue - - fail - TimeSeriesMetricType: - type: string - enum: - - counter - - gauge - - histogram - - position - - summary IndexOptions: type: string enum: @@ -310,9 +295,6 @@ components: type: string split_queries_on_whitespace: type: boolean - time_series_dimension: - description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false. - type: boolean type: type: string enum: @@ -535,86 +517,10 @@ components: type: array items: type: string - time_series_metric: - $ref: '#/components/schemas/TimeSeriesMetricType' required: - default_metric - metrics - type - DenseVectorProperty: - allOf: - - $ref: '#/components/schemas/PropertyBase' - - type: object - properties: - type: - type: string - enum: - - dense_vector - dims: - type: number - similarity: - type: string - index: - type: boolean - index_options: - $ref: '#/components/schemas/DenseVectorIndexOptions' - required: - - dims - - type - DenseVectorIndexOptions: - type: object - properties: - type: - type: string - m: - type: number - ef_construction: - type: number - required: - - ef_construction - - m - - type - SparseVectorProperty: - allOf: - - $ref: '#/components/schemas/PropertyBase' - - type: object - properties: - type: - type: string - enum: - - sparse_vector - required: - - type - FlattenedProperty: - allOf: - - $ref: '#/components/schemas/PropertyBase' - - type: object - properties: - boost: - type: number - format: double - depth_limit: - type: number - doc_values: - type: boolean - eager_global_ordinals: - type: boolean - index: - type: boolean - index_options: - $ref: '#/components/schemas/IndexOptions' - null_value: - type: string - similarity: - type: string - split_queries_on_whitespace: - type: boolean - type: - type: string - enum: - - flattened - required: - - type FlatObjectProperty: allOf: - $ref: '#/components/schemas/PropertyBase' @@ -751,13 +657,6 @@ components: type: boolean null_value: type: string - on_script_error: - $ref: '#/components/schemas/OnScriptError' - script: - $ref: '_common.yaml#/components/schemas/Script' - time_series_dimension: - description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false. - type: boolean type: type: string enum: @@ -843,20 +742,22 @@ components: - type GeoOrientation: oneOf: + # eslint-disable yml/sort-sequence-values - title: left type: string enum: + - left - LEFT - clockwise - cw - - left - title: right type: string enum: + - right - RIGHT - - ccw - counterclockwise - - right + - ccw + # eslint-enable yml/sort-sequence-values GeoStrategy: type: string enum: @@ -927,15 +828,6 @@ components: type: boolean index: type: boolean - on_script_error: - $ref: '#/components/schemas/OnScriptError' - script: - $ref: '_common.yaml#/components/schemas/Script' - time_series_metric: - $ref: '#/components/schemas/TimeSeriesMetricType' - time_series_dimension: - description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false. - type: boolean DoubleNumberProperty: allOf: - $ref: '#/components/schemas/NumberPropertyBase' @@ -1168,6 +1060,37 @@ components: required: - dimension - type + IcuCollationKeywordProperty: + allOf: + - $ref: '#/components/schemas/DocValuesPropertyBase' + - type: object + properties: + type: + type: string + enum: + - icu_collation_keyword + index: + type: boolean + null_value: + type: string + alternate: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationAlternate' + case_level: + type: boolean + case_first: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationCaseFirst' + decomposition: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationDecomposition' + hiragana_quaternary_mode: + type: boolean + numeric: + type: boolean + strength: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationStrength' + variable_top: + type: string + required: + - type MatchType: type: string enum: diff --git a/spec/schemas/_core.field_caps.yaml b/spec/schemas/_core.field_caps.yaml index 70b93d6d5..67d8b308c 100644 --- a/spec/schemas/_core.field_caps.yaml +++ b/spec/schemas/_core.field_caps.yaml @@ -28,25 +28,6 @@ components: metadata_field: description: Whether this field is registered as a metadata field. type: boolean - time_series_dimension: - description: Whether this field is used as a time series dimension. - type: boolean - time_series_metric: - $ref: '_common.mapping.yaml#/components/schemas/TimeSeriesMetricType' - non_dimension_indices: - description: |- - If this list is present in the response, then indexes not contained in the list have the - field marked as a dimension. Any indexes contained in the list are not marked as a dimension. - type: array - items: - $ref: '_common.yaml#/components/schemas/IndexName' - metric_conflicts_indices: - description: |- - The list of indexes in which this field is present if the indexes - don't have the same `time_series_metric` value for the field. - type: array - items: - $ref: '_common.yaml#/components/schemas/IndexName' required: - aggregatable - searchable diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index 10386d1ea..f7aa687f7 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -148,8 +148,6 @@ components: $ref: '#/components/schemas/IndexSettingsAnalysis' settings: $ref: '#/components/schemas/IndexSettings' - time_series: - $ref: '#/components/schemas/IndexSettingsTimeSeries' queries: $ref: '#/components/schemas/IndexSettingsQueries' similarity: @@ -457,13 +455,6 @@ components: type: object additionalProperties: $ref: '_common.analysis.yaml#/components/schemas/Tokenizer' - IndexSettingsTimeSeries: - type: object - properties: - end_time: - $ref: '_common.yaml#/components/schemas/DateTime' - start_time: - $ref: '_common.yaml#/components/schemas/DateTime' IndexSettingsQueries: type: object properties: From 592908160e31ec1f2d1a3d75e9fd490975e15f49 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 13 Nov 2024 12:44:09 +1300 Subject: [PATCH 2/6] Add PR number Signed-off-by: Thomas Farr --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b2ae2b1..05f3ba6b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `_search` with `sort: direction` ([#658](https://github.com/opensearch-project/opensearch-api-specification/pull/658)) - Added `_common.mapping:FlatObjectProperty` ([#661](https://github.com/opensearch-project/opensearch-api-specification/pull/661)) - Added `HEAD /{index}/_doc/{id}` returning `404` ([#670](https://github.com/opensearch-project/opensearch-api-specification/pull/670)) -- Added `_common.mapping:IcuCollationKeywordProperty` ([#]()) +- Added `_common.mapping:IcuCollationKeywordProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) -- Removed unsupported `_common.mapping:DenseVectorProperty`, `_common.mapping:SparseVectorProperty`, and `_common.mapping:FlattenedProperty` ([#]()) -- Removed unsupported time series dimension properties and types ([#]()) -- Removed unsupported runtime field properties (`script`/`on_script_error`) on `NumberPropertyBase` ([#]()) +- Removed unsupported `_common.mapping:DenseVectorProperty`, `_common.mapping:SparseVectorProperty`, and `_common.mapping:FlattenedProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) +- Removed unsupported time series dimension properties and types ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) +- Removed unsupported runtime field properties (`script`/`on_script_error`) on `NumberPropertyBase` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) ### Fixed - Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) From c65bf4f9e83c92beabd8b3967ee7fe0b7f15e8d8 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 13 Nov 2024 18:01:21 +1300 Subject: [PATCH 3/6] Correct numeric types Signed-off-by: Thomas Farr --- spec/namespaces/indices.yaml | 4 ++-- spec/schemas/indices._common.yaml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index fcf676adf..cae77c601 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2045,7 +2045,7 @@ components: The index template with the highest priority is chosen. If no priority is specified the template is treated as though it is of priority 0 (lowest priority). This number is not automatically generated by OpenSearch. - type: number + type: integer version: $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber' _meta: @@ -2225,7 +2225,7 @@ components: The index template with the highest priority is chosen. If no priority is specified the template is treated as though it is of priority 0 (lowest priority). This number is not automatically generated by OpenSearch. - type: number + type: integer version: $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber' _meta: diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index f7aa687f7..6135bd368 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -910,7 +910,8 @@ components: The index template with the highest priority is chosen. If no priority is specified the template is treated as though it is of priority 0 (lowest priority). This number is not automatically generated by OpenSearch. - type: number + type: integer + format: int64 _meta: $ref: '_common.yaml#/components/schemas/Metadata' allow_auto_create: From 1a248a485028a62786497249b53f5f5c7000b989 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 14 Nov 2024 00:16:37 +1300 Subject: [PATCH 4/6] Specify default Property variant Signed-off-by: Thomas Farr --- spec/schemas/_common.mapping.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 334076cea..563aa2f2c 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -109,6 +109,7 @@ components: type: object discriminator: propertyName: type + x-default: 'object' oneOf: - $ref: '#/components/schemas/BinaryProperty' - $ref: '#/components/schemas/BooleanProperty' From 39555cebec5c6d3ffe261270bc3b3e1d0a4cdcb8 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 14 Nov 2024 09:53:57 +1300 Subject: [PATCH 5/6] Add test for icu_collation_keyword Signed-off-by: Thomas Farr --- .github/workflows/test-spec.yml | 2 + spec/schemas/_common.mapping.yaml | 8 +- tests/plugins/analysis-icu/Dockerfile | 6 ++ tests/plugins/analysis-icu/docker-compose.yml | 17 ++++ .../mapping/icu_collation_keyword.yaml | 81 +++++++++++++++++++ 5 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 tests/plugins/analysis-icu/Dockerfile create mode 100644 tests/plugins/analysis-icu/docker-compose.yml create mode 100644 tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 03e9b9cb6..b15a6280d 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -44,6 +44,8 @@ jobs: tests: plugins/query_insights - version: 2.18.0 tests: plugins/workload-management + - version: 2.18.0 + tests: plugins/analysis-icu - version: 2.19.0 hub: opensearchstaging ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 563aa2f2c..12087d9ed 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -109,7 +109,7 @@ components: type: object discriminator: propertyName: type - x-default: 'object' + x-default: object oneOf: - $ref: '#/components/schemas/BinaryProperty' - $ref: '#/components/schemas/BooleanProperty' @@ -1090,6 +1090,12 @@ components: $ref: '_common.analysis.yaml#/components/schemas/IcuCollationStrength' variable_top: type: string + country: + type: string + language: + type: string + variant: + type: string required: - type MatchType: diff --git a/tests/plugins/analysis-icu/Dockerfile b/tests/plugins/analysis-icu/Dockerfile new file mode 100644 index 000000000..aa43b705b --- /dev/null +++ b/tests/plugins/analysis-icu/Dockerfile @@ -0,0 +1,6 @@ +ARG OPENSEARCH_DOCKER_HUB_PROJECT +ARG OPENSEARCH_VERSION +ARG OPENSEARCH_DOCKER_REF + +FROM ${OPENSEARCH_DOCKER_HUB_PROJECT}/opensearch:${OPENSEARCH_VERSION}${OPENSEARCH_DOCKER_REF} +RUN ./bin/opensearch-plugin install -b analysis-icu diff --git a/tests/plugins/analysis-icu/docker-compose.yml b/tests/plugins/analysis-icu/docker-compose.yml new file mode 100644 index 000000000..0f4c7da5d --- /dev/null +++ b/tests/plugins/analysis-icu/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + opensearch-cluster: + build: + context: . + args: + - OPENSEARCH_DOCKER_HUB_PROJECT=${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject} + - OPENSEARCH_DOCKER_REF=${OPENSEARCH_DOCKER_REF} + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node diff --git a/tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml b/tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml new file mode 100644 index 000000000..29e4b5e90 --- /dev/null +++ b/tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml @@ -0,0 +1,81 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test mappings endpoints. +warnings: + multiple-paths-detected: false +prologues: [] +epilogues: + - path: /collated + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create an index with a mapping that uses ICU collation. + path: /{index} + method: PUT + parameters: + index: collated + request: + payload: + mappings: + properties: + phrase: + type: icu_collation_keyword + language: tr + strength: primary + + - synopsis: Get mappings for index. + path: /{index}/_mapping + method: GET + parameters: + index: collated + response: + status: 200 + payload: + collated: + mappings: + properties: + phrase: + type: icu_collation_keyword + language: tr + strength: primary + + - synopsis: Index values with ICU collation. + path: /{index}/_bulk + method: POST + parameters: + index: collated + refresh: wait_for + request: + content_type: application/x-ndjson + payload: + - { create: { _id: '1' } } + - { phrase: 'I WİLL USE TURKİSH CASING' } + - { create: { _id: '2' } } + - { phrase: 'ı will use turkish casıng' } + + - synopsis: Search for values with ICU collation. + path: /{index}/_search + method: POST + parameters: + index: collated + request: + payload: + query: + term: + phrase: 'I WİLL USE TURKİSH CASING' + sort: + - phrase + - { _id: desc } + response: + status: 200 + payload: + hits: + total: + value: 2 + hits: + - _id: '2' + _source: + phrase: 'ı will use turkish casıng' + - _id: '1' + _source: + phrase: 'I WİLL USE TURKİSH CASING' From 7aa99de7cead7a7eb97ce5c7004322ded38f3b9c Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 14 Nov 2024 10:12:33 +1300 Subject: [PATCH 6/6] Rename test folder Signed-off-by: Thomas Farr --- .github/workflows/test-spec.yml | 2 +- .../{analysis-icu => analysis}/Dockerfile | 0 .../docker-compose.yml | 0 .../mapping/icu_collation_keyword.yaml | 21 ++++++++++++------- 4 files changed, 14 insertions(+), 9 deletions(-) rename tests/plugins/{analysis-icu => analysis}/Dockerfile (100%) rename tests/plugins/{analysis-icu => analysis}/docker-compose.yml (100%) rename tests/plugins/{analysis-icu => analysis}/indices/mapping/icu_collation_keyword.yaml (71%) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index b15a6280d..a6c1e78ec 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -45,7 +45,7 @@ jobs: - version: 2.18.0 tests: plugins/workload-management - version: 2.18.0 - tests: plugins/analysis-icu + tests: plugins/analysis - version: 2.19.0 hub: opensearchstaging ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' diff --git a/tests/plugins/analysis-icu/Dockerfile b/tests/plugins/analysis/Dockerfile similarity index 100% rename from tests/plugins/analysis-icu/Dockerfile rename to tests/plugins/analysis/Dockerfile diff --git a/tests/plugins/analysis-icu/docker-compose.yml b/tests/plugins/analysis/docker-compose.yml similarity index 100% rename from tests/plugins/analysis-icu/docker-compose.yml rename to tests/plugins/analysis/docker-compose.yml diff --git a/tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml b/tests/plugins/analysis/indices/mapping/icu_collation_keyword.yaml similarity index 71% rename from tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml rename to tests/plugins/analysis/indices/mapping/icu_collation_keyword.yaml index 29e4b5e90..1b4d78dda 100644 --- a/tests/plugins/analysis-icu/indices/mapping/icu_collation_keyword.yaml +++ b/tests/plugins/analysis/indices/mapping/icu_collation_keyword.yaml @@ -48,10 +48,12 @@ chapters: request: content_type: application/x-ndjson payload: - - { create: { _id: '1' } } - - { phrase: 'I WİLL USE TURKİSH CASING' } - - { create: { _id: '2' } } - - { phrase: 'ı will use turkish casıng' } + - {create: {_id: '1'}} + # eslint-disable-next-line @cspell/spellchecker + - {phrase: I WİLL USE TURKİSH CASING} + - {create: {_id: '2'}} + # eslint-disable-next-line @cspell/spellchecker + - {phrase: ı will use turkish casıng} - synopsis: Search for values with ICU collation. path: /{index}/_search @@ -62,10 +64,11 @@ chapters: payload: query: term: - phrase: 'I WİLL USE TURKİSH CASING' + # eslint-disable-next-line @cspell/spellchecker + phrase: I WİLL USE TURKİSH CASING sort: - phrase - - { _id: desc } + - {_id: desc} response: status: 200 payload: @@ -75,7 +78,9 @@ chapters: hits: - _id: '2' _source: - phrase: 'ı will use turkish casıng' + # eslint-disable-next-line @cspell/spellchecker + phrase: ı will use turkish casıng - _id: '1' _source: - phrase: 'I WİLL USE TURKİSH CASING' + # eslint-disable-next-line @cspell/spellchecker + phrase: I WİLL USE TURKİSH CASING