diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 03e9b9cb..a6c1e78e 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -44,6 +44,8 @@ jobs: tests: plugins/query_insights - version: 2.18.0 tests: plugins/workload-management + - version: 2.18.0 + tests: plugins/analysis - version: 2.19.0 hub: opensearchstaging ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' diff --git a/CHANGELOG.md b/CHANGELOG.md index 88698678..05f3ba6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `_search` with `sort: direction` ([#658](https://github.com/opensearch-project/opensearch-api-specification/pull/658)) - Added `_common.mapping:FlatObjectProperty` ([#661](https://github.com/opensearch-project/opensearch-api-specification/pull/661)) - Added `HEAD /{index}/_doc/{id}` returning `404` ([#670](https://github.com/opensearch-project/opensearch-api-specification/pull/670)) +- Added `_common.mapping:IcuCollationKeywordProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) +- Removed unsupported `_common.mapping:DenseVectorProperty`, `_common.mapping:SparseVectorProperty`, and `_common.mapping:FlattenedProperty` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) +- Removed unsupported time series dimension properties and types ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) +- Removed unsupported runtime field properties (`script`/`on_script_error`) on `NumberPropertyBase` ([#666](https://github.com/opensearch-project/opensearch-api-specification/pull/666)) ### Fixed - Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index fcf676ad..cae77c60 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2045,7 +2045,7 @@ components: The index template with the highest priority is chosen. If no priority is specified the template is treated as though it is of priority 0 (lowest priority). This number is not automatically generated by OpenSearch. - type: number + type: integer version: $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber' _meta: @@ -2225,7 +2225,7 @@ components: The index template with the highest priority is chosen. If no priority is specified the template is treated as though it is of priority 0 (lowest priority). This number is not automatically generated by OpenSearch. - type: number + type: integer version: $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber' _meta: diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 9e2bc2c5..12087d9e 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -109,6 +109,7 @@ components: type: object discriminator: propertyName: type + x-default: object oneOf: - $ref: '#/components/schemas/BinaryProperty' - $ref: '#/components/schemas/BooleanProperty' @@ -125,9 +126,6 @@ components: - $ref: '#/components/schemas/DateNanosProperty' - $ref: '#/components/schemas/DateProperty' - $ref: '#/components/schemas/AggregateMetricDoubleProperty' - - $ref: '#/components/schemas/DenseVectorProperty' - - $ref: '#/components/schemas/SparseVectorProperty' - - $ref: '#/components/schemas/FlattenedProperty' - $ref: '#/components/schemas/FlatObjectProperty' - $ref: '#/components/schemas/NestedProperty' - $ref: '#/components/schemas/ObjectProperty' @@ -158,6 +156,7 @@ components: - $ref: '#/components/schemas/IpRangeProperty' - $ref: '#/components/schemas/LongRangeProperty' - $ref: '#/components/schemas/KnnVectorProperty' + - $ref: '#/components/schemas/IcuCollationKeywordProperty' BinaryProperty: allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' @@ -227,19 +226,6 @@ components: - boolean required: - type - OnScriptError: - type: string - enum: - - continue - - fail - TimeSeriesMetricType: - type: string - enum: - - counter - - gauge - - histogram - - position - - summary IndexOptions: type: string enum: @@ -310,9 +296,6 @@ components: type: string split_queries_on_whitespace: type: boolean - time_series_dimension: - description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false. - type: boolean type: type: string enum: @@ -535,86 +518,10 @@ components: type: array items: type: string - time_series_metric: - $ref: '#/components/schemas/TimeSeriesMetricType' required: - default_metric - metrics - type - DenseVectorProperty: - allOf: - - $ref: '#/components/schemas/PropertyBase' - - type: object - properties: - type: - type: string - enum: - - dense_vector - dims: - type: number - similarity: - type: string - index: - type: boolean - index_options: - $ref: '#/components/schemas/DenseVectorIndexOptions' - required: - - dims - - type - DenseVectorIndexOptions: - type: object - properties: - type: - type: string - m: - type: number - ef_construction: - type: number - required: - - ef_construction - - m - - type - SparseVectorProperty: - allOf: - - $ref: '#/components/schemas/PropertyBase' - - type: object - properties: - type: - type: string - enum: - - sparse_vector - required: - - type - FlattenedProperty: - allOf: - - $ref: '#/components/schemas/PropertyBase' - - type: object - properties: - boost: - type: number - format: double - depth_limit: - type: number - doc_values: - type: boolean - eager_global_ordinals: - type: boolean - index: - type: boolean - index_options: - $ref: '#/components/schemas/IndexOptions' - null_value: - type: string - similarity: - type: string - split_queries_on_whitespace: - type: boolean - type: - type: string - enum: - - flattened - required: - - type FlatObjectProperty: allOf: - $ref: '#/components/schemas/PropertyBase' @@ -751,13 +658,6 @@ components: type: boolean null_value: type: string - on_script_error: - $ref: '#/components/schemas/OnScriptError' - script: - $ref: '_common.yaml#/components/schemas/Script' - time_series_dimension: - description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false. - type: boolean type: type: string enum: @@ -843,20 +743,22 @@ components: - type GeoOrientation: oneOf: + # eslint-disable yml/sort-sequence-values - title: left type: string enum: + - left - LEFT - clockwise - cw - - left - title: right type: string enum: + - right - RIGHT - - ccw - counterclockwise - - right + - ccw + # eslint-enable yml/sort-sequence-values GeoStrategy: type: string enum: @@ -927,15 +829,6 @@ components: type: boolean index: type: boolean - on_script_error: - $ref: '#/components/schemas/OnScriptError' - script: - $ref: '_common.yaml#/components/schemas/Script' - time_series_metric: - $ref: '#/components/schemas/TimeSeriesMetricType' - time_series_dimension: - description: For internal use by OpenSearch only. Marks the field as a time series dimension. Defaults to false. - type: boolean DoubleNumberProperty: allOf: - $ref: '#/components/schemas/NumberPropertyBase' @@ -1168,6 +1061,43 @@ components: required: - dimension - type + IcuCollationKeywordProperty: + allOf: + - $ref: '#/components/schemas/DocValuesPropertyBase' + - type: object + properties: + type: + type: string + enum: + - icu_collation_keyword + index: + type: boolean + null_value: + type: string + alternate: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationAlternate' + case_level: + type: boolean + case_first: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationCaseFirst' + decomposition: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationDecomposition' + hiragana_quaternary_mode: + type: boolean + numeric: + type: boolean + strength: + $ref: '_common.analysis.yaml#/components/schemas/IcuCollationStrength' + variable_top: + type: string + country: + type: string + language: + type: string + variant: + type: string + required: + - type MatchType: type: string enum: diff --git a/spec/schemas/_core.field_caps.yaml b/spec/schemas/_core.field_caps.yaml index 70b93d6d..67d8b308 100644 --- a/spec/schemas/_core.field_caps.yaml +++ b/spec/schemas/_core.field_caps.yaml @@ -28,25 +28,6 @@ components: metadata_field: description: Whether this field is registered as a metadata field. type: boolean - time_series_dimension: - description: Whether this field is used as a time series dimension. - type: boolean - time_series_metric: - $ref: '_common.mapping.yaml#/components/schemas/TimeSeriesMetricType' - non_dimension_indices: - description: |- - If this list is present in the response, then indexes not contained in the list have the - field marked as a dimension. Any indexes contained in the list are not marked as a dimension. - type: array - items: - $ref: '_common.yaml#/components/schemas/IndexName' - metric_conflicts_indices: - description: |- - The list of indexes in which this field is present if the indexes - don't have the same `time_series_metric` value for the field. - type: array - items: - $ref: '_common.yaml#/components/schemas/IndexName' required: - aggregatable - searchable diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index 10386d1e..6135bd36 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -148,8 +148,6 @@ components: $ref: '#/components/schemas/IndexSettingsAnalysis' settings: $ref: '#/components/schemas/IndexSettings' - time_series: - $ref: '#/components/schemas/IndexSettingsTimeSeries' queries: $ref: '#/components/schemas/IndexSettingsQueries' similarity: @@ -457,13 +455,6 @@ components: type: object additionalProperties: $ref: '_common.analysis.yaml#/components/schemas/Tokenizer' - IndexSettingsTimeSeries: - type: object - properties: - end_time: - $ref: '_common.yaml#/components/schemas/DateTime' - start_time: - $ref: '_common.yaml#/components/schemas/DateTime' IndexSettingsQueries: type: object properties: @@ -919,7 +910,8 @@ components: The index template with the highest priority is chosen. If no priority is specified the template is treated as though it is of priority 0 (lowest priority). This number is not automatically generated by OpenSearch. - type: number + type: integer + format: int64 _meta: $ref: '_common.yaml#/components/schemas/Metadata' allow_auto_create: diff --git a/tests/plugins/analysis/Dockerfile b/tests/plugins/analysis/Dockerfile new file mode 100644 index 00000000..aa43b705 --- /dev/null +++ b/tests/plugins/analysis/Dockerfile @@ -0,0 +1,6 @@ +ARG OPENSEARCH_DOCKER_HUB_PROJECT +ARG OPENSEARCH_VERSION +ARG OPENSEARCH_DOCKER_REF + +FROM ${OPENSEARCH_DOCKER_HUB_PROJECT}/opensearch:${OPENSEARCH_VERSION}${OPENSEARCH_DOCKER_REF} +RUN ./bin/opensearch-plugin install -b analysis-icu diff --git a/tests/plugins/analysis/docker-compose.yml b/tests/plugins/analysis/docker-compose.yml new file mode 100644 index 00000000..0f4c7da5 --- /dev/null +++ b/tests/plugins/analysis/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + opensearch-cluster: + build: + context: . + args: + - OPENSEARCH_DOCKER_HUB_PROJECT=${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject} + - OPENSEARCH_DOCKER_REF=${OPENSEARCH_DOCKER_REF} + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node diff --git a/tests/plugins/analysis/indices/mapping/icu_collation_keyword.yaml b/tests/plugins/analysis/indices/mapping/icu_collation_keyword.yaml new file mode 100644 index 00000000..1b4d78dd --- /dev/null +++ b/tests/plugins/analysis/indices/mapping/icu_collation_keyword.yaml @@ -0,0 +1,86 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test mappings endpoints. +warnings: + multiple-paths-detected: false +prologues: [] +epilogues: + - path: /collated + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create an index with a mapping that uses ICU collation. + path: /{index} + method: PUT + parameters: + index: collated + request: + payload: + mappings: + properties: + phrase: + type: icu_collation_keyword + language: tr + strength: primary + + - synopsis: Get mappings for index. + path: /{index}/_mapping + method: GET + parameters: + index: collated + response: + status: 200 + payload: + collated: + mappings: + properties: + phrase: + type: icu_collation_keyword + language: tr + strength: primary + + - synopsis: Index values with ICU collation. + path: /{index}/_bulk + method: POST + parameters: + index: collated + refresh: wait_for + request: + content_type: application/x-ndjson + payload: + - {create: {_id: '1'}} + # eslint-disable-next-line @cspell/spellchecker + - {phrase: I WİLL USE TURKİSH CASING} + - {create: {_id: '2'}} + # eslint-disable-next-line @cspell/spellchecker + - {phrase: ı will use turkish casıng} + + - synopsis: Search for values with ICU collation. + path: /{index}/_search + method: POST + parameters: + index: collated + request: + payload: + query: + term: + # eslint-disable-next-line @cspell/spellchecker + phrase: I WİLL USE TURKİSH CASING + sort: + - phrase + - {_id: desc} + response: + status: 200 + payload: + hits: + total: + value: 2 + hits: + - _id: '2' + _source: + # eslint-disable-next-line @cspell/spellchecker + phrase: ı will use turkish casıng + - _id: '1' + _source: + # eslint-disable-next-line @cspell/spellchecker + phrase: I WİLL USE TURKİSH CASING