From 106fd001f3409a9edb54be243c63924a31c5f88e Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sat, 3 Aug 2024 22:24:39 +0500 Subject: [PATCH 01/73] replication added Signed-off-by: Tokesh --- spec/namespaces/replication.yaml | 261 ++++++++++++++++++++++++++ spec/schemas/replication._common.yaml | 207 ++++++++++++++++++++ tests/replication/replica.yaml | 39 ++++ 3 files changed, 507 insertions(+) create mode 100644 spec/namespaces/replication.yaml create mode 100644 spec/schemas/replication._common.yaml create mode 100644 tests/replication/replica.yaml diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml new file mode 100644 index 000000000..200dd6356 --- /dev/null +++ b/spec/namespaces/replication.yaml @@ -0,0 +1,261 @@ +openapi: 3.1.0 +info: + title: OpenSearch Replication API + description: OpenSearch Replication API + version: 1.0.0 +paths: + /_plugins/_replication/{index}/_start: + put: + operationId: replication.start.0 + x-operation-group: replication.start + x-version-added: '1.1' + description: Initiate replication of an index from the leader cluster to the follower cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication + requestBody: + $ref: '#/components/requestBodies/replication.start' + responses: + '200': + $ref: '#/components/responses/replication.start@200' + /_plugins/_replication/{index}/_stop: + post: + operationId: replication.stop.0 + x-operation-group: replication.stop + x-version-added: '1.1' + description: Terminates replication and converts the follower index to a standard index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication + requestBody: + $ref: '#/components/requestBodies/replication.stop' + responses: + '200': + $ref: '#/components/responses/replication.stop@200' + /_plugins/_replication/{index}/_pause: + post: + operationId: replication.pause.0 + x-operation-group: replication.pause + x-version-added: '1.1' + description: Pauses replication of the leader index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication + requestBody: + $ref: '#/components/requestBodies/replication.pause' + responses: + '200': + $ref: '#/components/responses/replication.pause@200' + /_plugins/_replication/{index}/_resume: + post: + operationId: replication.resume.0 + x-operation-group: replication.resume + x-version-added: '1.1' + description: Resumes replication of the leader index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication + requestBody: + $ref: '#/components/requestBodies/replication.resume' + responses: + '200': + $ref: '#/components/responses/replication.resume@200' + /_plugins/_replication/{index}/_status: + get: + operationId: replication.status.0 + x-operation-group: replication.status + x-version-added: '1.1' + description: Gets the status of index replication. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status + responses: + '200': + $ref: '#/components/responses/replication.status@200' + /_plugins/_replication/leader_stats: + get: + operationId: replication.leader_stats.0 + x-operation-group: replication.leader_stats + x-version-added: '1.1' + description: Gets information about replicated leader indexes on a specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-leader-cluster-stats + responses: + '200': + $ref: '#/components/responses/replication.leader_stats@200' + /_plugins/_replication/follower_stats: + get: + operationId: replication.follower_stats.0 + x-operation-group: replication.follower_stats + x-version-added: '1.1' + description: Gets information about follower (syncing) indexes on a specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-follower-cluster-stats + responses: + '200': + $ref: '#/components/responses/replication.follower_stats@200' + /_plugins/_replication/autofollow_stats: + get: + operationId: replication.autofollow_stats.0 + x-operation-group: replication.autofollow_stats + x-version-added: '1.1' + description: Gets information about auto-follow activity and any replication rules configured on the specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-auto-follow-stats + responses: + '200': + $ref: '#/components/responses/replication.autofollow_stats@200' + /_plugins/_replication/{follower-index}/_update: + put: + operationId: replication.update_settings.0 + x-operation-group: replication.update_settings + x-version-added: '1.1' + description: Updates settings on the follower index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings + requestBody: + $ref: '#/components/requestBodies/replication.update_settings' + responses: + '200': + $ref: '#/components/responses/replication.update_settings@200' + /_plugins/_replication/_autofollow: + post: + operationId: replication.replication_rule.0 + x-operation-group: replication.replication_rule + x-version-added: '1.1' + description: Automatically starts replication on indexes matching a specified pattern. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#create-replication-rule + requestBody: + $ref: '#/components/requestBodies/replication.create_replication_rule' + responses: + '200': + $ref: '#/components/responses/replication.create_replication_rule@200' + delete: + operationId: replication.delete_replication_rule.0 + x-operation-group: replication.delete_replication_rule + x-version-added: '1.1' + description: Deletes the specified replication rule. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#delete-replication-rule + requestBody: + $ref: '#/components/requestBodies/replication.delete_replication_rule' + responses: + '200': + $ref: '#/components/responses/replication.delete_replication_rule@200' +components: + requestBodies: + replication.start: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/Replication' + required: true + replication.stop: + content: + application/json: + schema: + type: Object + required: true + replication.pause: + content: + application/json: + schema: + type: Object + required: true + replication.resume: + content: + application/json: + schema: + type: Object + required: true + replication.update_settings: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/UpdateSettings' + required: true + replication.create_replication_rule: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/CreateReplicationRule' + required: true + replication.delete_replication_rule: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/DeleteReplicationRule' + required: true + responses: + replication.start@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.stop@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.pause@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.resume@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.status@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/Status' + replication.leader_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/LeaderStatus' + replication.follower_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/FollowerStatus' + replication.autofollow_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/AutoFollowStatus' + replication.update_settings@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.create_replication_rule@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.delete_replication_rule@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean \ No newline at end of file diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml new file mode 100644 index 000000000..94febb860 --- /dev/null +++ b/spec/schemas/replication._common.yaml @@ -0,0 +1,207 @@ +openapi: 3.1.0 +info: + title: Schemas of replication._common category. + description: Schemas of replication._common category. + version: 1.0.0 +paths: {} +components: + schemas: + Replication: + type: object + properties: + leader_alias: + type: string + leader_index: + type: string + use_roles: + $ref: '#/components/schemas/UseRoles' + UseRoles: + type: object + properties: + leader_cluster_role: + type: string + follower_cluster_role: + type: string + Status: + type: object + properties: + status: + type: string + reason: + type: string + leader_alias: + type: string + leader_index: + type: string + follower_index: + type: string + syncing_details: + $ref: '#/components/schemas/SyncingDetails' + SyncingDetails: + type: object + properties: + leader_checkpoint: + type: integer + follower_checkpoint: + type: integer + seq_no: + type: integer + LeaderStatus: + type: object + properties: + num_replicated_indices: + type: number + operations_read: + type: number + translog_size_bytes: + type: number + operations_read_lucene: + type: number + operations_read_translog: + type: number + total_read_time_lucene_millis: + type: number + total_read_time_translog_millis: + type: number + bytes_read: + type: number + # index_stats: + # type: Object + # additionalProperties: + # type: object + # properties: + # $ref: '#/components/schemas/IndexStatus' + IndexStatus: + type: object + properties: + operations_read: + type: number + translog_size_bytes: + type: number + operations_read_lucene: + type: number + operations_read_translog: + type: number + total_read_time_lucene_millis: + type: number + total_read_time_translog_millis: + type: number + bytes_read: + type: number + FollowerStatus: + type: object + properties: + num_syncing_indices: + type: number + num_bootstrapping_indices: + type: number + num_paused_indices: + type: number + num_failed_indices: + type: number + num_shard_tasks: + type: number + num_index_tasks: + type: number + operations_written: + type: number + operations_read: + type: number + failed_read_requests: + type: number + throttled_read_requests: + type: number + failed_write_requests: + type: number + throttled_write_requests: + type: number + follower_checkpoint: + type: number + leader_checkpoint: + type: number + total_write_time_millis: + type: number + # index_stats: + # type: Object + # additionalProperties: + # type: object + # properties: + # $ref: '#/components/schemas/IndexFollowerStatus' + IndexFollowerStatus: + type: object + properties: + operations_written: + type: number + operations_read: + type: number + failed_read_requests: + type: number + throttled_read_requests: + type: number + failed_write_requests: + type: number + throttled_write_requests: + type: number + follower_checkpoint: + type: number + leader_checkpoint: + type: number + total_write_time_millis: + type: number + AutoFollowStatus: + type: object + properties: + num_success_start_replication: + type: number + num_failed_start_replication: + type: number + num_failed_leader_calls: + type: number + failed_indices: + type: array + items: + type: string + autofollow_stats: + type: array + items: + $ref: '#/components/schemas/AutoFollowStats' + AutoFollowStats: + type: object + properties: + name: + type: string # Предполагается, что это строка + pattern: + type: string # Предполагается, что это строка + num_success_start_replication: + type: number + num_failed_start_replication: + type: number + num_failed_leader_calls: + type: number + failed_indices: + type: array + items: + type: string + UpdateSettings: + type: object + properties: + settings: + type: string + CreateReplicationRule: + type: object + properties: + leader_alias: + type: string + name: + type: string + pattern: + type: string + use_roles: + $ref: '#/components/schemas/UseRoles' + DeleteReplicationRule: + type: object + properties: + leader_alias: + type: string + name: + type: string \ No newline at end of file diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml new file mode 100644 index 000000000..54cfde91e --- /dev/null +++ b/tests/replication/replica.yaml @@ -0,0 +1,39 @@ +$schema: ../../json_schemas/test_story.schema.yaml + +description: Test to explicitly clear the cursor context + +# prologues: + # - path: /{index} + # method: PUT + # parameters: + # index: books + # request_body: + # payload: {} + # - path: /_bulk + # method: POST + # request_body: + # content_type: application/x-ndjson + # payload: + # - {create: {_index: books, _id: book_1392214}} + # - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + # - {create: {_index: books, _id: book_1392215}} + # - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + # - path: /{index}/_refresh + # method: POST + # parameters: + # index: books +# epilogues: + # - path: /books + # method: DELETE + # status: [200, 404] +chapters: + - synopsis: Cat with a json response. + path: /_cat/nodes + version: '>= 2.0' + method: GET + parameters: + format: json + response: + status: 200 + payload: + - cluster_manager: '*' \ No newline at end of file From e912e13792b16d42e4b60fecbe0f36936eabfd9c Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sat, 3 Aug 2024 22:30:38 +0500 Subject: [PATCH 02/73] fixing lint and trying test verbose Signed-off-by: Tokesh --- .cspell | 1 + .github/workflows/test-spec.yml | 2 +- spec/schemas/replication._common.yaml | 12 ++++---- tests/replication/replica.yaml | 44 +++++++++++++-------------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.cspell b/.cspell index 80670e9b4..fff1b5cef 100644 --- a/.cspell +++ b/.cspell @@ -9,6 +9,7 @@ authc authinfo authtoken autocut +autofollow backendroles backpressure beider diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index c147cbf40..01388d80e 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -56,7 +56,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure \ + --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 94febb860..11dfc175b 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -67,10 +67,10 @@ components: type: number # index_stats: # type: Object - # additionalProperties: - # type: object - # properties: - # $ref: '#/components/schemas/IndexStatus' + # additionalProperties: + # type: object + # properties: + # $ref: '#/components/schemas/IndexStatus' IndexStatus: type: object properties: @@ -169,9 +169,9 @@ components: type: object properties: name: - type: string # Предполагается, что это строка + type: string pattern: - type: string # Предполагается, что это строка + type: string num_success_start_replication: type: number num_failed_start_replication: diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 54cfde91e..6a580f21b 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -3,29 +3,29 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context # prologues: - # - path: /{index} - # method: PUT - # parameters: - # index: books - # request_body: - # payload: {} - # - path: /_bulk - # method: POST - # request_body: - # content_type: application/x-ndjson - # payload: - # - {create: {_index: books, _id: book_1392214}} - # - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - # - {create: {_index: books, _id: book_1392215}} - # - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} - # - path: /{index}/_refresh - # method: POST - # parameters: - # index: books +# - path: /{index} +# method: PUT +# parameters: +# index: books +# request_body: +# payload: {} +# - path: /_bulk +# method: POST +# request_body: +# content_type: application/x-ndjson +# payload: +# - {create: {_index: books, _id: book_1392214}} +# - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} +# - {create: {_index: books, _id: book_1392215}} +# - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} +# - path: /{index}/_refresh +# method: POST +# parameters: +# index: books # epilogues: - # - path: /books - # method: DELETE - # status: [200, 404] +# - path: /books +# method: DELETE +# status: [200, 404] chapters: - synopsis: Cat with a json response. path: /_cat/nodes From bd877a8211b359a6ec143bcaebd70a67882a42ae Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 00:16:47 +0500 Subject: [PATCH 03/73] changing docker compose Signed-off-by: Tokesh --- .github/opensearch-cluster/docker-compose.yml | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index bdd6f2be9..3828f452f 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -1,13 +1,57 @@ version: '3' services: - opensearch-cluster: + opensearch-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - path.repo=/tmp/opensearch/repo + - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data1:/usr/share/opensearch/data ports: - - '9200:9200' - - '9600:9600' + - "9200:9200" + - "9600:9600" + networks: + - opensearch-net + + opensearch-node2: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node2 environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' - - discovery.type=single-node - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data2:/usr/share/opensearch/data + networks: + - opensearch-net + +volumes: + data1: + data2: + +networks: + opensearch-net: + driver: bridge From addcd769cdceeb7908d8b32b68c48d55dfcc43c3 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 01:43:52 +0500 Subject: [PATCH 04/73] adding tests for replication namespace Signed-off-by: Tokesh --- CHANGELOG.md | 1 + spec/namespaces/replication.yaml | 65 +++++++++++- spec/schemas/replication._common.yaml | 39 ++++--- tests/replication/replica.yaml | 147 +++++++++++++++++++++----- 4 files changed, 208 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e24e482b8..0394299b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added a warning on mulitple paths being tested in the same file ([#452](https://github.com/opensearch-project/opensearch-api-specification/pull/452)) - Added `/_plugins/_query/settings` ([#456](https://github.com/opensearch-project/opensearch-api-specification/pull/456)) - Added `/_plugins/_ppl`, `explain` and `stats` ([#460](https://github.com/opensearch-project/opensearch-api-specification/pull/460)) +- Added `replication` namespace specs ([464](https://github.com/opensearch-project/opensearch-api-specification/pull/464)) ### Changed diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index 200dd6356..5347131f9 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -12,6 +12,8 @@ paths: description: Initiate replication of an index from the leader cluster to the follower cluster. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication + parameters: + - $ref: '#/components/parameters/start::path.index' requestBody: $ref: '#/components/requestBodies/replication.start' responses: @@ -25,6 +27,8 @@ paths: description: Terminates replication and converts the follower index to a standard index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication + parameters: + - $ref: '#/components/parameters/stop::path.index' requestBody: $ref: '#/components/requestBodies/replication.stop' responses: @@ -38,6 +42,8 @@ paths: description: Pauses replication of the leader index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication + parameters: + - $ref: '#/components/parameters/pause::path.index' requestBody: $ref: '#/components/requestBodies/replication.pause' responses: @@ -51,6 +57,8 @@ paths: description: Resumes replication of the leader index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication + parameters: + - $ref: '#/components/parameters/resume::path.index' requestBody: $ref: '#/components/requestBodies/replication.resume' responses: @@ -64,6 +72,8 @@ paths: description: Gets the status of index replication. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status + parameters: + - $ref: '#/components/parameters/status::path.index' responses: '200': $ref: '#/components/responses/replication.status@200' @@ -100,7 +110,7 @@ paths: responses: '200': $ref: '#/components/responses/replication.autofollow_stats@200' - /_plugins/_replication/{follower-index}/_update: + /_plugins/_replication/{index}/_update: put: operationId: replication.update_settings.0 x-operation-group: replication.update_settings @@ -108,6 +118,8 @@ paths: description: Updates settings on the follower index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings + parameters: + - $ref: '#/components/parameters/update_settings::path.index' requestBody: $ref: '#/components/requestBodies/replication.update_settings' responses: @@ -139,6 +151,55 @@ paths: '200': $ref: '#/components/responses/replication.delete_replication_rule@200' components: + parameters: + start::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + stop::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + pause::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + resume::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + status::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + update_settings::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple requestBodies: replication.start: content: @@ -150,7 +211,7 @@ components: content: application/json: schema: - type: Object + type: object required: true replication.pause: content: diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 11dfc175b..6eea148bc 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -65,12 +65,10 @@ components: type: number bytes_read: type: number - # index_stats: - # type: Object - # additionalProperties: - # type: object - # properties: - # $ref: '#/components/schemas/IndexStatus' + index_stats: + type: object + additionalProperties: + $ref: '#/components/schemas/IndexStatus' IndexStatus: type: object properties: @@ -121,12 +119,10 @@ components: type: number total_write_time_millis: type: number - # index_stats: - # type: Object - # additionalProperties: - # type: object - # properties: - # $ref: '#/components/schemas/IndexFollowerStatus' + index_stats: + type: object + additionalProperties: + $ref: '#/components/schemas/IndexFollowerStatus' IndexFollowerStatus: type: object properties: @@ -186,7 +182,24 @@ components: type: object properties: settings: - type: string + anyOf: + - type: object + properties: + index.number_of_shards: + type: integer + index.number_of_replicas: + type: integer + - $ref: '#/components/schemas/SettingsBody' + SettingsBody: + type: object + properties: + index: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer CreateReplicationRule: type: object properties: diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 6a580f21b..afcd07422 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -2,38 +2,127 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context -# prologues: -# - path: /{index} -# method: PUT -# parameters: -# index: books -# request_body: -# payload: {} -# - path: /_bulk -# method: POST -# request_body: -# content_type: application/x-ndjson -# payload: -# - {create: {_index: books, _id: book_1392214}} -# - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} -# - {create: {_index: books, _id: book_1392215}} -# - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} -# - path: /{index}/_refresh -# method: POST -# parameters: -# index: books -# epilogues: -# - path: /books -# method: DELETE -# status: [200, 404] +prologues: +- path: /{index} + method: PUT + parameters: + index: books + request_body: + payload: {} +- path: /_bulk + method: POST + request_body: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} +- path: /{index}/_refresh + method: POST + parameters: + index: books +epilogues: +- path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request_body: + payload: {} +- path: /books-names + method: DELETE + status: [200, 404] +- path: /books + method: DELETE + status: [200, 404] chapters: - - synopsis: Cat with a json response. - path: /_cat/nodes - version: '>= 2.0' + - synopsis: Establish connection. + path: /_cluster/settings + method: PUT + request_body: + payload: + persistent: + cluster: + remote: + my-connection-alias: + seeds: ["172.18.0.3:9300"] + response: + status: 200 + - synopsis: Start replication. + path: /_plugins/_replication/{index}/_start + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request_body: + payload: + leader_alias: 'my-connection-alias' + leader_index: 'books' + use_roles: + leader_cluster_role: 'leader_role' + follower_cluster_role: 'follower_role' + response: + status: 200 + - synopsis: Get replication stats. + path: /_plugins/_replication/{index}/_status + version: '>= 1.1' + method: GET + parameters: + index: books-names + response: + status: 200 + - synopsis: Get leader stats. + path: /_plugins/_replication/leader_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get follower stats. + path: /_plugins/_replication/follower_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get auto-follow stats. + path: /_plugins/_replication/autofollow_stats + version: '>= 1.1' method: GET + response: + status: 200 + - synopsis: Update settings. + path: /_plugins/_replication/{index}/_update + version: '>= 1.1' + method: PUT parameters: - format: json + index: books-names + request_body: + payload: + settings: + index.number_of_shards: 4 + index.number_of_replicas: 2 + response: + status: 200 + - synopsis: Create replication rule. + path: /_plugins/_replication/_autofollow + version: '>= 1.1' + method: POST + request_body: + payload: + leader_alias: "my-connection-alias" + name: "books" + pattern: "books-*" + use_roles: + leader_cluster_role: 'leader_role' + follower_cluster_role: 'follower_role' response: status: 200 + - synopsis: Delete replication rule. + path: /_plugins/_replication/_autofollow + version: '>= 1.1' + method: DELETE + request_body: payload: - - cluster_manager: '*' \ No newline at end of file + leader_alias: "my-connection-alias" + name: "books" + response: + status: 200 \ No newline at end of file From 638d60e4978601537df10638e74bf5a72379bc0e Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 01:50:44 +0500 Subject: [PATCH 05/73] fixing lint, validate specs ci Signed-off-by: Tokesh --- .cspell | 2 + .github/opensearch-cluster/docker-compose.yml | 28 +++--- spec/namespaces/replication.yaml | 24 +++--- tests/replication/replica.yaml | 86 +++++++++---------- 4 files changed, 69 insertions(+), 71 deletions(-) diff --git a/.cspell b/.cspell index fff1b5cef..bc33abec4 100644 --- a/.cspell +++ b/.cspell @@ -79,6 +79,7 @@ lucene Lucene lycheeverse marvinpinto +memlock metaphone mget millis @@ -174,6 +175,7 @@ tokenfilters translog Translog tubone +ulimits unigrams Unmanaged unmatch diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index 3828f452f..cdf77a035 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -5,13 +5,13 @@ services: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} container_name: opensearch-node1 environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - cluster.name=opensearch-cluster - - node.name=opensearch-node1 - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - node.name=opensearch-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 ulimits: @@ -21,8 +21,8 @@ services: volumes: - data1:/usr/share/opensearch/data ports: - - "9200:9200" - - "9600:9600" + - '9200:9200' + - '9600:9600' networks: - opensearch-net @@ -30,13 +30,13 @@ services: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} container_name: opensearch-node2 environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - cluster.name=opensearch-cluster - - node.name=opensearch-node2 - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - node.name=opensearch-node2 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 ulimits: @@ -48,10 +48,6 @@ services: networks: - opensearch-net -volumes: - data1: - data2: - networks: opensearch-net: driver: bridge diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index 5347131f9..c08d4092e 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -13,7 +13,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication parameters: - - $ref: '#/components/parameters/start::path.index' + - $ref: '#/components/parameters/replication.start::path.index' requestBody: $ref: '#/components/requestBodies/replication.start' responses: @@ -28,7 +28,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication parameters: - - $ref: '#/components/parameters/stop::path.index' + - $ref: '#/components/parameters/replication.stop::path.index' requestBody: $ref: '#/components/requestBodies/replication.stop' responses: @@ -43,7 +43,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication parameters: - - $ref: '#/components/parameters/pause::path.index' + - $ref: '#/components/parameters/replication.pause::path.index' requestBody: $ref: '#/components/requestBodies/replication.pause' responses: @@ -58,7 +58,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication parameters: - - $ref: '#/components/parameters/resume::path.index' + - $ref: '#/components/parameters/replication.resume::path.index' requestBody: $ref: '#/components/requestBodies/replication.resume' responses: @@ -73,7 +73,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status parameters: - - $ref: '#/components/parameters/status::path.index' + - $ref: '#/components/parameters/replication.status::path.index' responses: '200': $ref: '#/components/responses/replication.status@200' @@ -119,7 +119,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings parameters: - - $ref: '#/components/parameters/update_settings::path.index' + - $ref: '#/components/parameters/replication.update_settings::path.index' requestBody: $ref: '#/components/requestBodies/replication.update_settings' responses: @@ -152,7 +152,7 @@ paths: $ref: '#/components/responses/replication.delete_replication_rule@200' components: parameters: - start::path.index: + replication.start::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -160,7 +160,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - stop::path.index: + replication.stop::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -168,7 +168,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - pause::path.index: + replication.pause::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -176,7 +176,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - resume::path.index: + replication.resume::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -184,7 +184,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - status::path.index: + replication.status::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -192,7 +192,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - update_settings::path.index: + replication.update_settings::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index afcd07422..c32e141f4 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -3,38 +3,38 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context prologues: -- path: /{index} - method: PUT - parameters: - index: books - request_body: - payload: {} -- path: /_bulk - method: POST - request_body: - content_type: application/x-ndjson - payload: - - {create: {_index: books, _id: book_1392214}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - - {create: {_index: books, _id: book_1392215}} - - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} -- path: /{index}/_refresh - method: POST - parameters: - index: books + - path: /{index} + method: PUT + parameters: + index: books + request_body: + payload: {} + - path: /_bulk + method: POST + request_body: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + - path: /{index}/_refresh + method: POST + parameters: + index: books epilogues: -- path: /_plugins/_replication/{index}/_stop - method: POST - parameters: - index: books-names - request_body: - payload: {} -- path: /books-names - method: DELETE - status: [200, 404] -- path: /books - method: DELETE - status: [200, 404] + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request_body: + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] chapters: - synopsis: Establish connection. path: /_cluster/settings @@ -45,7 +45,7 @@ chapters: cluster: remote: my-connection-alias: - seeds: ["172.18.0.3:9300"] + seeds: ['172.18.0.3:9300'] response: status: 200 - synopsis: Start replication. @@ -56,11 +56,11 @@ chapters: index: books-names request_body: payload: - leader_alias: 'my-connection-alias' - leader_index: 'books' + leader_alias: my-connection-alias + leader_index: books use_roles: - leader_cluster_role: 'leader_role' - follower_cluster_role: 'follower_role' + leader_cluster_role: leader_role + follower_cluster_role: follower_role response: status: 200 - synopsis: Get replication stats. @@ -108,12 +108,12 @@ chapters: method: POST request_body: payload: - leader_alias: "my-connection-alias" - name: "books" - pattern: "books-*" + leader_alias: my-connection-alias + name: books + pattern: 'books-*' use_roles: - leader_cluster_role: 'leader_role' - follower_cluster_role: 'follower_role' + leader_cluster_role: leader_role + follower_cluster_role: follower_role response: status: 200 - synopsis: Delete replication rule. @@ -122,7 +122,7 @@ chapters: method: DELETE request_body: payload: - leader_alias: "my-connection-alias" - name: "books" + leader_alias: my-connection-alias + name: books response: status: 200 \ No newline at end of file From a6e0a87da98b4eef04c195361d6d4e7dea2af5fd Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 01:55:13 +0500 Subject: [PATCH 06/73] fixing docker-compose and validate specs ci Signed-off-by: Tokesh --- .github/opensearch-cluster/docker-compose.yml | 4 ---- spec/namespaces/replication.yaml | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index cdf77a035..bfb05f01f 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -18,8 +18,6 @@ services: memlock: soft: -1 hard: -1 - volumes: - - data1:/usr/share/opensearch/data ports: - '9200:9200' - '9600:9600' @@ -43,8 +41,6 @@ services: memlock: soft: -1 hard: -1 - volumes: - - data2:/usr/share/opensearch/data networks: - opensearch-net diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index c08d4092e..b4f81902e 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -127,8 +127,8 @@ paths: $ref: '#/components/responses/replication.update_settings@200' /_plugins/_replication/_autofollow: post: - operationId: replication.replication_rule.0 - x-operation-group: replication.replication_rule + operationId: replication.create_replication_rule.0 + x-operation-group: replication.create_replication_rule x-version-added: '1.1' description: Automatically starts replication on indexes matching a specified pattern. externalDocs: From b012e7b7eed02e31e34e1bb90530b22a337083d5 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 02:03:35 +0500 Subject: [PATCH 07/73] fixing validate specs ci Signed-off-by: Tokesh --- spec/namespaces/replication.yaml | 4 ++-- spec/schemas/replication._common.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index b4f81902e..5bb133021 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -217,13 +217,13 @@ components: content: application/json: schema: - type: Object + type: object required: true replication.resume: content: application/json: schema: - type: Object + type: object required: true replication.update_settings: content: diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 6eea148bc..d33ff2b0d 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -183,13 +183,13 @@ components: properties: settings: anyOf: + - $ref: '#/components/schemas/SettingsBody' - type: object properties: index.number_of_shards: type: integer index.number_of_replicas: type: integer - - $ref: '#/components/schemas/SettingsBody' SettingsBody: type: object properties: From 236fea70ebb30d5b200d78f560509085428a0d66 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 02:06:25 +0500 Subject: [PATCH 08/73] trying to fix validate specs Signed-off-by: Tokesh --- spec/schemas/replication._common.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index d33ff2b0d..be864a967 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -178,28 +178,28 @@ components: type: array items: type: string + SettingsBody: + type: object + properties: + index: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer UpdateSettings: type: object properties: settings: anyOf: - - $ref: '#/components/schemas/SettingsBody' - type: object properties: index.number_of_shards: type: integer index.number_of_replicas: type: integer - SettingsBody: - type: object - properties: - index: - type: object - properties: - number_of_shards: - type: integer - number_of_replicas: - type: integer + - $ref: '#/components/schemas/SettingsBody' CreateReplicationRule: type: object properties: From 864ee68e743e209038ffd1062a1e0d9131e919f8 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 02:08:36 +0500 Subject: [PATCH 09/73] trying to fix validate specs v2 Signed-off-by: Tokesh --- spec/schemas/replication._common.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index be864a967..bdd11c3f3 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -182,12 +182,7 @@ components: type: object properties: index: - type: object - properties: - number_of_shards: - type: integer - number_of_replicas: - type: integer + $ref: '#/components/schemas/IndexSchema' UpdateSettings: type: object properties: @@ -200,6 +195,13 @@ components: index.number_of_replicas: type: integer - $ref: '#/components/schemas/SettingsBody' + IndexSchema: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer CreateReplicationRule: type: object properties: From cacd32e4baadb1fb1539e58503437b7f08fcbbe9 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Wed, 14 Aug 2024 23:34:11 +0500 Subject: [PATCH 10/73] fixing tests which using cluster of OS Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 52 ++++++++++++++++++++++++++++ tests/replication/replica.yaml | 19 +++++----- tests/snapshot/docker-compose.yml | 20 ++--------- 3 files changed, 63 insertions(+), 28 deletions(-) create mode 100644 tests/replication/docker-compose.yml diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml new file mode 100644 index 000000000..a5d803793 --- /dev/null +++ b/tests/replication/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3' + +services: + opensearch-node1: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node1 + environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - cluster.name=opensearch-cluster + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - node.name=opensearch-node1 + - path.repo=/tmp/opensearch/repo + - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data1:/usr/share/opensearch/data + ports: + - '9200:9200' + - '9600:9600' + networks: + - opensearch-net + + opensearch-node2: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node2 + environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - cluster.name=opensearch-cluster + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - node.name=opensearch-node2 + - path.repo=/tmp/opensearch/repo + - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data2:/usr/share/opensearch/data + networks: + - opensearch-net +networks: + opensearch-net: + driver: bridge \ No newline at end of file diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index c32e141f4..34026e705 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -1,17 +1,16 @@ $schema: ../../json_schemas/test_story.schema.yaml -description: Test to explicitly clear the cursor context - +description: Test to explicitly clear the cursor context. prologues: - path: /{index} method: PUT parameters: index: books - request_body: + request: payload: {} - path: /_bulk method: POST - request_body: + request: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book_1392214}} @@ -27,7 +26,7 @@ epilogues: method: POST parameters: index: books-names - request_body: + request: payload: {} - path: /books-names method: DELETE @@ -39,7 +38,7 @@ chapters: - synopsis: Establish connection. path: /_cluster/settings method: PUT - request_body: + request: payload: persistent: cluster: @@ -54,7 +53,7 @@ chapters: method: PUT parameters: index: books-names - request_body: + request: payload: leader_alias: my-connection-alias leader_index: books @@ -95,7 +94,7 @@ chapters: method: PUT parameters: index: books-names - request_body: + request: payload: settings: index.number_of_shards: 4 @@ -106,7 +105,7 @@ chapters: path: /_plugins/_replication/_autofollow version: '>= 1.1' method: POST - request_body: + request: payload: leader_alias: my-connection-alias name: books @@ -120,7 +119,7 @@ chapters: path: /_plugins/_replication/_autofollow version: '>= 1.1' method: DELETE - request_body: + request: payload: leader_alias: my-connection-alias name: books diff --git a/tests/snapshot/docker-compose.yml b/tests/snapshot/docker-compose.yml index f908c89e5..b084ec3ed 100644 --- a/tests/snapshot/docker-compose.yml +++ b/tests/snapshot/docker-compose.yml @@ -1,23 +1,8 @@ version: '3' services: - opensearch-node1: + opensearch-cluster: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} - container_name: opensearch-node1 - environment: - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' - - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - - bootstrap.memory_lock=true - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - cluster.name=opensearch-cluster - - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - node.name=opensearch-node1 - - path.repo=/tmp/opensearch/repo - - plugins.index_state_management.job_interval=1 - ulimits: - memlock: - soft: -1 - hard: -1 ports: - 9200:9200 - 9600:9600 @@ -25,5 +10,4 @@ services: - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} - discovery.type=single-node - - path.repo=/tmp/opensearch/repo - + - path.repo=/tmp/opensearch/repo \ No newline at end of file From 2548bbfa046feb03caf7a8b59df0d9ff0e2b861f Mon Sep 17 00:00:00 2001 From: Tokesh Date: Wed, 14 Aug 2024 23:42:10 +0500 Subject: [PATCH 11/73] add changelog changes, fixing ci issues Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 4 +++- CHANGELOG.md | 1 + spec/namespaces/replication.yaml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index a2175f11e..2a9997cd3 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -32,6 +32,8 @@ jobs: tests: plugins/index_state_management - version: 2.16.0 tests: snapshot + - version: 2.16.0 + tests: replication - version: 2.17.0 hub: opensearchstaging ref: '@sha256:ed4274522a50228f41b50f1a7ea86e6b52fa6737072fc151b2624d22aff80d56' @@ -73,7 +75,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure --verbose\ + --opensearch-insecure \ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ --tests=tests/${{ matrix.entry.tests || 'default' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index b8fd3cefc..56adee041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for request headers in tests [#461](https://github.com/opensearch-project/opensearch-api-specification/pull/461) - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) +- Added `replication` namespace API specifications ([#464](https://github.com/opensearch-project/opensearch-api-specification/pull/464)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) - Added `observability` namespace API specifications ([#474](https://github.com/opensearch-project/opensearch-api-specification/pull/474)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index 5bb133021..f10cb9b14 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Replication API - description: OpenSearch Replication API + description: OpenSearch Replication API. version: 1.0.0 paths: /_plugins/_replication/{index}/_start: From eaab2608031aa5f3d34460b1d206ab9ad43fe88c Mon Sep 17 00:00:00 2001 From: Tokesh Date: Wed, 14 Aug 2024 23:46:50 +0500 Subject: [PATCH 12/73] fixing empty volumes in docker-compose Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index a5d803793..2a7a64708 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -18,8 +18,6 @@ services: memlock: soft: -1 hard: -1 - volumes: - - data1:/usr/share/opensearch/data ports: - '9200:9200' - '9600:9600' @@ -43,8 +41,6 @@ services: memlock: soft: -1 hard: -1 - volumes: - - data2:/usr/share/opensearch/data networks: - opensearch-net networks: From 1fbca08112d32bd8fa4adcec1ce14bfe71235ed0 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Wed, 14 Aug 2024 23:55:31 +0500 Subject: [PATCH 13/73] trying to fix docker compose Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index 2a7a64708..eb82f213d 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -3,7 +3,6 @@ version: '3' services: opensearch-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} - container_name: opensearch-node1 environment: - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m @@ -26,7 +25,6 @@ services: opensearch-node2: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} - container_name: opensearch-node2 environment: - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m From 833d3fd98b24bc116f297a27cf62959b4f64e55f Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:02:46 +0500 Subject: [PATCH 14/73] adding supporting cluster containers Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 2a9997cd3..55b24e50d 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -70,7 +70,8 @@ jobs: - name: Get Container ID id: container run: | - echo "CONTAINER_ID=$(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}")" >> $GITHUB_OUTPUT + CONTAINER_ID=$(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}" | head -n 1) + echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_OUTPUT - name: Run Tests run: | From b861f514b86b0fd461cc955e414eef15fc2428b9 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:26:01 +0500 Subject: [PATCH 15/73] configuring docker compose Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 11 ++++++++--- tests/replication/replica.yaml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index eb82f213d..5257f82e3 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -21,7 +21,8 @@ services: - '9200:9200' - '9600:9600' networks: - - opensearch-net + opensearch-net: + ipv4_address: 172.22.0.3 opensearch-node2: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} @@ -40,7 +41,11 @@ services: soft: -1 hard: -1 networks: - - opensearch-net + opensearch-net: + ipv4_address: 172.22.0.4 networks: opensearch-net: - driver: bridge \ No newline at end of file + driver: bridge + ipam: + config: + - subnet: 172.22.0.0/24 \ No newline at end of file diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 34026e705..ee18f7762 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -44,7 +44,7 @@ chapters: cluster: remote: my-connection-alias: - seeds: ['172.18.0.3:9300'] + seeds: ['172.22.0.3:9300'] response: status: 200 - synopsis: Start replication. From e83589c1ac1e661264a04b1d9aeb373cb31765fa Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:31:52 +0500 Subject: [PATCH 16/73] trying to fix docker-compose problems Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index 5257f82e3..cfb219944 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -4,7 +4,7 @@ services: opensearch-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} environment: - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - bootstrap.memory_lock=true - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 From bc2800b8b0cf3e8553f5cd8ded0c901cf5aabb9a Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:35:20 +0500 Subject: [PATCH 17/73] adding verbose to check Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 55b24e50d..382bb88ea 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -76,7 +76,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure \ + --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ --tests=tests/${{ matrix.entry.tests || 'default' }} From 87bdbdb671375d57b8ee792496c1655469ba1dbb Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:38:38 +0500 Subject: [PATCH 18/73] changing follower cluster role Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index ee18f7762..dbd241fba 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -112,7 +112,7 @@ chapters: pattern: 'books-*' use_roles: leader_cluster_role: leader_role - follower_cluster_role: follower_role + follower_cluster_role: admin_role response: status: 200 - synopsis: Delete replication rule. From f075bd7780dc707af9e392b600b3fd1dd9eab442 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:40:23 +0500 Subject: [PATCH 19/73] adding default follower role Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index dbd241fba..337324038 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -112,7 +112,7 @@ chapters: pattern: 'books-*' use_roles: leader_cluster_role: leader_role - follower_cluster_role: admin_role + follower_cluster_role: admin response: status: 200 - synopsis: Delete replication rule. From 5cd2b95b932d32bbd0f42fd25bb7747b4d658d1c Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:53:47 +0500 Subject: [PATCH 20/73] trying to catch all possible roles Signed-off-by: Tokesh --- tests/replication/replica.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 337324038..3c095c2db 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -1,6 +1,6 @@ $schema: ../../json_schemas/test_story.schema.yaml -description: Test to explicitly clear the cursor context. +description: Test replication APIs specs. prologues: - path: /{index} method: PUT @@ -21,6 +21,14 @@ prologues: method: POST parameters: index: books + - path: /_plugins/security/authinfo + method: GET + response: + status: 200 + - path: /_plugins/_security/api/roles + method: GET + response: + status: 200 epilogues: - path: /_plugins/_replication/{index}/_stop method: POST From d114038152b992020cfbfd91cac55698874d4c79 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:55:31 +0500 Subject: [PATCH 21/73] deleting response Signed-off-by: Tokesh --- tests/replication/replica.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 3c095c2db..d3546fd57 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -23,12 +23,8 @@ prologues: index: books - path: /_plugins/security/authinfo method: GET - response: - status: 200 - path: /_plugins/_security/api/roles method: GET - response: - status: 200 epilogues: - path: /_plugins/_replication/{index}/_stop method: POST From ac1dc59b1089e999cbe448b9b3fe48dfeb41d175 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 00:58:46 +0500 Subject: [PATCH 22/73] adding all access Signed-off-by: Tokesh --- tests/replication/replica.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index d3546fd57..0397bfbf7 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -21,8 +21,6 @@ prologues: method: POST parameters: index: books - - path: /_plugins/security/authinfo - method: GET - path: /_plugins/_security/api/roles method: GET epilogues: @@ -63,7 +61,7 @@ chapters: leader_index: books use_roles: leader_cluster_role: leader_role - follower_cluster_role: follower_role + follower_cluster_role: all_access response: status: 200 - synopsis: Get replication stats. From 8c6ec73b9561d84557f03a9761a254510107f67f Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 01:06:57 +0500 Subject: [PATCH 23/73] checking all_access Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 0397bfbf7..919be191f 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -23,6 +23,8 @@ prologues: index: books - path: /_plugins/_security/api/roles method: GET + - path: /_plugins/_security/api/rolesmapping/all_access + method: GET epilogues: - path: /_plugins/_replication/{index}/_stop method: POST From 334b09d01e51b246e173790fca725080abfb9169 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 01:13:00 +0500 Subject: [PATCH 24/73] adding admin to rolemapping Signed-off-by: Tokesh --- tests/replication/replica.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 919be191f..6b490e409 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -39,6 +39,15 @@ epilogues: method: DELETE status: [200, 404] chapters: + - synopsis: Patch rolesmapping bulk. + path: /_plugins/_security/api/rolesmapping + method: PATCH + request: + payload: + - op: add + path: /all_access + value: + users: [admin] - synopsis: Establish connection. path: /_cluster/settings method: PUT From 018be31bdaa8eeecd5011f4fc7f653d6fb6d7f39 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 01:15:33 +0500 Subject: [PATCH 25/73] checking current user info Signed-off-by: Tokesh --- tests/replication/replica.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 6b490e409..5c0211889 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -39,6 +39,15 @@ epilogues: method: DELETE status: [200, 404] chapters: + - synopsis: Get current user info. + path: /_plugins/_security/whoami + method: GET + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false - synopsis: Patch rolesmapping bulk. path: /_plugins/_security/api/rolesmapping method: PATCH From b4f6a3cdf86c6567923906ae88fc1ab74e6d402a Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 21:28:18 +0500 Subject: [PATCH 26/73] testing new docker-compose Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index cfb219944..a9a4d4d07 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -13,6 +13,8 @@ services: - node.name=opensearch-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 @@ -36,6 +38,8 @@ services: - node.name=opensearch-node2 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 From f609a97e024824876b52a013c7c33d1aaf578367 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 21:34:37 +0500 Subject: [PATCH 27/73] adding opensearch_url Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- tests/replication/docker-compose.yml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 382bb88ea..b5d54ec55 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -50,7 +50,7 @@ jobs: OPENSEARCH_VERSION: ${{ matrix.entry.version }} OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }} OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g - + OPENSEARCH_URL: {{ matrix.entry.url || 'http://localhost:9200'}} steps: - name: Checkout Repo uses: actions/checkout@v4 diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index a9a4d4d07..cfb219944 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -13,8 +13,6 @@ services: - node.name=opensearch-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 @@ -38,8 +36,6 @@ services: - node.name=opensearch-node2 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 From b2d92e81628b5ff118af61dbff2ed9ba33dea9b0 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 21:40:20 +0500 Subject: [PATCH 28/73] trying to change url Signed-off-by: Tokesh --- .cspell | 1 + .github/workflows/test-spec.yml | 2 +- tests/replication/docker-compose.yml | 4 ++++ tools/src/OpenSearchHttpClient.ts | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.cspell b/.cspell index a11a97e01..4bd64e884 100644 --- a/.cspell +++ b/.cspell @@ -68,6 +68,7 @@ hybridfs integ internalusers Intertransport +ipam Jelinek kibanainfo kibanaserver diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index b5d54ec55..382bb88ea 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -50,7 +50,7 @@ jobs: OPENSEARCH_VERSION: ${{ matrix.entry.version }} OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }} OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g - OPENSEARCH_URL: {{ matrix.entry.url || 'http://localhost:9200'}} + steps: - name: Checkout Repo uses: actions/checkout@v4 diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index cfb219944..a9a4d4d07 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -13,6 +13,8 @@ services: - node.name=opensearch-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 @@ -36,6 +38,8 @@ services: - node.name=opensearch-node2 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index 314a73d45..b82558ef2 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -14,7 +14,7 @@ import { sleep } from './helpers' import { Logger } from './Logger' import { aws4Interceptor } from 'aws4-axios' -const DEFAULT_URL = 'https://localhost:9200' +const DEFAULT_URL = 'http://localhost:9200' const DEFAULT_USER = 'admin' const DEFAULT_INSECURE = false From bbc3ae109df726bdd2ddd4572ad52bef12b43621 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 21:55:02 +0500 Subject: [PATCH 29/73] trying to handle in https Signed-off-by: Tokesh --- .env | 0 tests/replication/docker-compose.yml | 4 ---- tests/replication/replica.yaml | 24 +----------------------- 3 files changed, 1 insertion(+), 27 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 000000000..e69de29bb diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index a9a4d4d07..cfb219944 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -13,8 +13,6 @@ services: - node.name=opensearch-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 @@ -38,8 +36,6 @@ services: - node.name=opensearch-node2 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true ulimits: memlock: soft: -1 diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 5c0211889..1c85d4403 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -21,10 +21,6 @@ prologues: method: POST parameters: index: books - - path: /_plugins/_security/api/roles - method: GET - - path: /_plugins/_security/api/rolesmapping/all_access - method: GET epilogues: - path: /_plugins/_replication/{index}/_stop method: POST @@ -39,24 +35,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get current user info. - path: /_plugins/_security/whoami - method: GET - response: - status: 200 - payload: - dn: null - is_admin: false - is_node_certificate_request: false - - synopsis: Patch rolesmapping bulk. - path: /_plugins/_security/api/rolesmapping - method: PATCH - request: - payload: - - op: add - path: /all_access - value: - users: [admin] - synopsis: Establish connection. path: /_cluster/settings method: PUT @@ -81,7 +59,7 @@ chapters: leader_index: books use_roles: leader_cluster_role: leader_role - follower_cluster_role: all_access + follower_cluster_role: cross_cluster_replication_follower_full_access response: status: 200 - synopsis: Get replication stats. From a1c8cc05b5e625292ded0fa5cbf47734fb615abc Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 21:55:09 +0500 Subject: [PATCH 30/73] trying to handle in https Signed-off-by: Tokesh --- tools/src/OpenSearchHttpClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index b82558ef2..314a73d45 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -14,7 +14,7 @@ import { sleep } from './helpers' import { Logger } from './Logger' import { aws4Interceptor } from 'aws4-axios' -const DEFAULT_URL = 'http://localhost:9200' +const DEFAULT_URL = 'https://localhost:9200' const DEFAULT_USER = 'admin' const DEFAULT_INSECURE = false From f2a898dc4e05beea968fd9c99957ae3c41631090 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:01:14 +0500 Subject: [PATCH 31/73] checking role mapping Signed-off-by: Tokesh --- tests/replication/replica.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 1c85d4403..ff8a2f699 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -35,6 +35,9 @@ epilogues: method: DELETE status: [200, 404] chapters: + - synopsis: Check role mapping + path: /_plugins/_security/api/rolesmapping/admin + method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT From bd92a0cac6d2948c8e57f38ee72b9be3f747db9d Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:03:04 +0500 Subject: [PATCH 32/73] adding round at the end of synopsis Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index ff8a2f699..9b0ee4216 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -35,7 +35,7 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Check role mapping + - synopsis: Check role mapping. path: /_plugins/_security/api/rolesmapping/admin method: GET - synopsis: Establish connection. From 78033833e99c9dad9fd63bf58ed5a44c1c0678e8 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:06:11 +0500 Subject: [PATCH 33/73] adding getting users and user detail Signed-off-by: Tokesh --- tests/replication/replica.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 9b0ee4216..c664b12c2 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -35,9 +35,14 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Check role mapping. - path: /_plugins/_security/api/rolesmapping/admin + - synopsis: Get user bulk. + path: /_plugins/_security/api/user method: GET + - synopsis: Get user. + path: /_plugins/_security/api/user/{username} + method: GET + parameters: + username: admin - synopsis: Establish connection. path: /_cluster/settings method: PUT From e04fec9f59eb972efc95f02ad64696db4a81e57e Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:14:42 +0500 Subject: [PATCH 34/73] trying to create Signed-off-by: Tokesh --- tests/replication/replica.yaml | 57 +++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index c664b12c2..3e4e07eed 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -43,6 +43,53 @@ chapters: method: GET parameters: username: admin + + - synopsis: Create leader role. + path: /_plugins/_security/api/roles/leader_role + method: PUT + request: + payload: + cluster_permissions: + - "cluster:monitor/state" + index_permissions: + - index_patterns: ["*"] + allowed_actions: ["indices:data/read/*"] + response: + status: 200 + + - synopsis: Create follower role. + path: /_plugins/_security/api/roles/cross_cluster_replication_follower_full_access + method: PUT + request: + payload: + cluster_permissions: + - "cluster:admin/xpack/ccr/auto_follow_pattern/*" + index_permissions: + - index_patterns: ["*"] + allowed_actions: ["indices:data/write/*"] + response: + status: 200 + + - synopsis: Map leader role to user. + path: /_plugins/_security/api/rolesmapping/leader_role + method: PUT + request: + payload: + backend_roles: ["admin"] + users: ["admin"] + response: + status: 200 + + - synopsis: Map follower role to user. + path: /_plugins/_security/api/rolesmapping/cross_cluster_replication_follower_full_access + method: PUT + request: + payload: + backend_roles: ["admin"] + users: ["admin"] + response: + status: 200 + - synopsis: Establish connection. path: /_cluster/settings method: PUT @@ -55,6 +102,7 @@ chapters: seeds: ['172.22.0.3:9300'] response: status: 200 + - synopsis: Start replication. path: /_plugins/_replication/{index}/_start version: '>= 1.1' @@ -70,6 +118,7 @@ chapters: follower_cluster_role: cross_cluster_replication_follower_full_access response: status: 200 + - synopsis: Get replication stats. path: /_plugins/_replication/{index}/_status version: '>= 1.1' @@ -78,24 +127,28 @@ chapters: index: books-names response: status: 200 + - synopsis: Get leader stats. path: /_plugins/_replication/leader_stats version: '>= 1.1' method: GET response: status: 200 + - synopsis: Get follower stats. path: /_plugins/_replication/follower_stats version: '>= 1.1' method: GET response: status: 200 + - synopsis: Get auto-follow stats. path: /_plugins/_replication/autofollow_stats version: '>= 1.1' method: GET response: status: 200 + - synopsis: Update settings. path: /_plugins/_replication/{index}/_update version: '>= 1.1' @@ -109,6 +162,7 @@ chapters: index.number_of_replicas: 2 response: status: 200 + - synopsis: Create replication rule. path: /_plugins/_replication/_autofollow version: '>= 1.1' @@ -123,6 +177,7 @@ chapters: follower_cluster_role: admin response: status: 200 + - synopsis: Delete replication rule. path: /_plugins/_replication/_autofollow version: '>= 1.1' @@ -132,4 +187,4 @@ chapters: leader_alias: my-connection-alias name: books response: - status: 200 \ No newline at end of file + status: 200 From 90e72da6827789734ca2c5b211c5e013cbc92fc3 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:17:55 +0500 Subject: [PATCH 35/73] fixing uri Signed-off-by: Tokesh --- tests/replication/replica.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 3e4e07eed..e6aee9780 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -45,7 +45,9 @@ chapters: username: admin - synopsis: Create leader role. - path: /_plugins/_security/api/roles/leader_role + path: /_plugins/_security/api/roles/{role} + parameters: + role: leader_role method: PUT request: payload: @@ -58,8 +60,10 @@ chapters: status: 200 - synopsis: Create follower role. - path: /_plugins/_security/api/roles/cross_cluster_replication_follower_full_access + path: /_plugins/_security/api/roles/{role} method: PUT + parameters: + role: cross_cluster_replication_follower_full_access request: payload: cluster_permissions: @@ -71,8 +75,10 @@ chapters: status: 200 - synopsis: Map leader role to user. - path: /_plugins/_security/api/rolesmapping/leader_role + path: /_plugins/_security/api/rolesmapping/{role} method: PUT + parameters: + role: leader_role request: payload: backend_roles: ["admin"] @@ -81,8 +87,10 @@ chapters: status: 200 - synopsis: Map follower role to user. - path: /_plugins/_security/api/rolesmapping/cross_cluster_replication_follower_full_access + path: /_plugins/_security/api/rolesmapping/{role} method: PUT + parameters: + role: cross_cluster_replication_follower_full_access request: payload: backend_roles: ["admin"] From e79cb978ca0707a556b8f0e3844e060ac1722bf8 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:23:15 +0500 Subject: [PATCH 36/73] deleting not neccessary synopsis Signed-off-by: Tokesh --- tests/replication/replica.yaml | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index e6aee9780..240617f98 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -35,15 +35,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get user bulk. - path: /_plugins/_security/api/user - method: GET - - synopsis: Get user. - path: /_plugins/_security/api/user/{username} - method: GET - parameters: - username: admin - - synopsis: Create leader role. path: /_plugins/_security/api/roles/{role} parameters: @@ -58,12 +49,11 @@ chapters: allowed_actions: ["indices:data/read/*"] response: status: 200 - - synopsis: Create follower role. path: /_plugins/_security/api/roles/{role} method: PUT parameters: - role: cross_cluster_replication_follower_full_access + role: follower request: payload: cluster_permissions: @@ -73,7 +63,6 @@ chapters: allowed_actions: ["indices:data/write/*"] response: status: 200 - - synopsis: Map leader role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT @@ -85,19 +74,17 @@ chapters: users: ["admin"] response: status: 200 - - synopsis: Map follower role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT parameters: - role: cross_cluster_replication_follower_full_access + role: follower request: payload: backend_roles: ["admin"] users: ["admin"] response: status: 200 - - synopsis: Establish connection. path: /_cluster/settings method: PUT @@ -110,7 +97,6 @@ chapters: seeds: ['172.22.0.3:9300'] response: status: 200 - - synopsis: Start replication. path: /_plugins/_replication/{index}/_start version: '>= 1.1' @@ -123,10 +109,9 @@ chapters: leader_index: books use_roles: leader_cluster_role: leader_role - follower_cluster_role: cross_cluster_replication_follower_full_access + follower_cluster_role: follower response: status: 200 - - synopsis: Get replication stats. path: /_plugins/_replication/{index}/_status version: '>= 1.1' @@ -135,28 +120,24 @@ chapters: index: books-names response: status: 200 - - synopsis: Get leader stats. path: /_plugins/_replication/leader_stats version: '>= 1.1' method: GET response: status: 200 - - synopsis: Get follower stats. path: /_plugins/_replication/follower_stats version: '>= 1.1' method: GET response: status: 200 - - synopsis: Get auto-follow stats. path: /_plugins/_replication/autofollow_stats version: '>= 1.1' method: GET response: status: 200 - - synopsis: Update settings. path: /_plugins/_replication/{index}/_update version: '>= 1.1' @@ -170,7 +151,6 @@ chapters: index.number_of_replicas: 2 response: status: 200 - - synopsis: Create replication rule. path: /_plugins/_replication/_autofollow version: '>= 1.1' @@ -185,7 +165,6 @@ chapters: follower_cluster_role: admin response: status: 200 - - synopsis: Delete replication rule. path: /_plugins/_replication/_autofollow version: '>= 1.1' From fd4349cda47a13f549575798d44221752217c58f Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:25:23 +0500 Subject: [PATCH 37/73] deleting not neccessary synopsis Signed-off-by: Tokesh --- tests/replication/replica.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 240617f98..44445c9bc 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -47,8 +47,6 @@ chapters: index_permissions: - index_patterns: ["*"] allowed_actions: ["indices:data/read/*"] - response: - status: 200 - synopsis: Create follower role. path: /_plugins/_security/api/roles/{role} method: PUT @@ -61,8 +59,6 @@ chapters: index_permissions: - index_patterns: ["*"] allowed_actions: ["indices:data/write/*"] - response: - status: 200 - synopsis: Map leader role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT @@ -72,8 +68,6 @@ chapters: payload: backend_roles: ["admin"] users: ["admin"] - response: - status: 200 - synopsis: Map follower role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT @@ -83,8 +77,6 @@ chapters: payload: backend_roles: ["admin"] users: ["admin"] - response: - status: 200 - synopsis: Establish connection. path: /_cluster/settings method: PUT From 2b6dbdb9e47de270489f745a86e064f394bbb5ba Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:28:51 +0500 Subject: [PATCH 38/73] waiting 201 not 200 Signed-off-by: Tokesh --- tests/replication/replica.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 44445c9bc..f76a90d4a 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -47,6 +47,8 @@ chapters: index_permissions: - index_patterns: ["*"] allowed_actions: ["indices:data/read/*"] + response: + status: 201 - synopsis: Create follower role. path: /_plugins/_security/api/roles/{role} method: PUT @@ -59,6 +61,8 @@ chapters: index_permissions: - index_patterns: ["*"] allowed_actions: ["indices:data/write/*"] + response: + status: 201 - synopsis: Map leader role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT @@ -68,6 +72,8 @@ chapters: payload: backend_roles: ["admin"] users: ["admin"] + response: + status: 201 - synopsis: Map follower role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT @@ -77,6 +83,8 @@ chapters: payload: backend_roles: ["admin"] users: ["admin"] + response: + status: 201 - synopsis: Establish connection. path: /_cluster/settings method: PUT From 56e1c56fb54655bb6b51841f2ae0135be9573c00 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:35:33 +0500 Subject: [PATCH 39/73] adding permissions Signed-off-by: Tokesh --- tests/replication/replica.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index f76a90d4a..c0f8eeddb 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -44,9 +44,10 @@ chapters: payload: cluster_permissions: - "cluster:monitor/state" + - "cluster:admin/plugins/replication/index/setup/validate" index_permissions: - index_patterns: ["*"] - allowed_actions: ["indices:data/read/*"] + allowed_actions: ["indices:data/read/*", "indices:admin/plugins/replication/*"] response: status: 201 - synopsis: Create follower role. @@ -58,9 +59,10 @@ chapters: payload: cluster_permissions: - "cluster:admin/xpack/ccr/auto_follow_pattern/*" + - "cluster:admin/plugins/replication/index/setup/validate" index_permissions: - index_patterns: ["*"] - allowed_actions: ["indices:data/write/*"] + allowed_actions: ["indices:data/write/*", "indices:admin/plugins/replication/*"] response: status: 201 - synopsis: Map leader role to user. From ba4dcea37300f37515da78dc262d17589169a186 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Thu, 15 Aug 2024 22:45:10 +0500 Subject: [PATCH 40/73] formatting yaml Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index c0f8eeddb..4bd848897 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -62,7 +62,7 @@ chapters: - "cluster:admin/plugins/replication/index/setup/validate" index_permissions: - index_patterns: ["*"] - allowed_actions: ["indices:data/write/*", "indices:admin/plugins/replication/*"] + allowed_actions: ["indices:data/write/*", "indices:admin/plugins/replication/*"] response: status: 201 - synopsis: Map leader role to user. From 4e20ee956f66824cd1873d3a75ff12ddfeb67916 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:44:40 +0500 Subject: [PATCH 41/73] testing concept from documentation Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- tests/replication/replica.yaml | 45 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 4bd848897..563a8ee14 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -36,18 +36,21 @@ epilogues: status: [200, 404] chapters: - synopsis: Create leader role. - path: /_plugins/_security/api/roles/{role} - parameters: - role: leader_role - method: PUT - request: - payload: - cluster_permissions: - - "cluster:monitor/state" - - "cluster:admin/plugins/replication/index/setup/validate" - index_permissions: - - index_patterns: ["*"] - allowed_actions: ["indices:data/read/*", "indices:admin/plugins/replication/*"] + path: /_plugins/_security/api/roles/{role} + parameters: + role: leader_role + method: PUT + request: + payload: + cluster_permissions: + - "cluster:monitor/state" + - "cluster:admin/plugins/replication/index/setup/validate" + index_permissions: + - index_patterns: ["your-leader-index-pattern"] # Укажите конкретный шаблон индекса + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:data/read/plugins/replication/file_chunk" + - "indices:data/read/plugins/replication/changes" response: status: 201 - synopsis: Create follower role. @@ -58,13 +61,21 @@ chapters: request: payload: cluster_permissions: - - "cluster:admin/xpack/ccr/auto_follow_pattern/*" - "cluster:admin/plugins/replication/index/setup/validate" + - "cluster:admin/plugins/replication/autofollow/update" index_permissions: - - index_patterns: ["*"] - allowed_actions: ["indices:data/write/*", "indices:admin/plugins/replication/*"] - response: - status: 201 + - index_patterns: ["your-follower-index-pattern"] # Укажите конкретный шаблон индекса + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:admin/plugins/replication/index/start" + - "indices:admin/plugins/replication/index/pause" + - "indices:admin/plugins/replication/index/resume" + - "indices:admin/plugins/replication/index/stop" + - "indices:admin/plugins/replication/index/update" + - "indices:admin/plugins/replication/index/status_check" + - "indices:data/write/plugins/replication/changes" + response: + status: 201 - synopsis: Map leader role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT From 03c4bc2ff99ac232e78d913a9c6a01b12debbe9e Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:47:26 +0500 Subject: [PATCH 42/73] fixing tabling Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- tests/replication/replica.yaml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 563a8ee14..ddef9810c 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -36,23 +36,23 @@ epilogues: status: [200, 404] chapters: - synopsis: Create leader role. - path: /_plugins/_security/api/roles/{role} - parameters: - role: leader_role - method: PUT - request: - payload: - cluster_permissions: - - "cluster:monitor/state" - - "cluster:admin/plugins/replication/index/setup/validate" - index_permissions: - - index_patterns: ["your-leader-index-pattern"] # Укажите конкретный шаблон индекса - allowed_actions: - - "indices:admin/plugins/replication/index/setup/validate" - - "indices:data/read/plugins/replication/file_chunk" - - "indices:data/read/plugins/replication/changes" - response: - status: 201 + path: /_plugins/_security/api/roles/{role} + parameters: + role: leader_role + method: PUT + request: + payload: + cluster_permissions: + - "cluster:monitor/state" + - "cluster:admin/plugins/replication/index/setup/validate" + index_permissions: + - index_patterns: ["your-leader-index-pattern"] # Укажите конкретный шаблон индекса + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:data/read/plugins/replication/file_chunk" + - "indices:data/read/plugins/replication/changes" + response: + status: 201 - synopsis: Create follower role. path: /_plugins/_security/api/roles/{role} method: PUT From 345c0544cbbbc2be05c05319b4bdb3114d2a0227 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:49:59 +0500 Subject: [PATCH 43/73] fixing tabling v2 Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- tests/replication/replica.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index ddef9810c..2bdb63b54 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -51,8 +51,8 @@ chapters: - "indices:admin/plugins/replication/index/setup/validate" - "indices:data/read/plugins/replication/file_chunk" - "indices:data/read/plugins/replication/changes" - response: - status: 201 + response: + status: 201 - synopsis: Create follower role. path: /_plugins/_security/api/roles/{role} method: PUT @@ -74,8 +74,8 @@ chapters: - "indices:admin/plugins/replication/index/update" - "indices:admin/plugins/replication/index/status_check" - "indices:data/write/plugins/replication/changes" - response: - status: 201 + response: + status: 201 - synopsis: Map leader role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT From 3f98811e7591a263c0af0a8e09aae3efcd955d48 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:53:09 +0500 Subject: [PATCH 44/73] adding index pattern Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- tests/replication/replica.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 2bdb63b54..b82c4428d 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -46,7 +46,7 @@ chapters: - "cluster:monitor/state" - "cluster:admin/plugins/replication/index/setup/validate" index_permissions: - - index_patterns: ["your-leader-index-pattern"] # Укажите конкретный шаблон индекса + - index_patterns: ["books-*"] allowed_actions: - "indices:admin/plugins/replication/index/setup/validate" - "indices:data/read/plugins/replication/file_chunk" @@ -64,7 +64,7 @@ chapters: - "cluster:admin/plugins/replication/index/setup/validate" - "cluster:admin/plugins/replication/autofollow/update" index_permissions: - - index_patterns: ["your-follower-index-pattern"] # Укажите конкретный шаблон индекса + - index_patterns: ["books-*"] allowed_actions: - "indices:admin/plugins/replication/index/setup/validate" - "indices:admin/plugins/replication/index/start" From 8798e824d1d7e6509a944b7fcb3d2e4fd24baf35 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:59:02 +0500 Subject: [PATCH 45/73] getting all role mappings Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- tests/replication/replica.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index b82c4428d..a2c83dde4 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -98,6 +98,9 @@ chapters: users: ["admin"] response: status: 201 + - synopsis: Getting all role mappings. + path: /_plugins/_security/api/rolesmapping + method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT From 5ed10fbfa5bf7611497eb1453691ac6a5ee15bc7 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Fri, 6 Sep 2024 01:05:10 +0500 Subject: [PATCH 46/73] checking roles Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- tests/replication/replica.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index a2c83dde4..59e64e986 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -101,6 +101,21 @@ chapters: - synopsis: Getting all role mappings. path: /_plugins/_security/api/rolesmapping method: GET + - synopsis: Get `follower` role permissions. + path: /_plugins/_security/api/roles/{role} + parameters: + role: follower + method: GET + - synopsis: Get `leader` role permissions. + path: /_plugins/_security/api/roles/{role} + parameters: + role: leader_role + method: GET + - synopsis: Get `leader` role permissions. + path: /_plugins/_security/api/roles/{role} + parameters: + role: all_access + method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT From 50e336da1cf428833daabcad469b54766d5c31f6 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 02:20:18 +0500 Subject: [PATCH 47/73] getting login roles Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 59e64e986..f74842b57 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -99,7 +99,7 @@ chapters: response: status: 201 - synopsis: Getting all role mappings. - path: /_plugins/_security/api/rolesmapping + path: /_plugins/_security/authinfo method: GET - synopsis: Get `follower` role permissions. path: /_plugins/_security/api/roles/{role} From 0b3279b7e343e34a040d479dcac091550bfc7974 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 02:31:40 +0500 Subject: [PATCH 48/73] trying roles Signed-off-by: Tokesh --- tests/replication/replica.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index f74842b57..7475a956b 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -139,8 +139,8 @@ chapters: leader_alias: my-connection-alias leader_index: books use_roles: - leader_cluster_role: leader_role - follower_cluster_role: follower + leader_cluster_role: all_access + follower_cluster_role: all_access response: status: 200 - synopsis: Get replication stats. From a49a4e55a119f70326233a3291ec1e07365d59b5 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 03:43:42 +0500 Subject: [PATCH 49/73] testing docker compose change Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 31 +++++++++++++++++----------- tests/replication/replica.yaml | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index cfb219944..16d5c09fc 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -1,18 +1,19 @@ version: '3' services: - opensearch-node1: + opensearch-leader-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} environment: - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - bootstrap.memory_lock=true - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - cluster.name=opensearch-cluster - - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - node.name=opensearch-node1 + - cluster.initial_master_nodes=opensearch-leader-node1 + - cluster.name=leader-cluster + - discovery.seed_hosts=opensearch-leader-node1 + - node.name=opensearch-leader-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + - network.host=0.0.0.0 ulimits: memlock: soft: -1 @@ -24,28 +25,34 @@ services: opensearch-net: ipv4_address: 172.22.0.3 - opensearch-node2: + opensearch-follower-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} environment: - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - bootstrap.memory_lock=true - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - cluster.name=opensearch-cluster - - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - node.name=opensearch-node2 + - cluster.initial_master_nodes=opensearch-follower-node1 + - cluster.name=follower-cluster + - discovery.seed_hosts=opensearch-follower-node1 + - node.name=opensearch-follower-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + - network.host=0.0.0.0 + - cluster.remote.leader-cluster.seeds=172.22.0.3:9300 ulimits: memlock: soft: -1 hard: -1 + ports: + - '9201:9200' + - '9601:9600' networks: opensearch-net: ipv4_address: 172.22.0.4 + networks: opensearch-net: driver: bridge ipam: config: - - subnet: 172.22.0.0/24 \ No newline at end of file + - subnet: 172.22.0.0/24 diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 7475a956b..a336a6fa3 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -124,7 +124,7 @@ chapters: persistent: cluster: remote: - my-connection-alias: + leader-cluster: seeds: ['172.22.0.3:9300'] response: status: 200 From 2006adb61bf4d871ed29f157dbea93b2b88add94 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 03:46:03 +0500 Subject: [PATCH 50/73] hotfix cluster alias in tests Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index a336a6fa3..f60b5bc34 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -136,7 +136,7 @@ chapters: index: books-names request: payload: - leader_alias: my-connection-alias + leader_alias: leader-cluster leader_index: books use_roles: leader_cluster_role: all_access From 78f73ff909f05211b1358cd22fe03b1abce1b143 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 04:01:09 +0500 Subject: [PATCH 51/73] giving all access to users Signed-off-by: Tokesh --- tests/replication/replica.yaml | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index f60b5bc34..af4b52bfb 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -61,41 +61,22 @@ chapters: request: payload: cluster_permissions: - - "cluster:admin/plugins/replication/index/setup/validate" - - "cluster:admin/plugins/replication/autofollow/update" + - 'cluster:all' index_permissions: - - index_patterns: ["books-*"] + - index_patterns: ["*"] allowed_actions: - - "indices:admin/plugins/replication/index/setup/validate" - - "indices:admin/plugins/replication/index/start" - - "indices:admin/plugins/replication/index/pause" - - "indices:admin/plugins/replication/index/resume" - - "indices:admin/plugins/replication/index/stop" - - "indices:admin/plugins/replication/index/update" - - "indices:admin/plugins/replication/index/status_check" - - "indices:data/write/plugins/replication/changes" + ["*"] response: status: 201 - synopsis: Map leader role to user. path: /_plugins/_security/api/rolesmapping/{role} method: PUT parameters: - role: leader_role - request: - payload: - backend_roles: ["admin"] - users: ["admin"] - response: - status: 201 - - synopsis: Map follower role to user. - path: /_plugins/_security/api/rolesmapping/{role} - method: PUT - parameters: - role: follower + role: all_access request: payload: - backend_roles: ["admin"] - users: ["admin"] + backend_roles: ["admin", "leader-role", "follower"] + users: ["admin", "leader-role", "follower"] response: status: 201 - synopsis: Getting all role mappings. From 485e213040ad3c5e66d132c2eaa4646ed8be0bf9 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 04:04:06 +0500 Subject: [PATCH 52/73] hotfix status code in response Signed-off-by: Tokesh --- tests/replication/replica.yaml | 39 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index af4b52bfb..8e474e6b5 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -78,25 +78,7 @@ chapters: backend_roles: ["admin", "leader-role", "follower"] users: ["admin", "leader-role", "follower"] response: - status: 201 - - synopsis: Getting all role mappings. - path: /_plugins/_security/authinfo - method: GET - - synopsis: Get `follower` role permissions. - path: /_plugins/_security/api/roles/{role} - parameters: - role: follower - method: GET - - synopsis: Get `leader` role permissions. - path: /_plugins/_security/api/roles/{role} - parameters: - role: leader_role - method: GET - - synopsis: Get `leader` role permissions. - path: /_plugins/_security/api/roles/{role} - parameters: - role: all_access - method: GET + status: 200 - synopsis: Establish connection. path: /_cluster/settings method: PUT @@ -187,3 +169,22 @@ chapters: name: books response: status: 200 + +# - synopsis: Getting all role mappings. +# path: /_plugins/_security/authinfo +# method: GET +# - synopsis: Get `follower` role permissions. +# path: /_plugins/_security/api/roles/{role} +# parameters: +# role: follower +# method: GET +# - synopsis: Get `leader` role permissions. +# path: /_plugins/_security/api/roles/{role} +# parameters: +# role: leader_role +# method: GET +# - synopsis: Get `leader` role permissions. + path: /_plugins/_security/api/roles/{role} + parameters: + role: all_access + method: GET \ No newline at end of file From aaac7d709304da25d6d671b64177218e34b75491 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 04:06:06 +0500 Subject: [PATCH 53/73] status code fixing Signed-off-by: Tokesh --- tests/replication/replica.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 8e474e6b5..c86d45d3e 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -78,7 +78,7 @@ chapters: backend_roles: ["admin", "leader-role", "follower"] users: ["admin", "leader-role", "follower"] response: - status: 200 + status: [200, 201] - synopsis: Establish connection. path: /_cluster/settings method: PUT From 8ede2deba07a719aa106b34413c0daec236c03df Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 04:09:59 +0500 Subject: [PATCH 54/73] deleting response for tests Signed-off-by: Tokesh --- tests/replication/replica.yaml | 39 ++++++++++++++++------------------ 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index c86d45d3e..b5339aac9 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -77,8 +77,24 @@ chapters: payload: backend_roles: ["admin", "leader-role", "follower"] users: ["admin", "leader-role", "follower"] - response: - status: [200, 201] + # - synopsis: Getting all role mappings. + # path: /_plugins/_security/authinfo + # method: GET + # - synopsis: Get `follower` role permissions. + # path: /_plugins/_security/api/roles/{role} + # parameters: + # role: follower + # method: GET + # - synopsis: Get `leader` role permissions. + # path: /_plugins/_security/api/roles/{role} + # parameters: + # role: leader_role + # method: GET + # - synopsis: Get `leader` role permissions. + # path: /_plugins/_security/api/roles/{role} + # parameters: + # role: all_access + # method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT @@ -169,22 +185,3 @@ chapters: name: books response: status: 200 - -# - synopsis: Getting all role mappings. -# path: /_plugins/_security/authinfo -# method: GET -# - synopsis: Get `follower` role permissions. -# path: /_plugins/_security/api/roles/{role} -# parameters: -# role: follower -# method: GET -# - synopsis: Get `leader` role permissions. -# path: /_plugins/_security/api/roles/{role} -# parameters: -# role: leader_role -# method: GET -# - synopsis: Get `leader` role permissions. - path: /_plugins/_security/api/roles/{role} - parameters: - role: all_access - method: GET \ No newline at end of file From aa7021063bc1fff58d8180f15f55006ae9728981 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 04:18:04 +0500 Subject: [PATCH 55/73] adding all_access Signed-off-by: Tokesh --- tests/replication/replica.yaml | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index b5339aac9..b004af21a 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -43,14 +43,11 @@ chapters: request: payload: cluster_permissions: - - "cluster:monitor/state" - - "cluster:admin/plugins/replication/index/setup/validate" + - 'cluster:all' index_permissions: - - index_patterns: ["books-*"] + - index_patterns: ["*"] allowed_actions: - - "indices:admin/plugins/replication/index/setup/validate" - - "indices:data/read/plugins/replication/file_chunk" - - "indices:data/read/plugins/replication/changes" + ["*"] response: status: 201 - synopsis: Create follower role. @@ -77,24 +74,6 @@ chapters: payload: backend_roles: ["admin", "leader-role", "follower"] users: ["admin", "leader-role", "follower"] - # - synopsis: Getting all role mappings. - # path: /_plugins/_security/authinfo - # method: GET - # - synopsis: Get `follower` role permissions. - # path: /_plugins/_security/api/roles/{role} - # parameters: - # role: follower - # method: GET - # - synopsis: Get `leader` role permissions. - # path: /_plugins/_security/api/roles/{role} - # parameters: - # role: leader_role - # method: GET - # - synopsis: Get `leader` role permissions. - # path: /_plugins/_security/api/roles/{role} - # parameters: - # role: all_access - # method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT From 7f1ea97b1fc1e89ef9caad0cf3ad2677c6cd3f84 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 04:25:17 +0500 Subject: [PATCH 56/73] trying role mappings Signed-off-by: Tokesh --- tests/replication/replica.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index b004af21a..9ed27272a 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -74,6 +74,9 @@ chapters: payload: backend_roles: ["admin", "leader-role", "follower"] users: ["admin", "leader-role", "follower"] + - synopsis: Getting all role mappings. + path: /_plugins/_security/api/rolesmapping + method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT From 3b977fed6495af80444eab4c8f0b5c34a169c752 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 19:22:52 +0500 Subject: [PATCH 57/73] testing without security plugin Signed-off-by: Tokesh --- tests/replication/docker-compose.yml | 4 +++ tests/replication/replica.yaml | 46 ++-------------------------- 2 files changed, 6 insertions(+), 44 deletions(-) diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index 16d5c09fc..5f17dfaed 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -12,6 +12,8 @@ services: - discovery.seed_hosts=opensearch-leader-node1 - node.name=opensearch-leader-node1 - path.repo=/tmp/opensearch/repo + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true - plugins.index_state_management.job_interval=1 - network.host=0.0.0.0 ulimits: @@ -36,6 +38,8 @@ services: - discovery.seed_hosts=opensearch-follower-node1 - node.name=opensearch-follower-node1 - path.repo=/tmp/opensearch/repo + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true - plugins.index_state_management.job_interval=1 - network.host=0.0.0.0 - cluster.remote.leader-cluster.seeds=172.22.0.3:9300 diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 9ed27272a..447a2ad80 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -35,48 +35,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Create leader role. - path: /_plugins/_security/api/roles/{role} - parameters: - role: leader_role - method: PUT - request: - payload: - cluster_permissions: - - 'cluster:all' - index_permissions: - - index_patterns: ["*"] - allowed_actions: - ["*"] - response: - status: 201 - - synopsis: Create follower role. - path: /_plugins/_security/api/roles/{role} - method: PUT - parameters: - role: follower - request: - payload: - cluster_permissions: - - 'cluster:all' - index_permissions: - - index_patterns: ["*"] - allowed_actions: - ["*"] - response: - status: 201 - - synopsis: Map leader role to user. - path: /_plugins/_security/api/rolesmapping/{role} - method: PUT - parameters: - role: all_access - request: - payload: - backend_roles: ["admin", "leader-role", "follower"] - users: ["admin", "leader-role", "follower"] - - synopsis: Getting all role mappings. - path: /_plugins/_security/api/rolesmapping - method: GET - synopsis: Establish connection. path: /_cluster/settings method: PUT @@ -149,7 +107,7 @@ chapters: method: POST request: payload: - leader_alias: my-connection-alias + leader_alias: leader-cluster name: books pattern: 'books-*' use_roles: @@ -163,7 +121,7 @@ chapters: method: DELETE request: payload: - leader_alias: my-connection-alias + leader_alias: leader-cluster name: books response: status: 200 From f1c43e480f772de53c39edbe0f2f073486ed4483 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 19:31:06 +0500 Subject: [PATCH 58/73] testing without https Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index bc3ea73bb..b67135c3b 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -38,6 +38,7 @@ jobs: tests: snapshot - version: 2.16.0 tests: replication + url: http://localhost:9200 - version: 2.17.0 hub: opensearchstaging ref: '@sha256:1273489ebbedcb470ea13563dae4c6dc6b2ed431e87e686ed931ae0733034b25' @@ -83,6 +84,7 @@ jobs: --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ + --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} --tests=tests/${{ matrix.entry.tests || 'default' }} - name: Get Container Logs From 35ab7fee46bcf2da06e6854b5fed8929933807eb Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 19:32:19 +0500 Subject: [PATCH 59/73] fixing small tabular misstake in ci cd Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index b67135c3b..0bd1f9286 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -84,7 +84,7 @@ jobs: --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ - --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} + --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ --tests=tests/${{ matrix.entry.tests || 'default' }} - name: Get Container Logs From 74900fb1fd7ad9858522cbe4609b3480f8389c92 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 19:41:35 +0500 Subject: [PATCH 60/73] fixing order of env parametrs and adding links to lycheeignore Signed-off-by: Tokesh --- .lycheeignore | 1 + tests/replication/docker-compose.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.lycheeignore b/.lycheeignore index c859dd4da..57eb3c6b0 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -1 +1,2 @@ https://localhost:* +http://localhost:* \ No newline at end of file diff --git a/tests/replication/docker-compose.yml b/tests/replication/docker-compose.yml index 5f17dfaed..041be3f9d 100644 --- a/tests/replication/docker-compose.yml +++ b/tests/replication/docker-compose.yml @@ -4,18 +4,18 @@ services: opensearch-leader-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} environment: + - DISABLE_INSTALL_DEMO_CONFIG=true + - DISABLE_SECURITY_PLUGIN=true - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - bootstrap.memory_lock=true - cluster.initial_master_nodes=opensearch-leader-node1 - cluster.name=leader-cluster - discovery.seed_hosts=opensearch-leader-node1 + - network.host=0.0.0.0 - node.name=opensearch-leader-node1 - path.repo=/tmp/opensearch/repo - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - plugins.index_state_management.job_interval=1 - - network.host=0.0.0.0 ulimits: memlock: soft: -1 @@ -30,19 +30,19 @@ services: opensearch-follower-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} environment: + - DISABLE_INSTALL_DEMO_CONFIG=true + - DISABLE_SECURITY_PLUGIN=true - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - bootstrap.memory_lock=true - cluster.initial_master_nodes=opensearch-follower-node1 - cluster.name=follower-cluster + - cluster.remote.leader-cluster.seeds=172.22.0.3:9300 - discovery.seed_hosts=opensearch-follower-node1 + - network.host=0.0.0.0 - node.name=opensearch-follower-node1 - path.repo=/tmp/opensearch/repo - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - plugins.index_state_management.job_interval=1 - - network.host=0.0.0.0 - - cluster.remote.leader-cluster.seeds=172.22.0.3:9300 ulimits: memlock: soft: -1 From 8fc1987cace2b24871ab0f7ee3595cf38d284de1 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 23:20:42 +0500 Subject: [PATCH 61/73] ready for basic review Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- tests/{ => plugins}/replication/docker-compose.yml | 0 tests/{ => plugins}/replication/replica.yaml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/{ => plugins}/replication/docker-compose.yml (100%) rename tests/{ => plugins}/replication/replica.yaml (98%) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 0bd1f9286..29e7b02ee 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -81,7 +81,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure --verbose\ + --opensearch-insecure \ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ diff --git a/tests/replication/docker-compose.yml b/tests/plugins/replication/docker-compose.yml similarity index 100% rename from tests/replication/docker-compose.yml rename to tests/plugins/replication/docker-compose.yml diff --git a/tests/replication/replica.yaml b/tests/plugins/replication/replica.yaml similarity index 98% rename from tests/replication/replica.yaml rename to tests/plugins/replication/replica.yaml index 447a2ad80..d29445557 100644 --- a/tests/replication/replica.yaml +++ b/tests/plugins/replication/replica.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test replication APIs specs. prologues: From c1918b94b5a5ca9d286544b84894b173db1bc3ae Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 23:25:12 +0500 Subject: [PATCH 62/73] fixing path in ci/cd Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 4d2b206da..f006843fa 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,7 +37,7 @@ jobs: - version: 2.16.0 tests: snapshot - version: 2.16.0 - tests: replication + tests: plugins/replication url: http://localhost:9200 - version: 2.17.0 tests: plugins/streaming @@ -80,12 +80,6 @@ jobs: working-directory: tests/${{ matrix.entry.tests || 'default' }} run: docker compose up -d - - name: Get Container ID - id: container - run: | - CONTAINER_ID=$(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}" | head -n 1) - echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_OUTPUT - - name: Run Tests run: | npm run test:spec -- \ From 60a6ad32f9d33c4c707c64e07ef543d721d9cd60 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 10 Nov 2024 23:33:16 +0500 Subject: [PATCH 63/73] deleting env file Signed-off-by: Tokesh --- .env | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index e69de29bb..000000000 From 1c671fc687eb7c81fc90fb758e94f3541742a1a6 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 17 Nov 2024 17:32:20 +0500 Subject: [PATCH 64/73] deleting extra spaces and moving one chapter to prologue Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- tests/plugins/replication/replica.yaml | 21 +++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index f006843fa..cfcd224f3 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -79,7 +79,7 @@ jobs: - name: Run OpenSearch Cluster working-directory: tests/${{ matrix.entry.tests || 'default' }} run: docker compose up -d - + - name: Run Tests run: | npm run test:spec -- \ diff --git a/tests/plugins/replication/replica.yaml b/tests/plugins/replication/replica.yaml index d29445557..65676fe98 100644 --- a/tests/plugins/replication/replica.yaml +++ b/tests/plugins/replication/replica.yaml @@ -21,6 +21,15 @@ prologues: method: POST parameters: index: books + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] epilogues: - path: /_plugins/_replication/{index}/_stop method: POST @@ -35,18 +44,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Establish connection. - path: /_cluster/settings - method: PUT - request: - payload: - persistent: - cluster: - remote: - leader-cluster: - seeds: ['172.22.0.3:9300'] - response: - status: 200 - synopsis: Start replication. path: /_plugins/_replication/{index}/_start version: '>= 1.1' From 9ced3eff62be0d8c04b8a37a91edbd2c0179b0c0 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 17 Nov 2024 17:51:12 +0500 Subject: [PATCH 65/73] divide one file to few smaller Signed-off-by: Tokesh --- .../{replica.yaml => autofollow.yaml} | 74 +++++----------- tests/plugins/replication/index.yaml | 74 ++++++++++++++++ tests/plugins/replication/stats.yaml | 86 +++++++++++++++++++ 3 files changed, 180 insertions(+), 54 deletions(-) rename tests/plugins/replication/{replica.yaml => autofollow.yaml} (70%) create mode 100644 tests/plugins/replication/index.yaml create mode 100644 tests/plugins/replication/stats.yaml diff --git a/tests/plugins/replication/replica.yaml b/tests/plugins/replication/autofollow.yaml similarity index 70% rename from tests/plugins/replication/replica.yaml rename to tests/plugins/replication/autofollow.yaml index 65676fe98..297cc1460 100644 --- a/tests/plugins/replication/replica.yaml +++ b/tests/plugins/replication/autofollow.yaml @@ -30,22 +30,7 @@ prologues: remote: leader-cluster: seeds: ['172.22.0.3:9300'] -epilogues: - - path: /_plugins/_replication/{index}/_stop - method: POST - parameters: - index: books-names - request: - payload: {} - - path: /books-names - method: DELETE - status: [200, 404] - - path: /books - method: DELETE - status: [200, 404] -chapters: - - synopsis: Start replication. - path: /_plugins/_replication/{index}/_start + - path: /_plugins/_replication/{index}/_start version: '>= 1.1' method: PUT parameters: @@ -59,45 +44,20 @@ chapters: follower_cluster_role: all_access response: status: 200 - - synopsis: Get replication stats. - path: /_plugins/_replication/{index}/_status - version: '>= 1.1' - method: GET - parameters: - index: books-names - response: - status: 200 - - synopsis: Get leader stats. - path: /_plugins/_replication/leader_stats - version: '>= 1.1' - method: GET - response: - status: 200 - - synopsis: Get follower stats. - path: /_plugins/_replication/follower_stats - version: '>= 1.1' - method: GET - response: - status: 200 - - synopsis: Get auto-follow stats. - path: /_plugins/_replication/autofollow_stats - version: '>= 1.1' - method: GET - response: - status: 200 - - synopsis: Update settings. - path: /_plugins/_replication/{index}/_update - version: '>= 1.1' - method: PUT - parameters: +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: index: books-names request: - payload: - settings: - index.number_of_shards: 4 - index.number_of_replicas: 2 - response: - status: 200 + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +chapters: - synopsis: Create replication rule. path: /_plugins/_replication/_autofollow version: '>= 1.1' @@ -112,6 +72,12 @@ chapters: follower_cluster_role: admin response: status: 200 + - synopsis: Get auto-follow stats. + path: /_plugins/_replication/autofollow_stats + version: '>= 1.1' + method: GET + response: + status: 200 - synopsis: Delete replication rule. path: /_plugins/_replication/_autofollow version: '>= 1.1' @@ -121,4 +87,4 @@ chapters: leader_alias: leader-cluster name: books response: - status: 200 + status: 200 \ No newline at end of file diff --git a/tests/plugins/replication/index.yaml b/tests/plugins/replication/index.yaml new file mode 100644 index 000000000..a70a0b991 --- /dev/null +++ b/tests/plugins/replication/index.yaml @@ -0,0 +1,74 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test replication APIs specs. +prologues: + - path: /{index} + method: PUT + parameters: + index: books + request: + payload: {} + - path: /_bulk + method: POST + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + - path: /{index}/_refresh + method: POST + parameters: + index: books + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request: + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +chapters: + - synopsis: Start replication. + path: /_plugins/_replication/{index}/_start + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request: + payload: + leader_alias: leader-cluster + leader_index: books + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access + response: + status: 200 + - synopsis: Update settings. + path: /_plugins/_replication/{index}/_update + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request: + payload: + settings: + index.number_of_shards: 4 + index.number_of_replicas: 2 + response: + status: 200 \ No newline at end of file diff --git a/tests/plugins/replication/stats.yaml b/tests/plugins/replication/stats.yaml new file mode 100644 index 000000000..0932911fa --- /dev/null +++ b/tests/plugins/replication/stats.yaml @@ -0,0 +1,86 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test replication APIs specs. +prologues: + - path: /{index} + method: PUT + parameters: + index: books + request: + payload: {} + - path: /_bulk + method: POST + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + - path: /{index}/_refresh + method: POST + parameters: + index: books + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] + - path: /_plugins/_replication/{index}/_start + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request: + payload: + leader_alias: leader-cluster + leader_index: books + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access + response: + status: 200 +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request: + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get replication stats. + path: /_plugins/_replication/{index}/_status + version: '>= 1.1' + method: GET + parameters: + index: books-names + response: + status: 200 + - synopsis: Get leader stats. + path: /_plugins/_replication/leader_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get follower stats. + path: /_plugins/_replication/follower_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get auto-follow stats. + path: /_plugins/_replication/autofollow_stats + version: '>= 1.1' + method: GET + response: + status: 200 \ No newline at end of file From 03f959dbb4c93608079bd47eb90a032ac1143b48 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 17 Nov 2024 17:55:47 +0500 Subject: [PATCH 66/73] hotfix extra space Signed-off-by: Tokesh --- tests/plugins/replication/autofollow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugins/replication/autofollow.yaml b/tests/plugins/replication/autofollow.yaml index 297cc1460..30fa77d53 100644 --- a/tests/plugins/replication/autofollow.yaml +++ b/tests/plugins/replication/autofollow.yaml @@ -30,7 +30,7 @@ prologues: remote: leader-cluster: seeds: ['172.22.0.3:9300'] - - path: /_plugins/_replication/{index}/_start + - path: /_plugins/_replication/{index}/_start version: '>= 1.1' method: PUT parameters: From 08e4a2ded3eb9f6caa0153690f88fedbdbb79b30 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 17 Nov 2024 18:00:05 +0500 Subject: [PATCH 67/73] deleting not necessary response parts Signed-off-by: Tokesh --- tests/plugins/replication/autofollow.yaml | 2 -- tests/plugins/replication/stats.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/tests/plugins/replication/autofollow.yaml b/tests/plugins/replication/autofollow.yaml index 30fa77d53..89eabce30 100644 --- a/tests/plugins/replication/autofollow.yaml +++ b/tests/plugins/replication/autofollow.yaml @@ -42,8 +42,6 @@ prologues: use_roles: leader_cluster_role: all_access follower_cluster_role: all_access - response: - status: 200 epilogues: - path: /_plugins/_replication/{index}/_stop method: POST diff --git a/tests/plugins/replication/stats.yaml b/tests/plugins/replication/stats.yaml index 0932911fa..6584200ed 100644 --- a/tests/plugins/replication/stats.yaml +++ b/tests/plugins/replication/stats.yaml @@ -42,8 +42,6 @@ prologues: use_roles: leader_cluster_role: all_access follower_cluster_role: all_access - response: - status: 200 epilogues: - path: /_plugins/_replication/{index}/_stop method: POST From 9e1e5022ee9628e29a9a50f77cfff5cdb164653f Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 17 Nov 2024 18:19:06 +0500 Subject: [PATCH 68/73] hotfix structure Signed-off-by: Tokesh --- spec/schemas/replication._common.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index bdd11c3f3..6a2c40ea0 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -178,6 +178,8 @@ components: type: array items: type: string + last_execution_time: + type: number SettingsBody: type: object properties: From 0ab0105880f49627adf0b97d3880220afc025190 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Mon, 18 Nov 2024 00:45:44 +0500 Subject: [PATCH 69/73] using common structures in replication schemes Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 3 +- spec/namespaces/replication.yaml | 35 +++++------------------ spec/schemas/replication._common.yaml | 41 +++++++++++++++------------ 3 files changed, 31 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 6593fc130..8bc1abeea 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -36,10 +36,9 @@ jobs: tests: routing - version: 2.18.0 tests: snapshot - - version: 2.16.0 + - version: 2.18.0 tests: plugins/replication url: http://localhost:9200 - - version: 2.17.0 - version: 2.18.0 tests: plugins/streaming - version: 2.18.0 diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index f10cb9b14..db8e0c7b8 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -248,34 +248,22 @@ components: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' replication.stop@200: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' replication.pause@200: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' replication.resume@200: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' replication.status@200: content: application/json: @@ -300,23 +288,14 @@ components: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' replication.create_replication_rule@200: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' replication.delete_replication_rule@200: content: application/json: schema: - type: object - properties: - acknowledged: - type: boolean \ No newline at end of file + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' \ No newline at end of file diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 6a2c40ea0..096172962 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -10,9 +10,9 @@ components: type: object properties: leader_alias: - type: string + $ref: '_common.yaml#/components/schemas/IndexAlias' leader_index: - type: string + $ref: '_common.yaml#/components/schemas/IndexName' use_roles: $ref: '#/components/schemas/UseRoles' UseRoles: @@ -27,14 +27,19 @@ components: properties: status: type: string + enum: + - BOOTSTRAPPING + - PAUSED + - REPLICATION NOT IN PROGRESS + - SYNCING reason: type: string leader_alias: - type: string + $ref: '_common.yaml#/components/schemas/IndexAlias' leader_index: - type: string + $ref: '_common.yaml#/components/schemas/IndexName' follower_index: - type: string + $ref: '_common.yaml#/components/schemas/IndexName' syncing_details: $ref: '#/components/schemas/SyncingDetails' SyncingDetails: @@ -45,7 +50,7 @@ components: follower_checkpoint: type: integer seq_no: - type: integer + $ref: '_common.yaml#/components/schemas/SequenceNumber' LeaderStatus: type: object properties: @@ -54,17 +59,17 @@ components: operations_read: type: number translog_size_bytes: - type: number + $ref: '_common.yaml#/components/schemas/ByteCount' operations_read_lucene: type: number operations_read_translog: type: number total_read_time_lucene_millis: - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' total_read_time_translog_millis: - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' bytes_read: - type: number + $ref: '_common.yaml#/components/schemas/ByteCount' index_stats: type: object additionalProperties: @@ -75,17 +80,17 @@ components: operations_read: type: number translog_size_bytes: - type: number + $ref: '_common.yaml#/components/schemas/ByteCount' operations_read_lucene: type: number operations_read_translog: type: number total_read_time_lucene_millis: - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' total_read_time_translog_millis: - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' bytes_read: - type: number + $ref: '_common.yaml#/components/schemas/ByteCount' FollowerStatus: type: object properties: @@ -118,7 +123,7 @@ components: leader_checkpoint: type: number total_write_time_millis: - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' index_stats: type: object additionalProperties: @@ -143,7 +148,7 @@ components: leader_checkpoint: type: number total_write_time_millis: - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' AutoFollowStatus: type: object properties: @@ -208,7 +213,7 @@ components: type: object properties: leader_alias: - type: string + $ref: '_common.yaml#/components/schemas/IndexAlias' name: type: string pattern: @@ -219,6 +224,6 @@ components: type: object properties: leader_alias: - type: string + $ref: '_common.yaml#/components/schemas/IndexAlias' name: type: string \ No newline at end of file From 229fb1900c205e650fedf111bbc717283942dd9e Mon Sep 17 00:00:00 2001 From: Tokesh Date: Mon, 18 Nov 2024 00:50:03 +0500 Subject: [PATCH 70/73] adding verbose to see tests Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 8bc1abeea..b175a9814 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -89,7 +89,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure \ + --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \ --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ From cca5c99f88aab5de5f5fc8dff7f8684262b78114 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Mon, 18 Nov 2024 00:54:13 +0500 Subject: [PATCH 71/73] adding enum type Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- spec/schemas/replication._common.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index b175a9814..8bc1abeea 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -89,7 +89,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure --verbose\ + --opensearch-insecure \ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \ --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 096172962..236a8219f 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -32,6 +32,7 @@ components: - PAUSED - REPLICATION NOT IN PROGRESS - SYNCING + - RUNNING reason: type: string leader_alias: From 91a61f82e2ba5b3fa59e60c7abe8d05c63d58059 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Mon, 18 Nov 2024 00:56:47 +0500 Subject: [PATCH 72/73] fixing lint Signed-off-by: Tokesh --- spec/schemas/replication._common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 236a8219f..ca02ab939 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -31,8 +31,8 @@ components: - BOOTSTRAPPING - PAUSED - REPLICATION NOT IN PROGRESS - - SYNCING - RUNNING + - SYNCING reason: type: string leader_alias: From 462975479172ea3a8afb158f3678e583a7afb23c Mon Sep 17 00:00:00 2001 From: dblock Date: Mon, 18 Nov 2024 08:51:21 -0500 Subject: [PATCH 73/73] Replaced anyOf properties by allOf. Signed-off-by: dblock --- spec/schemas/replication._common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index ca02ab939..e2b965b0e 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -195,7 +195,7 @@ components: type: object properties: settings: - anyOf: + allOf: - type: object properties: index.number_of_shards: