From 2c38fe396ab12b048e606a74eb234a71cb732785 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 30 Jan 2024 16:09:51 +0000 Subject: [PATCH 1/3] [CI] Updates 8.x versions in stack test matrices --- .buildkite/pipeline.yml | 4 ++-- .github/workflows/7.17-8.x.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b917923a08..f92f91aa23 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -29,11 +29,11 @@ steps: skip: true # Compatibility tests for 8.x: - with: - stack_version: "8.10-SNAPSHOT" + stack_version: "8.12-SNAPSHOT" suite: "free" ruby: "3.2" - with: - stack_version: "8.10-SNAPSHOT" + stack_version: "8.12-SNAPSHOT" suite: "platinum" ruby: "3.2" command: ./.buildkite/run-tests.sh diff --git a/.github/workflows/7.17-8.x.yml b/.github/workflows/7.17-8.x.yml index b6fa9ee5d4..dd1d995f14 100644 --- a/.github/workflows/7.17-8.x.yml +++ b/.github/workflows/7.17-8.x.yml @@ -26,7 +26,7 @@ jobs: sudo sysctl -w vm.max_map_count=262144 - uses: elastic/elastic-github-actions/elasticsearch@master with: - stack-version: 8.9-SNAPSHOT + stack-version: 8.12-SNAPSHOT security-enabled: false - uses: ruby/setup-ruby@v1 with: From 1ad5da6a688abce6e67d55eb546810e4e8bc64ca Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Wed, 31 Jan 2024 08:25:10 +0000 Subject: [PATCH 2/3] [XPACK] Skips index data stream test --- elasticsearch-xpack/spec/skipped_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticsearch-xpack/spec/skipped_tests.yml b/elasticsearch-xpack/spec/skipped_tests.yml index c214069ffc..5a7f05d501 100644 --- a/elasticsearch-xpack/spec/skipped_tests.yml +++ b/elasticsearch-xpack/spec/skipped_tests.yml @@ -37,3 +37,6 @@ - :file: 'ml/categorization_agg.yml' :description: 'Test categorization aggregation with poor settings' +- + :file: 'data_stream/80_resolve_index_data_streams.yml' + :description: 'Resolve index with hidden and closed indices' From 221acebef565ab1d553d5481df11c2eca0b6d578 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 30 Jan 2024 16:43:11 +0000 Subject: [PATCH 3/3] [API] Test Runner: Updates wipe cluster for 8.x --- api-spec-testing/wipe_cluster.rb | 2 + api-spec-testing/wipe_cluster_8.rb | 107 ++++++++++++++++------------- 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/api-spec-testing/wipe_cluster.rb b/api-spec-testing/wipe_cluster.rb index 8b0e32780c..3bb7c8356a 100644 --- a/api-spec-testing/wipe_cluster.rb +++ b/api-spec-testing/wipe_cluster.rb @@ -175,6 +175,8 @@ def clear_templates_xpack(client) ].freeze def xpack_template?(template) + return true if template.include?('@') + xpack_prefixes = [ '.monitoring', '.watch', '.triggered-watches', '.data-frame', '.ml-', '.transform', 'data-streams-mappings', 'elastic-connectors' diff --git a/api-spec-testing/wipe_cluster_8.rb b/api-spec-testing/wipe_cluster_8.rb index 33b0b26180..844bdaeb66 100644 --- a/api-spec-testing/wipe_cluster_8.rb +++ b/api-spec-testing/wipe_cluster_8.rb @@ -23,10 +23,13 @@ module RestAPIYAMLTests module WipeCluster8 PRESERVE_ILM_POLICY_IDS = [ 'ilm-history-ilm-policy', 'slm-history-ilm-policy', 'watch-history-ilm-policy', - 'ml-size-based-ilm-policy', 'logs', 'metrics', 'synthetics', '7-days-default', - '30-days-default', '90-days-default', '180-days-default', '365-days-default', - '.fleet-actions-results-ilm-policy', '.deprecation-indexing-ilm-policy', - 'watch-history-ilm-policy-16', '.monitoring-8-ilm-policy' + 'watch-history-ilm-policy-16', 'ml-size-based-ilm-policy', 'logs', 'metrics', 'profiling', + 'synthetics', '7-days-default', '30-days-default', '90-days-default', '180-days-default', + '365-days-default', '.fleet-files-ilm-policy', '.fleet-file-data-ilm-policy', + '.fleet-actions-results-ilm-policy', '.fleet-file-fromhost-data-ilm-policy', + '.fleet-file-fromhost-meta-ilm-policy', '.fleet-file-tohost-data-ilm-policy', + '.fleet-file-tohost-meta-ilm-policy', '.deprecation-indexing-ilm-policy', + '.monitoring-8-ilm-policy', 'behavioral_analytics-events-default_policy' ].freeze PLATINUM_TEMPLATES = [ @@ -37,9 +40,7 @@ module WipeCluster8 'synthetics', 'synthetics-settings', 'synthetics-mappings', '.snapshot-blob-cache', '.deprecation-indexing-template', '.deprecation-indexing-mappings', '.deprecation-indexing-settings', - 'behavioral_analytics-events-mappings', 'behavioral_analytics-events-settings', - 'security-index-template', 'data-streams-mappings', 'ecs@dynamic_templates', - 'search-acl-filter' + 'security-index-template', 'data-streams-mappings', 'search-acl-filter' ].freeze # Wipe Cluster, based on PHP's implementation of ESRestTestCase.java:wipeCluster() @@ -51,6 +52,13 @@ def self.run(client) check_for_unexpectedly_recreated_objects(client) end + def self.create_xpack_rest_user(client) + client.security.put_user( + username: 'x_pack_rest_user', + body: { password: 'x-pack-test-password', roles: ['superuser'] } + ) + end + class << self private @@ -58,7 +66,7 @@ def wipe_cluster(client) read_plugins(client) if @has_rollups wipe_rollup_jobs(client) - wait_for_pending_rollup_tasks(client) + # wait_for_pending_rollup_tasks(client) end delete_all_slm_policies(client) wipe_searchable_snapshot_indices(client) if @has_xpack @@ -71,12 +79,13 @@ def wipe_cluster(client) clear_ml_jobs(client) clear_datafeeds(client) delete_data_frame_analytics(client) - delete_filters(client) - clear_transforms(client) + clear_ml_filters(client) end delete_all_ilm_policies(client) if @has_ilm delete_all_follow_patterns(client) if @has_ccr delete_all_node_shutdown_metadata(client) + clear_tasks(client) + clear_transforms(client) wipe_calendars(client) end @@ -86,7 +95,7 @@ def ensure_no_initializing_shards(client) def check_for_unexpectedly_recreated_objects(client) unexpected_ilm_policies = client.ilm.get_lifecycle - unexpected_ilm_policies.reject! { |k, _| PRESERVE_ILM_POLICY_IDS.include? k } + unexpected_ilm_policies.reject! { |k, _| preserve_policy?(k) } unless unexpected_ilm_policies.empty? logger.info( "Expected no ILM policies after deletions, but found #{unexpected_ilm_policies.keys.join(',')}" @@ -97,12 +106,12 @@ def check_for_unexpectedly_recreated_objects(client) templates = client.indices.get_index_template unexpected_templates = templates['index_templates'].reject do |t| # reject platinum templates - PLATINUM_TEMPLATES.include? t['name'] + platinum_template?(t['name']) end.map { |t| t['name'] } # only keep the names legacy_templates = client.indices.get_template - unexpected_templates << legacy_templates.keys.reject { |t| PLATINUM_TEMPLATES.include? t } + unexpected_templates << legacy_templates.keys.reject { |t| platinum_template?(t) } - unless unexpected_templates.empty? + unless unexpected_templates.reject(&:empty?).empty? logger.info( "Expected no templates after deletions, but found #{unexpected_templates.join(',')}" ) @@ -136,19 +145,19 @@ def wipe_rollup_jobs(client) def wait_for_pending_rollup_tasks(client) filter = 'xpack/rollup/job' + start_time = Time.now.to_i + count = 0 loop do results = client.cat.tasks(detailed: true).split("\n") - count = 0 - time = Time.now.to_i results.each do |task| - next if task.empty? + next if task.empty? || skippable_task?(task) || task.include?(filter) - logger.debug("Pending task: #{task}") - count += 1 if task.include?(filter) + count += 1 end - break unless count.positive? && Time.now.to_i < (time + 30) + break unless count.positive? && Time.now.to_i < (start_time + 1) end + logger.debug("Waited for #{count} pending rollup tasks for #{Time.now.to_i - start_time}s.") if count.positive? end def delete_all_slm_policies(client) @@ -229,7 +238,7 @@ def wipe_all_templates(client) # Always check for legacy templates templates = client.indices.get_template templates.each do |name, _| - next if platinum_template? name + next if platinum_template?(name) begin client.indices.delete_template(name: name) @@ -240,35 +249,51 @@ def wipe_all_templates(client) end def platinum_template?(template) + return true if template.include?('@') + platinum_prefixes = [ - '.monitoring', '.watch', '.triggered-watches', '.data-frame', '.ml-', '.transform', - 'data-streams-mappings', 'elastic-connectors' + '.monitoring', '.watch', '.triggered-watches', '.data-frame', '.ml-', + '.transform', '.deprecation', 'data-streams-mappings', '.fleet', + 'behavioral_analytics-', 'profiling', 'elastic-connectors', 'ilm-history', '.slm-history' ].freeze - platinum_prefixes.map { |a| return true if (a.include?(template) || template.start_with?(a)) } + return true if template.start_with?(*platinum_prefixes) PLATINUM_TEMPLATES.include? template end + def preserve_policy?(policy) + PRESERVE_ILM_POLICY_IDS.include?(policy) || policy.include?('@') + end + def wait_for_cluster_tasks(client) - time = Time.now.to_i + start_time = Time.now.to_i count = 0 - loop do results = client.cluster.pending_tasks results['tasks'].each do |task| - next if task.empty? + next if task.empty? || skippable_task?(task) - logger.debug "Pending cluster task: #{task}" count += 1 end - break unless count.positive? && Time.now.to_i < (time + 10) + break unless count.positive? && Time.now.to_i < (start_time + 5) + end + logger.debug("Waited for #{count} pending cluster tasks for #{Time.now.to_i - start_time}s.") if count.positive? + end + + def skippable_task?(task) + names = ['health-node', 'cluster:monitor/tasks/lists', 'create-index-template-v2', + 'remove-component-template'] + if task.is_a?(String) + names.select { |n| task.match? n }.any? + elsif task.is_a?(Hash) + names.select { |n| task['source'].match? n }.any? end end def delete_all_ilm_policies(client) policies = client.ilm.get_lifecycle policies.each do |policy| - client.ilm.delete_lifecycle(policy: policy[0]) unless PRESERVE_ILM_POLICY_IDS.include? policy[0] + client.ilm.delete_lifecycle(policy: policy[0]) unless preserve_policy?(policy[0]) end end @@ -294,13 +319,6 @@ def delete_all_follow_patterns(client) end end - def create_xpack_rest_user(client) - client.security.put_user( - username: 'x_pack_rest_user', - body: { password: 'x-pack-test-password', roles: ['superuser'] } - ) - end - def clear_roles(client) client.security.get_role.each do |role, _| begin; client.security.delete_role(name: role); rescue; end @@ -376,13 +394,6 @@ def delete_all_node_shutdown_metadata(client) end end - def wipe_calendars(client) - calendars = client.ml.get_calendars(calendar_id: '_all')['calendars'] - calendars.each do |calendar| - client.ml.delete_calendar(calendar_id: calendar['calendar_id']) - end - end - def delete_data_frame_analytics(client) dfs = client.ml.get_data_frame_analytics return unless dfs['data_frame_analytics'] @@ -392,10 +403,10 @@ def delete_data_frame_analytics(client) end end - def delete_filters(client) - filters = client.ml.get_filters - filters['filters'].each do |filter| - client.ml.delete_filter(filter_id: filter['filter_id']) + def wipe_calendars(client) + calendars = client.ml.get_calendars(calendar_id: '_all')['calendars'] + calendars.each do |calendar| + client.ml.delete_calendar(calendar_id: calendar['calendar_id']) end end end