From 2a43b6a8705cc4e794da739378a7deb83249d8a5 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Thu, 18 Apr 2024 14:00:56 +0100 Subject: [PATCH 1/2] [Gem] Fixes ES|QL Helper When passing in a parser, if the value was nil, the helper would error. Now it returns nil for nil values. --- .../lib/elasticsearch/helpers/esql_helper.rb | 2 +- .../integration/helpers/esql_helper_spec.rb | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/elasticsearch/lib/elasticsearch/helpers/esql_helper.rb b/elasticsearch/lib/elasticsearch/helpers/esql_helper.rb index 9105850967..5ba26e20b1 100644 --- a/elasticsearch/lib/elasticsearch/helpers/esql_helper.rb +++ b/elasticsearch/lib/elasticsearch/helpers/esql_helper.rb @@ -61,7 +61,7 @@ def self.query(client, query, params = {}, parser: {}) response['values'].map do |value| (value.length - 1).downto(0).map do |index| key = columns[index]['name'] - value[index] = parser[key].call value[index] if parser[key] + value[index] = parser[key].call(value[index]) if value[index] && parser[key] { key => value[index] } end.reduce({}, :merge) end diff --git a/elasticsearch/spec/integration/helpers/esql_helper_spec.rb b/elasticsearch/spec/integration/helpers/esql_helper_spec.rb index 83355aba45..12122c7157 100644 --- a/elasticsearch/spec/integration/helpers/esql_helper_spec.rb +++ b/elasticsearch/spec/integration/helpers/esql_helper_spec.rb @@ -16,6 +16,7 @@ # under the License. require_relative 'helpers_spec_helper' require 'elasticsearch/helpers/esql_helper' +require 'ipaddr' context 'Elasticsearch client helpers' do let(:index) { 'esql_helper_test' } @@ -76,8 +77,6 @@ end it 'parses iterated objects when procs are passed in' do - require 'ipaddr' - parser = { '@timestamp' => Proc.new { |t| DateTime.parse(t) }, 'client.ip' => Proc.new { |i| IPAddr.new(i) }, @@ -91,4 +90,29 @@ expect(r['event.duration']).to be_a String end end + + it 'parser does not error when value is nil, leaves nil' do + client.index( + index: index, + body: { + '@timestamp' => nil, + 'client.ip' => nil, + message: 'Connected to 10.1.0.1', + 'event.duration' => 1756465 + }, + refresh: true + ) + parser = { + '@timestamp' => Proc.new { |t| DateTime.parse(t) }, + 'client.ip' => Proc.new { |i| IPAddr.new(i) }, + 'event.duration' => Proc.new { |d| d.to_s } + } + response = esql_helper.query(client, query, parser: parser) + response.each do |r| + expect [DateTime, NilClass].include?(r['@timestamp'].class) + expect [IPAddr, NilClass].include?(r['client.ip'].class) + expect(r['message']).to be_a String + expect(r['event.duration']).to be_a String + end + end end From 3b1977ae6da3e23cac1e5cc4ac808bdc296bcc81 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Thu, 18 Apr 2024 14:12:25 +0100 Subject: [PATCH 2/2] [Gem] Fixes Bulk Helper Addresses an issue where when using `slice`, the wrong hash was being passed again to `ingest` therefore adding unneccessary hash nesting. --- elasticsearch/lib/elasticsearch/helpers/bulk_helper.rb | 4 +++- elasticsearch/spec/integration/helpers/bulk_helper_spec.rb | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/elasticsearch/lib/elasticsearch/helpers/bulk_helper.rb b/elasticsearch/lib/elasticsearch/helpers/bulk_helper.rb index 80c906c38f..e5982eccb7 100644 --- a/elasticsearch/lib/elasticsearch/helpers/bulk_helper.rb +++ b/elasticsearch/lib/elasticsearch/helpers/bulk_helper.rb @@ -48,7 +48,9 @@ def initialize(client, index, params = {}) def ingest(docs, params = {}, body = {}, &block) ingest_docs = docs.map { |doc| { index: { _index: @index, data: doc} } } if (slice = params.delete(:slice)) - ingest_docs.each_slice(slice) { |items| ingest(items, params, &block) } + ingest_docs.each_slice(slice) do |items| + ingest(items.map { |item| item[:index][:data] }, params, &block) + end else bulk_request(ingest_docs, params, &block) end diff --git a/elasticsearch/spec/integration/helpers/bulk_helper_spec.rb b/elasticsearch/spec/integration/helpers/bulk_helper_spec.rb index 8fb7cb5485..31da9a0b84 100644 --- a/elasticsearch/spec/integration/helpers/bulk_helper_spec.rb +++ b/elasticsearch/spec/integration/helpers/bulk_helper_spec.rb @@ -21,6 +21,7 @@ context 'Elasticsearch client helpers' do context 'Bulk helper' do let(:index) { 'bulk_animals' } + let(:index_slice) { 'bulk_animals_slice' } let(:params) { { refresh: 'wait_for' } } let(:bulk_helper) { Elasticsearch::Helpers::BulkHelper.new(client, index, params) } let(:docs) do @@ -40,6 +41,7 @@ after do client.indices.delete(index: index, ignore: 404) + client.indices.delete(index: index_slice, ignore: 404) end it 'Ingests documents' do @@ -76,10 +78,13 @@ it 'Ingests documents and yields response and docs' do slice = 2 + bulk_helper = Elasticsearch::Helpers::BulkHelper.new(client, index_slice, params) response = bulk_helper.ingest(docs, {slice: slice}) do |response, docs| expect(response).to be_an_instance_of Elasticsearch::API::Response expect(docs.count).to eq slice end + response = client.search(index: index_slice, size: 200) + expect(response['hits']['hits'].map { |a| a['_source'].transform_keys(&:to_sym) }).to eq docs end context 'JSON File helper' do