From 3180161768e13431b70c075374783e95d6e713d7 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Thu, 25 Jul 2024 11:18:32 -0400 Subject: [PATCH] feat: Add `compress_events` option to support gzipping event payload (#290) --- contract-tests/client_entity.rb | 1 + contract-tests/service.rb | 1 + lib/ldclient-rb/config.rb | 20 ++++++++++++ lib/ldclient-rb/impl/event_sender.rb | 15 ++++++--- spec/http_util.rb | 12 ++++--- spec/impl/event_sender_spec.rb | 49 +++++++++++++++++++++------- 6 files changed, 78 insertions(+), 20 deletions(-) diff --git a/contract-tests/client_entity.rb b/contract-tests/client_entity.rb index 9c840912..203ac923 100644 --- a/contract-tests/client_entity.rb +++ b/contract-tests/client_entity.rb @@ -36,6 +36,7 @@ def initialize(log, config) opts[:private_attributes] = events[:globalPrivateAttributes] opts[:flush_interval] = (events[:flushIntervalMs] / 1_000) unless events[:flushIntervalMs].nil? opts[:omit_anonymous_contexts] = !!events[:omitAnonymousContexts] + opts[:compress_events] = !!events[:enableGzip] else opts[:send_events] = false end diff --git a/contract-tests/service.rb b/contract-tests/service.rb index dc5705e5..7db6e0f9 100644 --- a/contract-tests/service.rb +++ b/contract-tests/service.rb @@ -35,6 +35,7 @@ 'tags', 'migrations', 'event-gzip', + 'optional-event-gzip', 'event-sampling', 'context-comparison', 'polling-gzip', diff --git a/lib/ldclient-rb/config.rb b/lib/ldclient-rb/config.rb index 9de83df0..9fb0d8a4 100644 --- a/lib/ldclient-rb/config.rb +++ b/lib/ldclient-rb/config.rb @@ -65,6 +65,7 @@ def initialize(opts = {}) @all_attributes_private = opts[:all_attributes_private] || false @private_attributes = opts[:private_attributes] || [] @send_events = opts.has_key?(:send_events) ? opts[:send_events] : Config.default_send_events + @compress_events = opts.has_key?(:compress_events) ? opts[:compress_events] : Config.default_compress_events @context_keys_capacity = opts[:context_keys_capacity] || Config.default_context_keys_capacity @context_keys_flush_interval = opts[:context_keys_flush_interval] || Config.default_context_keys_flush_interval @data_source = opts[:data_source] @@ -254,6 +255,17 @@ def offline? # attr_reader :send_events + # + # Should the event payload sent to LaunchDarkly use gzip compression. By default this is false to prevent backward + # breaking compatibility issues with older versions of the relay proxy. + # + # Customers not using the relay proxy are strongly encouraged to enable this feature to reduce egress bandwidth + # cost. + # + # @return [Boolean] + # + attr_reader :compress_events + # # The number of context keys that the event processor can remember at any one time. This reduces the # amount of duplicate context details sent in analytics events. @@ -539,6 +551,14 @@ def self.default_send_events true end + # + # The default value for {#compress_events}. + # @return [Boolean] false + # + def self.default_compress_events + false + end + # # The default value for {#context_keys_capacity}. # @return [Integer] 1000 diff --git a/lib/ldclient-rb/impl/event_sender.rb b/lib/ldclient-rb/impl/event_sender.rb index 4fcdaf13..c730b04a 100644 --- a/lib/ldclient-rb/impl/event_sender.rb +++ b/lib/ldclient-rb/impl/event_sender.rb @@ -44,17 +44,24 @@ def send_event_data(event_data, description, is_diagnostic) @logger.debug { "[LDClient] sending #{description}: #{event_data}" } headers = {} headers["content-type"] = "application/json" - headers["content-encoding"] = "gzip" + headers["content-encoding"] = "gzip" if @config.compress_events Impl::Util.default_http_headers(@sdk_key, @config).each { |k, v| headers[k] = v } unless is_diagnostic headers["X-LaunchDarkly-Event-Schema"] = CURRENT_SCHEMA_VERSION.to_s headers["X-LaunchDarkly-Payload-ID"] = payload_id end - gzip = Zlib::GzipWriter.new(StringIO.new) - gzip << event_data + + body = event_data + if @config.compress_events + gzip = Zlib::GzipWriter.new(StringIO.new) + gzip << event_data + + body = gzip.close.string + end + response = http_client.request("POST", uri, { headers: headers, - body: gzip.close.string, + body: body, }) rescue StandardError => exn @logger.warn { "[LDClient] Error sending events: #{exn.inspect}." } diff --git a/spec/http_util.rb b/spec/http_util.rb index ffc318b0..a52e4bb6 100644 --- a/spec/http_util.rb +++ b/spec/http_util.rb @@ -9,8 +9,9 @@ class StubHTTPServer @@next_port = 50000 - def initialize + def initialize(enable_compression: false) @port = StubHTTPServer.next_port + @enable_compression = enable_compression begin base_opts = { BindAddress: '127.0.0.1', @@ -77,11 +78,14 @@ def record_request(req, res) def await_request_with_body r = @requests_queue.pop + request = r[0] body = r[1] + return [request, body.to_s] unless @enable_compression + gz = Zlib::GzipReader.new(StringIO.new(body.to_s)) - [r[0], gz.read] + [request, gz.read] end end @@ -91,8 +95,8 @@ def method_missing(*) end end -def with_server(server = nil) - server = StubHTTPServer.new if server.nil? +def with_server(enable_compression: false) + server = StubHTTPServer.new(enable_compression: enable_compression) begin server.start yield server diff --git a/spec/impl/event_sender_spec.rb b/spec/impl/event_sender_spec.rb index dd48fcb4..e6a971bd 100644 --- a/spec/impl/event_sender_spec.rb +++ b/spec/impl/event_sender_spec.rb @@ -11,24 +11,48 @@ module Impl subject { EventSender } let(:sdk_key) { "sdk_key" } - let(:fake_data) { '{"things":[]}' } + let(:fake_data) { '{"things":[],"stuff":false,"other examples":["you", "me", "us", "we"]}' } - def make_sender(server) - make_sender_with_events_uri(server.base_uri.to_s) + def make_sender(config_options = {}) + config_options = {logger: $null_log}.merge(config_options) + subject.new(sdk_key, Config.new(config_options), nil, 0.1) end - def make_sender_with_events_uri(events_uri) - subject.new(sdk_key, Config.new(events_uri: events_uri, logger: $null_log, application: {id: "id", version: "version"}), nil, 0.1) + def with_sender_and_server(config_options = {}) + enable_compression = config_options[:compress_events] || false + with_server(enable_compression: enable_compression) do |server| + config_options[:events_uri] = server.base_uri.to_s + yield make_sender(config_options), server + end end - def with_sender_and_server - with_server do |server| - yield make_sender(server), server + it "sends analytics event data without compression enabled" do + with_sender_and_server(compress_events: false) do |es, server| + server.setup_ok_response("/bulk", "") + + result = es.send_event_data(fake_data, "", false) + + expect(result.success).to be true + expect(result.must_shutdown).to be false + expect(result.time_from_server).not_to be_nil + + req, body = server.await_request_with_body + expect(body).to eq fake_data + expect(req.header).to include({ + "authorization" => [ sdk_key ], + "content-type" => [ "application/json" ], + "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ], + "x-launchdarkly-event-schema" => [ "4" ], + "connection" => [ "Keep-Alive" ], + }) + expect(req.header['x-launchdarkly-payload-id']).not_to eq [] + expect(req.header['content-encoding']).to eq [] + expect(req.header['content-length'][0].to_i).to eq fake_data.length end end - it "sends analytics event data" do - with_sender_and_server do |es, server| + it "sends analytics event data with compression enabled" do + with_sender_and_server(compress_events: true) do |es, server| server.setup_ok_response("/bulk", "") result = es.send_event_data(fake_data, "", false) @@ -41,13 +65,14 @@ def with_sender_and_server expect(body).to eq fake_data expect(req.header).to include({ "authorization" => [ sdk_key ], + "content-encoding" => [ "gzip" ], "content-type" => [ "application/json" ], "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ], "x-launchdarkly-event-schema" => [ "4" ], - "x-launchdarkly-tags" => [ "application-id/id application-version/version" ], "connection" => [ "Keep-Alive" ], }) expect(req.header['x-launchdarkly-payload-id']).not_to eq [] + expect(req.header['content-length'][0].to_i).to be > fake_data.length end end @@ -123,7 +148,7 @@ def with_sender_and_server begin ENV["http_proxy"] = proxy.base_uri.to_s - es = make_sender_with_events_uri(fake_target_uri) + es = make_sender(events_uri: fake_target_uri) result = es.send_event_data(fake_data, "", false)