diff --git a/instrumentation/aws_sdk/Appraisals b/instrumentation/aws_sdk/Appraisals index 5ecdbf591..c9c9b908b 100644 --- a/instrumentation/aws_sdk/Appraisals +++ b/instrumentation/aws_sdk/Appraisals @@ -4,58 +4,27 @@ # # SPDX-License-Identifier: Apache-2.0 -appraise 'aws-sdk-3.1' do - gem 'aws-sdk', '~> 3.1' +appraise 'aws-sdk-3' do + gem 'aws-sdk-core', '~> 3' + gem 'aws-sdk-lambda', '~> 1' + gem 'aws-sdk-dynamodb', '~> 1' + gem 'aws-sdk-sns', '~> 1' + gem 'aws-sdk-sqs', '~> 1' end -appraise 'aws-sdk-3.0' do - gem 'aws-sdk', '~> 3.0' +# pre-Observability support in V3 SDK +appraise 'aws-sdk-3.202' do + gem 'aws-sdk-core', '~> 3.202' + gem 'aws-sdk-lambda', '~> 1.127' + gem 'aws-sdk-dynamodb', '~> 1.118' + gem 'aws-sdk-sns', '~> 1.82' + gem 'aws-sdk-sqs', '~> 1.80' end appraise 'aws-sdk-2.11' do gem 'aws-sdk', '~> 2.11' end -appraise 'aws-sdk-2.10' do - gem 'aws-sdk', '~> 2.10' -end - -appraise 'aws-sdk-2.9' do - gem 'aws-sdk', '~> 2.9' -end - -appraise 'aws-sdk-2.8' do - gem 'aws-sdk', '~> 2.8' -end - -appraise 'aws-sdk-2.7' do - gem 'aws-sdk', '~> 2.7' -end - -appraise 'aws-sdk-2.6' do - gem 'aws-sdk', '~> 2.6' -end - -appraise 'aws-sdk-2.5' do - gem 'aws-sdk', '~> 2.5' -end - -appraise 'aws-sdk-2.4' do - gem 'aws-sdk', '~> 2.4' -end - -appraise 'aws-sdk-2.3' do - gem 'aws-sdk', '~> 2.3' -end - -appraise 'aws-sdk-2.2' do - gem 'aws-sdk', '~> 2.2' -end - -appraise 'aws-sdk-2.1' do - gem 'aws-sdk', '~> 2.1' -end - appraise 'aws-sdk-2.0' do gem 'aws-sdk', '~> 2.0' end diff --git a/instrumentation/aws_sdk/README.md b/instrumentation/aws_sdk/README.md index e1885d7a9..fcf4de368 100644 --- a/instrumentation/aws_sdk/README.md +++ b/instrumentation/aws_sdk/README.md @@ -32,6 +32,30 @@ OpenTelemetry::SDK.configure do |c| c.use_all end ``` +### Configuration options +This instrumentation offers the following configuration options: +* `:inject_messaging_context` (default: `false`): When set to `true`, adds context key/value + to Message Attributes for SQS/SNS messages. +* `suppress_internal_instrumentation` (default: `false`): When set to `true`, any spans with + span kind of `internal` are suppressed from traces. + +## Integration with SDK V3's Telemetry support +AWS SDK for Ruby V3 added support for Observability which includes a new configuration, +`telemetry_provider` and an OpenTelemetry-based telemetry provider. Only applies to +AWS service gems released after 2024-09-03. + +Using later versions of these gems will give more details on the internal spans. +See below for example usage: +```ruby +# configures the OpenTelemetry SDK with instrumentation defaults +OpenTelemetry::SDK.configure do |c| + c.use 'OpenTelemetry::Instrumentation::AwsSdk' +end + +# create open-telemetry provider and pass to client config +otel_provider = Aws::Telemetry::OTelProvider.new +client = Aws::S3::Client.new(telemetry_provider: otel_provider) +``` ## Example diff --git a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler.rb b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler.rb index c0b040956..40ec604c7 100644 --- a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler.rb +++ b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler.rb @@ -7,26 +7,23 @@ module OpenTelemetry module Instrumentation module AwsSdk - # Generates Spans for all interactions with AwsSdk + # This handler supports specifically supports V2 and V3 + # prior to Observability support released on 2024-09-03. class Handler < Seahorse::Client::Handler def call(context) return super unless context - service_id = service_name(context) - operation = context.operation&.name - client_method = "#{service_id}.#{operation}" + service_id = HandlerHelper.service_id(context, legacy: true) + client_method = HandlerHelper.client_method(service_id, context) tracer.in_span( - span_name(context, client_method, service_id), - attributes: attributes(context, client_method, service_id, operation), - kind: span_kind(client_method, service_id) + HandlerHelper.span_name(context, client_method, service_id, legacy: true), + attributes: HandlerHelper.span_attributes(context, client_method, service_id, legacy: true), + kind: HandlerHelper.span_kind(client_method, service_id) ) do |span| - if instrumentation_config[:inject_messaging_context] && - %w[SQS SNS].include?(service_id) - MessagingHelper.inject_context(context, client_method) - end + MessagingHelper.inject_context_if_supported(context, client_method, service_id) - if instrumentation_config[:suppress_internal_instrumentation] + if HandlerHelper.instrumentation_config[:suppress_internal_instrumentation] OpenTelemetry::Common::Utilities.untraced { super } else super @@ -49,47 +46,6 @@ def call(context) def tracer AwsSdk::Instrumentation.instance.tracer end - - def instrumentation_config - AwsSdk::Instrumentation.instance.config - end - - def service_name(context) - # Support aws-sdk v2.0.x, which 'metadata' has a setter method only - return context.client.class.to_s.split('::')[1] if ::Seahorse::Model::Api.instance_method(:metadata).parameters.length.positive? - - context.client.class.api.metadata['serviceId'] || context.client.class.to_s.split('::')[1] - end - - def span_kind(client_method, service_id) - case service_id - when 'SQS', 'SNS' - MessagingHelper.span_kind(client_method) - else - OpenTelemetry::Trace::SpanKind::CLIENT - end - end - - def span_name(context, client_method, service_id) - case service_id - when 'SQS', 'SNS' - MessagingHelper.legacy_span_name(context, client_method) - else - client_method - end - end - - def attributes(context, client_method, service_id, operation) - { - 'aws.region' => context.config.region, - OpenTelemetry::SemanticConventions::Trace::RPC_SYSTEM => 'aws-api', - OpenTelemetry::SemanticConventions::Trace::RPC_METHOD => operation, - OpenTelemetry::SemanticConventions::Trace::RPC_SERVICE => service_id - }.tap do |attrs| - attrs[SemanticConventions::Trace::DB_SYSTEM] = 'dynamodb' if service_id == 'DynamoDB' - MessagingHelper.apply_span_attributes(context, attrs, client_method, service_id) if %w[SQS SNS].include?(service_id) - end - end end # A Seahorse::Client::Plugin that enables instrumentation for all AWS services diff --git a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler_helper.rb b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler_helper.rb new file mode 100644 index 000000000..9b5a01304 --- /dev/null +++ b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/handler_helper.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Instrumentation + module AwsSdk + # Utility module that contains shared methods between AwsSdk and Telemetry handlers + module HandlerHelper + class << self + def instrumentation_config + AwsSdk::Instrumentation.instance.config + end + + def client_method(service_id, context) + "#{service_id}.#{context.operation.name}".delete(' ') + end + + def span_attributes(context, client_method, service_id, legacy: false) + { + 'aws.region' => context.config.region, + OpenTelemetry::SemanticConventions::Trace::CODE_FUNCTION => context.operation_name.to_s, + OpenTelemetry::SemanticConventions::Trace::CODE_NAMESPACE => 'Aws::Plugins::Telemetry', + OpenTelemetry::SemanticConventions::Trace::RPC_METHOD => context.operation.name, + OpenTelemetry::SemanticConventions::Trace::RPC_SERVICE => service_id, + OpenTelemetry::SemanticConventions::Trace::RPC_SYSTEM => 'aws-api' + }.tap do |attrs| + attrs[OpenTelemetry::SemanticConventions::Trace::CODE_NAMESPACE] = 'Aws::Plugins::AwsSdk' if legacy + attrs[SemanticConventions::Trace::DB_SYSTEM] = 'dynamodb' if service_id == 'DynamoDB' + + MessagingHelper.apply_span_attributes(context, attrs, client_method, service_id) if MessagingHelper::SUPPORTED_SERVICES.include?(service_id) + end + end + + def span_kind(client_method, service_id) + case service_id + when *MessagingHelper::SUPPORTED_SERVICES + MessagingHelper.span_kind(client_method) + else + OpenTelemetry::Trace::SpanKind::CLIENT + end + end + + def span_name(context, client_method, service_id, legacy: false) + case service_id + when *MessagingHelper::SUPPORTED_SERVICES + if legacy + MessagingHelper.legacy_span_name(context, client_method) + else + MessagingHelper.span_name(context, client_method) + end + else + client_method + end + end + + def service_id(context, legacy: false) + if legacy + legacy_service_id(context) + else + context.config.api.metadata['serviceId'] || + context.config.api.metadata['serviceAbbreviation'] || + context.config.api.metadata['serviceFullName'] + end + end + + private + + def legacy_service_id(context) + # Support aws-sdk v2.0.x, which 'metadata' has a setter method only + return context.client.class.to_s.split('::')[1] if ::Seahorse::Model::Api.instance_method(:metadata).parameters.length.positive? + + context.client.class.api.metadata['serviceId'] || context.client.class.to_s.split('::')[1] + end + end + end + end + end +end diff --git a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/instrumentation.rb b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/instrumentation.rb index 928b9b98e..fe8651c25 100644 --- a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/instrumentation.rb +++ b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/instrumentation.rb @@ -7,12 +7,38 @@ module OpenTelemetry module Instrumentation module AwsSdk - # Instrumentation class that detects and installs the AwsSdk instrumentation + # The `OpenTelemetry::Instrumentation::AwsSdk::Instrumentation` class contains + # logic to detect and install the AwsSdk instrumentation. + # + # ## Configuration keys and options + # + # ### `:inject_messaging_context` + # + # Allows adding of context key/value to Message Attributes for SQS/SNS messages. + # + # - `false` **(default)** - Context key/value will not be added. + # - `true` - Context key/value will be added. + # + # ### `:suppress_internal_instrumentation` + # + # Disables tracing of spans of `internal` span kind. + # + # - `false` **(default)** - Internal spans are traced. + # - `true` - Internal spans are not traced. + # + # @example An explicit default configuration + # OpenTelemetry::SDK.configure do |c| + # c.use 'OpenTelemetry::Instrumentation::AwsSdk', { + # inject_messaging_context: false, + # suppress_internal_instrumentation: false + # } + # end class Instrumentation < OpenTelemetry::Instrumentation::Base MINIMUM_VERSION = Gem::Version.new('2.0.0') install do |_config| require_dependencies + patch_telemetry_plugin if telemetry_plugin? add_plugins(Seahorse::Client::Base, *loaded_service_clients) end @@ -41,12 +67,34 @@ def gem_version def require_dependencies require_relative 'handler' + require_relative 'handler_helper' require_relative 'message_attributes' require_relative 'messaging_helper' + require_relative 'patches/telemetry' end def add_plugins(*targets) - targets.each { |klass| klass.add_plugin(AwsSdk::Plugin) } + targets.each do |klass| + next if supports_telemetry_plugin?(klass) + + klass.add_plugin(AwsSdk::Plugin) + end + end + + def supports_telemetry_plugin?(klass) + telemetry_plugin? && + klass.plugins.include?(Aws::Plugins::Telemetry) + end + + def telemetry_plugin? + ::Aws::Plugins.const_defined?(:Telemetry) + end + + # Patches AWS SDK V3's telemetry plugin for integration + # This patch supports configuration set by this gem and + # additional span attributes that was not provided by the plugin + def patch_telemetry_plugin + ::Aws::Plugins::Telemetry::Handler.prepend(Patches::Handler) end def loaded_service_clients diff --git a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/messaging_helper.rb b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/messaging_helper.rb index 27064aa74..b821aca56 100644 --- a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/messaging_helper.rb +++ b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/messaging_helper.rb @@ -9,6 +9,7 @@ module Instrumentation module AwsSdk # An utility class to help SQS/SNS-related span attributes/context injection class MessagingHelper + SUPPORTED_SERVICES = %w[SQS SNS].freeze class << self SQS_SEND_MESSAGE = 'SQS.SendMessage' SQS_SEND_MESSAGE_BATCH = 'SQS.SendMessageBatch' @@ -16,6 +17,10 @@ class << self SNS_PUBLISH = 'SNS.Publish' SEND_MESSAGE_CLIENT_METHODS = [SQS_SEND_MESSAGE, SQS_SEND_MESSAGE_BATCH, SNS_PUBLISH].freeze + def supported_services + SUPPORTED_SERVICES + end + def queue_name(context) topic_arn = context.params[:topic_arn] target_arn = context.params[:target_arn] @@ -34,6 +39,17 @@ def queue_name(context) 'unknown' end + def span_name(context, client_method) + case client_method + when SQS_SEND_MESSAGE, SQS_SEND_MESSAGE_BATCH, SNS_PUBLISH + "#{client_method}.#{queue_name(context)}.Publish" + when SQS_RECEIVE_MESSAGE + "#{client_method}.#{queue_name(context)}.Receive" + else + client_method + end + end + def legacy_span_name(context, client_method) case client_method when SQS_SEND_MESSAGE, SQS_SEND_MESSAGE_BATCH, SNS_PUBLISH @@ -65,6 +81,13 @@ def span_kind(client_method) end end + def inject_context_if_supported(context, client_method, service_id) + if HandlerHelper.instrumentation_config[:inject_messaging_context] && + SUPPORTED_SERVICES.include?(service_id) + inject_context(context, client_method) + end + end + def inject_context(context, client_method) return unless SEND_MESSAGE_CLIENT_METHODS.include?(client_method) diff --git a/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/patches/telemetry.rb b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/patches/telemetry.rb new file mode 100644 index 000000000..31bb41c39 --- /dev/null +++ b/instrumentation/aws_sdk/lib/opentelemetry/instrumentation/aws_sdk/patches/telemetry.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Instrumentation + module AwsSdk + module Patches + # Patch for Telemetry Plugin Handler in V3 SDK + module Handler + def call(context) + span_wrapper(context) { @handler.call(context) } + end + + private + + def span_wrapper(context, &block) + service_id = HandlerHelper.service_id(context) + client_method = HandlerHelper.client_method(service_id, context) + context.tracer.in_span( + HandlerHelper.span_name(context, client_method, service_id), + attributes: HandlerHelper.span_attributes(context, client_method, service_id), + kind: HandlerHelper.span_kind(client_method, service_id) + ) do |span| + MessagingHelper.inject_context_if_supported(context, client_method, service_id) + + if HandlerHelper.instrumentation_config[:suppress_internal_instrumentation] + OpenTelemetry::Common::Utilities.untraced { super } + else + yield span + end + end + end + end + end + end + end +end diff --git a/instrumentation/aws_sdk/test/opentelemetry/handler_test.rb b/instrumentation/aws_sdk/test/opentelemetry/handler_test.rb new file mode 100644 index 000000000..8b178948c --- /dev/null +++ b/instrumentation/aws_sdk/test/opentelemetry/handler_test.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::Instrumentation::AwsSdk do + describe 'AwsSdk Plugin' do + let(:instrumentation_gem_version) do + OpenTelemetry::Instrumentation::AwsSdk::Instrumentation.instance.gem_version + end + let(:otel_semantic) { OpenTelemetry::SemanticConventions::Trace } + let(:exporter) { EXPORTER } + let(:span) { exporter.finished_spans.last } + let(:span_attrs) do + { + 'aws.region' => 'us-stubbed-1', + otel_semantic::HTTP_STATUS_CODE => 200, + otel_semantic::RPC_SYSTEM => 'aws-api' + } + end + + before do + exporter.reset + end + + describe 'Lambda' do + let(:service_name) { 'Lambda' } + let(:client) { Aws::Lambda::Client.new(stub_responses: true) } + let(:expected_attrs) do + span_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'ListFunctions' + attrs[otel_semantic::RPC_SERVICE] = service_name + end + end + + it 'creates a span with all the supplied parameters' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.list_functions + + _(span.name).must_equal('Lambda.ListFunctions') + _(span.kind).must_equal(:client) + TestHelper.match_span_attrs(expected_attrs, span, self) + end + + it 'should have correct span attributes when error' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.stub_responses(:list_functions, 'NotFound') + + begin + client.list_functions + rescue Aws::Lambda::Errors::NotFound + _(span.status.code).must_equal(2) + _(span.events[0].name).must_equal('exception') + _(span.attributes[otel_semantic::HTTP_STATUS_CODE]).must_equal(400) + end + end + end + + describe 'SNS' do + let(:service_name) { 'SNS' } + let(:client) { Aws::SNS::Client.new(stub_responses: true) } + let(:expected_attrs) do + span_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'Publish' + attrs[otel_semantic::RPC_SERVICE] = service_name + attrs[otel_semantic::MESSAGING_DESTINATION_KIND] = 'topic' + attrs[otel_semantic::MESSAGING_DESTINATION] = 'TopicName' + attrs[otel_semantic::MESSAGING_SYSTEM] = 'aws.sns' + end + end + + it 'creates a span with appropriate messaging attributes' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.publish( + message: 'msg', + topic_arn: 'arn:aws:sns:fake:123:TopicName' + ) + + _(span.name).must_equal('TopicName publish') + _(span.kind).must_equal(:producer) + TestHelper.match_span_attrs(expected_attrs, span, self) + end + + it 'creates a span that includes a phone number' do + # skip if using aws-sdk version before phone_number supported (v2.3.18) + skip if Gem::Version.new('2.3.18') > instrumentation_gem_version + skip if TestHelper.telemetry_plugin?(service_name) + + client.publish(message: 'msg', phone_number: '123456') + + _(span.name).must_equal('phone_number publish') + _(span.attributes[otel_semantic::MESSAGING_DESTINATION]) + .must_equal('phone_number') + end + end + + describe 'SQS' do + let(:service_name) { 'SQS' } + let(:client) { Aws::SQS::Client.new(stub_responses: true) } + let(:queue_url) { 'https://sqs.fake.amazonaws.com/1/QueueName' } + let(:expected_base_attrs) do + span_attrs.tap do |attrs| + attrs[otel_semantic::RPC_SERVICE] = service_name + attrs[otel_semantic::MESSAGING_DESTINATION_KIND] = 'queue' + attrs[otel_semantic::MESSAGING_DESTINATION] = 'QueueName' + attrs[otel_semantic::MESSAGING_SYSTEM] = 'aws.sqs' + attrs[otel_semantic::MESSAGING_URL] = queue_url + end + end + + describe '#SendMessage' do + let(:expected_attrs) do + span_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'SendMessage' + end + end + + it 'creates a span with appropriate messaging attributes' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.send_message(message_body: 'msg', queue_url: queue_url) + + _(span.name).must_equal('QueueName publish') + _(span.kind).must_equal(:producer) + TestHelper.match_span_attrs(expected_attrs, span, self) + end + end + + describe '#SendMessageBatch' do + let(:expected_attrs) do + expected_base_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'SendMessageBatch' + end + end + + it 'creates a span with appropriate messaging attributes' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.send_message_batch( + queue_url: queue_url, + entries: [{ id: 'Message1', message_body: 'Body1' }] + ) + + _(span.name).must_equal('QueueName publish') + _(span.kind).must_equal(:producer) + TestHelper.match_span_attrs(expected_attrs, span, self) + end + end + + describe '#ReceiveMessage' do + let(:expected_attrs) do + expected_base_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'ReceiveMessage' + attrs[otel_semantic::MESSAGING_OPERATION] = 'receive' + end + end + + it 'creates a span with appropriate messaging attributes' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.receive_message(queue_url: queue_url) + + _(span.name).must_equal('QueueName receive') + _(span.kind).must_equal(:consumer) + TestHelper.match_span_attrs(expected_attrs, span, self) + end + end + + describe '#GetQueueUrl' do + it 'creates a span with appropriate messaging attributes' do + skip if TestHelper.telemetry_plugin?(service_name) + + client.get_queue_url(queue_name: 'queue-name') + + _(span.attributes['messaging.destination']) + .must_equal('unknown') + _(span.attributes).wont_include('messaging.url') + end + end + end + + describe 'DynamoDB' do + let(:client) { Aws::DynamoDB::Client.new(stub_responses: true) } + + it 'creates a span with dynamodb-specific attribute' do + skip if TestHelper.telemetry_plugin?('DynamoDB') + + client.list_tables + + _(span.attributes[otel_semantic::DB_SYSTEM]) + .must_equal('dynamodb') + end + end + end +end diff --git a/instrumentation/aws_sdk/test/opentelemetry/instrumentation_test.rb b/instrumentation/aws_sdk/test/opentelemetry/instrumentation_test.rb index f44e81bc5..7ffe3d33f 100644 --- a/instrumentation/aws_sdk/test/opentelemetry/instrumentation_test.rb +++ b/instrumentation/aws_sdk/test/opentelemetry/instrumentation_test.rb @@ -9,8 +9,6 @@ describe OpenTelemetry::Instrumentation::AwsSdk do let(:instrumentation) { OpenTelemetry::Instrumentation::AwsSdk::Instrumentation.instance } let(:minimum_version) { OpenTelemetry::Instrumentation::AwsSdk::Instrumentation::MINIMUM_VERSION } - let(:exporter) { EXPORTER } - let(:last_span) { exporter.finished_spans.last } it 'has #name' do _(instrumentation.name).must_equal 'OpenTelemetry::Instrumentation::AwsSdk' @@ -28,12 +26,24 @@ _(instrumentation.compatible?).must_equal false end - Gem.stub(:loaded_specs, { 'aws-sdk-core' => nil, 'aws-sdk' => Gem::Specification.new { |s| s.version = '1.0.0' } }) do + Gem.stub( + :loaded_specs, + { + 'aws-sdk-core' => nil, + 'aws-sdk' => Gem::Specification.new { |s| s.version = '1.0.0' } + } + ) do hide_const('::Aws::CORE_GEM_VERSION') _(instrumentation.compatible?).must_equal false end - Gem.stub(:loaded_specs, { 'aws-sdk-core' => Gem::Specification.new { |s| s.version = '1.0.0' }, 'aws-sdk' => nil }) do + Gem.stub( + :loaded_specs, + { + 'aws-sdk-core' => Gem::Specification.new { |s| s.version = '1.0.0' }, + 'aws-sdk' => nil + } + ) do hide_const('::Aws::CORE_GEM_VERSION') _(instrumentation.compatible?).must_equal false end @@ -55,216 +65,4 @@ instrumentation.instance_variable_set(:@installed, false) end end - - describe 'validate_spans' do - describe 'SNS' do - it 'should have correct attributes' do - sns = Aws::SNS::Client.new(stub_responses: true) - sns.stub_responses(:publish) - - sns.publish message: 'msg' - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['rpc.service']).must_equal 'SNS' - _(last_span.attributes['rpc.method']).must_equal 'Publish' - _(last_span.attributes['aws.region']).must_include 'stubbed' - _(last_span.attributes['db.system']).must_be_nil - - _(last_span.attributes[OpenTelemetry::SemanticConventions::Trace::HTTP_STATUS_CODE]).must_equal 200 - - _(last_span.status.code).must_equal OpenTelemetry::Trace::Status::UNSET - end - end - - describe 'S3' do - it 'should have correct attributes when success' do - s3 = Aws::S3::Client.new(stub_responses: { list_buckets: { buckets: [{ name: 'bucket1' }] } }) - - s3.list_buckets - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['rpc.service']).must_equal 'S3' - _(last_span.attributes['rpc.method']).must_equal 'ListBuckets' - _(last_span.attributes['aws.region']).must_include 'stubbed' - _(last_span.attributes['db.system']).must_be_nil - - _(last_span.attributes[OpenTelemetry::SemanticConventions::Trace::HTTP_STATUS_CODE]).must_equal 200 - - _(last_span.status.code).must_equal OpenTelemetry::Trace::Status::UNSET - end - - it 'should have correct attributes when error' do - s3 = Aws::S3::Client.new(stub_responses: true) - s3.stub_responses(:list_buckets, 'NotFound') - - begin - s3.list_buckets - rescue StandardError - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['rpc.service']).must_equal 'S3' - _(last_span.attributes['rpc.method']).must_equal 'ListBuckets' - _(last_span.attributes['aws.region']).must_include 'stubbed' - _(last_span.attributes['db.system']).must_be_nil - end - - _(last_span.attributes[OpenTelemetry::SemanticConventions::Trace::HTTP_STATUS_CODE]).must_equal 400 - - _(last_span.status.code).must_equal OpenTelemetry::Trace::Status::ERROR - end - end - - describe 'dynamodb' do - it 'should have db.system attribute' do - dynamodb_client = Aws::DynamoDB::Client.new(stub_responses: true) - - dynamodb_client.list_tables - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['db.system']).must_equal 'dynamodb' - - _(last_span.attributes[OpenTelemetry::SemanticConventions::Trace::HTTP_STATUS_CODE]).must_equal 200 - end - end - - describe 'sqs' do - it 'should have messaging attributes for send_message' do - sqs_client = Aws::SQS::Client.new(stub_responses: true) - - sqs_client.send_message message_body: 'msg', queue_url: 'https://sqs.fake.amazonaws.com/1/queue-name' - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['messaging.system']).must_equal 'aws.sqs' - _(last_span.attributes['messaging.destination_kind']).must_equal 'queue' - _(last_span.attributes['messaging.destination']).must_equal 'queue-name' - _(last_span.attributes['messaging.url']).must_equal 'https://sqs.fake.amazonaws.com/1/queue-name' - end - - it 'should have messaging attributes for send_message_batch' do - sqs_client = Aws::SQS::Client.new(stub_responses: true) - - entries = [ - { - id: 'Message1', - message_body: 'This is the first message.' - }, - { - id: 'Message2', - message_body: 'This is the second message.', - message_attributes: { - attr1: { - data_type: 'String', - string_value: 'value1' - } - } - } - ] - - sqs_client.send_message_batch( - queue_url: 'https://sqs.fake.amazonaws.com/1/queue-name', - entries: entries - ) - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['messaging.system']).must_equal 'aws.sqs' - _(last_span.attributes['messaging.destination_kind']).must_equal 'queue' - _(last_span.attributes['messaging.destination']).must_equal 'queue-name' - _(last_span.attributes['messaging.url']).must_equal 'https://sqs.fake.amazonaws.com/1/queue-name' - end - - it 'should have messaging attributes for get_queue_url' do - sqs_client = Aws::SQS::Client.new(stub_responses: true) - - sqs_client.get_queue_url queue_name: 'queue-name' - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['messaging.system']).must_equal 'aws.sqs' - _(last_span.attributes['messaging.destination_kind']).must_equal 'queue' - _(last_span.attributes['messaging.destination']).must_equal 'unknown' - _(last_span.attributes).wont_include('messaging.url') - end - end - - describe 'sns' do - it 'should have messaging attributes for publish' do - sns_client = Aws::SNS::Client.new(stub_responses: true) - - sns_client.publish message: 'msg', topic_arn: 'arn:aws:sns:fake:123:topic-name' - - _(last_span.attributes['rpc.system']).must_equal 'aws-api' - _(last_span.attributes['messaging.system']).must_equal 'aws.sns' - _(last_span.attributes['messaging.destination_kind']).must_equal 'topic' - _(last_span.attributes['messaging.destination']).must_equal 'topic-name' - end - - it 'should handle phone numbers' do - # skip if using aws-sdk version before phone_number supported (v2.3.18) - return if Gem::Version.new('2.3.18') > instrumentation.gem_version - - sns_client = Aws::SNS::Client.new(stub_responses: true) - - sns_client.publish message: 'msg', phone_number: '123456' - - _(last_span.attributes['messaging.destination']).must_equal 'phone_number' - _(last_span.name).must_equal 'phone_number publish' - end - end - end - - describe 'MessageAttributeSetter' do - it 'set when hash length is lower than 10' do - key = 'foo' - value = 'bar' - metadata_attributes = {} - OpenTelemetry::Instrumentation::AwsSdk::MessageAttributeSetter.set(metadata_attributes, key, value) - _(metadata_attributes[key]).must_equal(string_value: value, data_type: 'String') - end - - it 'should keep existing attributes' do - key = 'foo' - value = 'bar' - metadata_attributes = { - 'existingKey' => { string_value: 'existingValue', data_type: 'String' } - } - OpenTelemetry::Instrumentation::AwsSdk::MessageAttributeSetter.set(metadata_attributes, key, value) - _(metadata_attributes[key]).must_equal(string_value: value, data_type: 'String') - _(metadata_attributes['existingKey']).must_equal(string_value: 'existingValue', data_type: 'String') - end - - it 'should not add if there are 10 or more existing attributes' do - metadata_attributes = { - 'existingKey0' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey1' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey2' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey3' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey4' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey5' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey6' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey7' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey8' => { string_value: 'existingValue', data_type: 'String' }, - 'existingKey9' => { string_value: 'existingValue', data_type: 'String' } - } - OpenTelemetry::Instrumentation::AwsSdk::MessageAttributeSetter.set(metadata_attributes, 'new10', 'value') - _(metadata_attributes.keys).must_equal(%w[existingKey0 existingKey1 existingKey2 existingKey3 existingKey4 existingKey5 existingKey6 existingKey7 existingKey8 existingKey9]) - end - - describe 'MessageAttributeGetter' do - let(:getter) { OpenTelemetry::Instrumentation::AwsSdk::MessageAttributeGetter } - let(:carrier) do - { - 'traceparent' => { data_type: 'String', string_value: 'tp' }, - 'tracestate' => { data_type: 'String', string_value: 'ts' }, - 'x-source-id' => { data_type: 'String', string_value: '123' } - } - end - - it 'reads key from carrier' do - _(getter.get(carrier, 'traceparent')).must_equal('tp') - _(getter.get(carrier, 'x-source-id')).must_equal('123') - end - - it 'returns nil for non-existant key' do - _(getter.get(carrier, 'not-here')).must_be_nil - end - end - end end diff --git a/instrumentation/aws_sdk/test/opentelemetry/message_attributes_test.rb b/instrumentation/aws_sdk/test/opentelemetry/message_attributes_test.rb new file mode 100644 index 000000000..1d3486399 --- /dev/null +++ b/instrumentation/aws_sdk/test/opentelemetry/message_attributes_test.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::Instrumentation::AwsSdk do + let(:instrumentation) { OpenTelemetry::Instrumentation::AwsSdk } + + describe 'MessageAttributeSetter' do + it 'set when hash length is lower than 10' do + key = 'foo' + value = 'bar' + metadata_attributes = {} + instrumentation::MessageAttributeSetter.set(metadata_attributes, key, value) + _(metadata_attributes[key]).must_equal(string_value: value, data_type: 'String') + end + + it 'should keep existing attributes' do + key = 'foo' + value = 'bar' + metadata_attributes = { + 'existingKey' => { string_value: 'existingValue', data_type: 'String' } + } + instrumentation::MessageAttributeSetter.set(metadata_attributes, key, value) + _(metadata_attributes[key]).must_equal(string_value: value, data_type: 'String') + _(metadata_attributes['existingKey']) + .must_equal(string_value: 'existingValue', data_type: 'String') + end + + it 'should not add if there are 10 or more existing attributes' do + metadata_attributes = { + 'existingKey0' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey1' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey2' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey3' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey4' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey5' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey6' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey7' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey8' => { string_value: 'existingValue', data_type: 'String' }, + 'existingKey9' => { string_value: 'existingValue', data_type: 'String' } + } + instrumentation::MessageAttributeSetter.set(metadata_attributes, 'new10', 'value') + _(metadata_attributes.keys) + .must_equal( + %w[ + existingKey0 + existingKey1 + existingKey2 + existingKey3 + existingKey4 + existingKey5 + existingKey6 + existingKey7 + existingKey8 + existingKey9 + ] + ) + end + end + + describe 'MessageAttributeGetter' do + let(:getter) { instrumentation::MessageAttributeGetter } + let(:carrier) do + { + 'traceparent' => { data_type: 'String', string_value: 'tp' }, + 'tracestate' => { data_type: 'String', string_value: 'ts' }, + 'x-source-id' => { data_type: 'String', string_value: '123' } + } + end + + it 'reads key from carrier' do + _(getter.get(carrier, 'traceparent')).must_equal('tp') + _(getter.get(carrier, 'x-source-id')).must_equal('123') + end + + it 'returns nil for non-existant key' do + _(getter.get(carrier, 'not-here')).must_be_nil + end + end +end diff --git a/instrumentation/aws_sdk/test/opentelemetry/patches/telemetry_test.rb b/instrumentation/aws_sdk/test/opentelemetry/patches/telemetry_test.rb new file mode 100644 index 000000000..55fd83886 --- /dev/null +++ b/instrumentation/aws_sdk/test/opentelemetry/patches/telemetry_test.rb @@ -0,0 +1,297 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative '../../test_helper' + +describe OpenTelemetry::Instrumentation::AwsSdk do + describe 'Telemetry plugin' do + let(:instrumentation_gem_version) do + OpenTelemetry::Instrumentation::AwsSdk::Instrumentation.instance.gem_version + end + let(:otel_semantic) { OpenTelemetry::SemanticConventions::Trace } + let(:exporter) { EXPORTER } + let(:spans) { exporter.finished_spans } + let(:otel_provider) { Aws::Telemetry::OTelProvider.new } + let(:stub_span) { spans.find { |s| s.name == 'Handler.StubResponses' } } + let(:client_attrs) do + { + 'aws.region' => 'us-stubbed-1', + otel_semantic::CODE_NAMESPACE => 'Aws::Plugins::Telemetry', + otel_semantic::RPC_SYSTEM => 'aws-api' + } + end + + let(:stub_attrs) do + { + 'http.status_code' => '200', + 'net.protocol.name' => 'http', + 'net.protocol.version' => '1.1' + } + end + + before do + exporter.reset + end + + describe 'Lambda' do + let(:service_name) { 'Lambda' } + let(:service_uri) do + 'https://lambda.us-east-1.amazonaws.com/2015-03-31/functions/' + end + let(:client) do + Aws::Lambda::Client.new( + telemetry_provider: otel_provider, + stub_responses: true + ) + end + let(:client_span) { spans.find { |s| s.name == 'Lambda.ListFunctions' } } + let(:internal_span) { spans.find { |s| s.name == 'Handler.NetHttp' } } + + let(:expected_client_attrs) do + client_attrs.tap do |attrs| + attrs[otel_semantic::CODE_FUNCTION] = 'list_functions' + attrs[otel_semantic::RPC_METHOD] = 'ListFunctions' + attrs[otel_semantic::RPC_SERVICE] = service_name + end + end + + let(:expected_stub_attrs) { stub_attrs.tap { |a| a['http.method'] = 'GET' } } + + let(:expected_internal_attrs) do + stub_attrs.tap do |attrs| + attrs['net.peer.name'] = 'lambda.us-east-1.amazonaws.com' + attrs['net.peer.port'] = '443' + end + end + + it 'creates spans with all the supplied parameters' do + skip unless TestHelper.telemetry_plugin?(service_name) + client.list_functions + + _(client_span.name).must_equal('Lambda.ListFunctions') + _(stub_span.name).must_equal('Handler.StubResponses') + _(client_span.kind).must_equal(:client) + _(stub_span.kind).must_equal(:internal) + TestHelper.match_span_attrs(expected_client_attrs, client_span, self) + TestHelper.match_span_attrs(expected_stub_attrs, stub_span, self) + _(stub_span.parent_span_id).must_equal(client_span.span_id) + end + + it 'creates spans with all the non-stubbed parameters' do + skip unless TestHelper.telemetry_plugin?(service_name) + stub_request(:get, 'https://lambda.us-east-1.amazonaws.com/2015-03-31/functions/') + + client = Aws::Lambda::Client.new( + telemetry_provider: otel_provider, + credentials: Aws::Credentials.new('akid', 'secret'), + region: 'us-east-1' + ) + client.list_functions + + _(client_span.name).must_equal('Lambda.ListFunctions') + _(internal_span.name).must_equal('Handler.NetHttp') + _(client_span.kind).must_equal(:client) + _(internal_span.kind).must_equal(:internal) + _(client_span.attributes['aws.region']).must_equal('us-east-1') + TestHelper.match_span_attrs(expected_internal_attrs, internal_span, self) + end + + it 'should have correct span attributes when error' do + skip unless TestHelper.telemetry_plugin?(service_name) + stub_request(:get, 'foo').to_return(status: 400) + + begin + client.list_functions + rescue Aws::Lambda::Errors::BadRequest + _(client_span.status.code).must_equal(2) + _(client_span.events[0].name).must_equal('exception') + _(internal_span.attributes['http.status_code']).must_equal('400') + end + end + end + + describe 'SNS' do + let(:service_name) { 'SNS' } + let(:client) do + Aws::SNS::Client.new( + telemetry_provider: otel_provider, + stub_responses: true + ) + end + let(:client_span) { spans.find { |s| s.name.include?('SNS.Publish') } } + + let(:expected_client_attrs) do + client_attrs.tap do |attrs| + attrs[otel_semantic::CODE_FUNCTION] = 'publish' + attrs[otel_semantic::RPC_METHOD] = 'Publish' + attrs[otel_semantic::RPC_SERVICE] = service_name + attrs[otel_semantic::MESSAGING_DESTINATION_KIND] = 'topic' + attrs[otel_semantic::MESSAGING_DESTINATION] = 'TopicName' + attrs[otel_semantic::MESSAGING_SYSTEM] = 'aws.sns' + end + end + + let(:expected_stub_attrs) { stub_attrs.tap { |a| a['http.method'] = 'POST' } } + + it 'creates spans with appropriate messaging attributes' do + skip unless TestHelper.telemetry_plugin?(service_name) + + client.publish( + message: 'msg', + topic_arn: 'arn:aws:sns:fake:123:TopicName' + ) + + _(client_span.name).must_equal('SNS.Publish.TopicName.Publish') + _(client_span.kind).must_equal(:producer) + _(stub_span.name).must_equal('Handler.StubResponses') + _(stub_span.kind).must_equal(:internal) + TestHelper.match_span_attrs(expected_client_attrs, client_span, self) + TestHelper.match_span_attrs(expected_stub_attrs, stub_span, self) + _(stub_span.parent_span_id).must_equal(client_span.span_id) + end + + it 'creates a span that includes a phone number' do + # skip if using aws-sdk version before phone_number supported (v2.3.18) + skip if Gem::Version.new('2.3.18') > instrumentation_gem_version + skip unless TestHelper.telemetry_plugin?(service_name) + + client.publish(message: 'msg', phone_number: '123456') + + _(client_span.name).must_equal('SNS.Publish.phone_number.Publish') + _(client_span.attributes[otel_semantic::MESSAGING_DESTINATION]) + .must_equal('phone_number') + end + end + + describe 'SQS' do + let(:service_name) { 'SQS' } + let(:client) do + Aws::SQS::Client.new( + telemetry_provider: otel_provider, + stub_responses: true + ) + end + let(:queue_url) { 'https://sqs.us-east-1.amazonaws.com/1/QueueName' } + let(:expected_client_base_attrs) do + client_attrs.tap do |attrs| + attrs[otel_semantic::RPC_SERVICE] = service_name + attrs[otel_semantic::MESSAGING_DESTINATION_KIND] = 'queue' + attrs[otel_semantic::MESSAGING_DESTINATION] = 'QueueName' + attrs[otel_semantic::MESSAGING_SYSTEM] = 'aws.sqs' + attrs[otel_semantic::MESSAGING_URL] = queue_url + end + end + + let(:expected_stub_attrs) { stub_attrs.tap { |a| a['http.method'] = 'POST' } } + + describe '#SendMessage' do + let(:client_span) { spans.find { |s| s.name.include?('SQS.SendMessage') } } + let(:expected_client_attrs) do + expected_client_base_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'SendMessage' + end + end + + it 'creates spans with appropriate messaging attributes' do + skip unless TestHelper.telemetry_plugin?(service_name) + + client.send_message(message_body: 'msg', queue_url: queue_url) + + _(client_span.name).must_equal('SQS.SendMessage.QueueName.Publish') + _(client_span.kind).must_equal(:producer) + _(stub_span.name).must_equal('Handler.StubResponses') + _(stub_span.kind).must_equal(:internal) + TestHelper.match_span_attrs(expected_client_attrs, client_span, self) + TestHelper.match_span_attrs(expected_stub_attrs, stub_span, self) + _(stub_span.parent_span_id).must_equal(client_span.span_id) + end + end + + describe '#SendMessageBatch' do + let(:client_span) { spans.find { |s| s.name.include?('SQS.SendMessageBatch') } } + let(:expected_client_attrs) do + expected_client_base_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'SendMessageBatch' + end + end + + it 'creates spans with appropriate messaging attributes' do + skip unless TestHelper.telemetry_plugin?(service_name) + + client.send_message_batch( + queue_url: queue_url, + entries: [{ id: 'Message1', message_body: 'Body1' }] + ) + + _(client_span.name).must_equal('SQS.SendMessageBatch.QueueName.Publish') + _(client_span.kind).must_equal(:producer) + _(stub_span.name).must_equal('Handler.StubResponses') + _(stub_span.kind).must_equal(:internal) + TestHelper.match_span_attrs(expected_client_attrs, client_span, self) + TestHelper.match_span_attrs(expected_stub_attrs, stub_span, self) + _(stub_span.parent_span_id).must_equal(client_span.span_id) + end + end + + describe '#ReceiveMessage' do + let(:client_span) { spans.find { |s| s.name.include?('SQS.ReceiveMessage') } } + let(:expected_client_attrs) do + expected_client_base_attrs.tap do |attrs| + attrs[otel_semantic::RPC_METHOD] = 'ReceiveMessage' + attrs[otel_semantic::MESSAGING_OPERATION] = 'receive' + end + end + + it 'creates spans with appropriate messaging attributes' do + skip unless TestHelper.telemetry_plugin?(service_name) + + client.receive_message(queue_url: queue_url) + + _(client_span.name).must_equal('SQS.ReceiveMessage.QueueName.Receive') + _(client_span.kind).must_equal(:consumer) + _(stub_span.name).must_equal('Handler.StubResponses') + _(stub_span.kind).must_equal(:internal) + TestHelper.match_span_attrs(expected_client_attrs, client_span, self) + TestHelper.match_span_attrs(expected_stub_attrs, stub_span, self) + _(stub_span.parent_span_id).must_equal(client_span.span_id) + end + end + + describe '#GetQueueUrl' do + let(:client_span) { spans.find { |s| s.name.include?('SQS.GetQueueUrl') } } + + it 'creates a span with appropriate messaging attributes' do + skip unless TestHelper.telemetry_plugin?(service_name) + + client.get_queue_url(queue_name: 'queue-name') + + _(client_span.attributes[otel_semantic::MESSAGING_DESTINATION]).must_equal('unknown') + _(client_span.attributes).wont_include(otel_semantic::MESSAGING_URL) + end + end + end + + describe 'DynamoDB' do + let(:client) do + Aws::DynamoDB::Client.new( + telemetry_provider: otel_provider, + stub_responses: true + ) + end + let(:client_span) { TestHelper.find_span(spans, 'DynamoDB.ListTables') } + let(:client_span) { spans.find { |s| s.name == 'DynamoDB.ListTables' } } + + it 'creates a span with dynamodb-specific attribute' do + skip unless TestHelper.telemetry_plugin?('DynamoDB') + + client.list_tables + + _(client_span.attributes[otel_semantic::DB_SYSTEM]) + .must_equal('dynamodb') + end + end + end +end diff --git a/instrumentation/aws_sdk/test/test_helper.rb b/instrumentation/aws_sdk/test/test_helper.rb index 7153e99b2..b588f8c75 100644 --- a/instrumentation/aws_sdk/test/test_helper.rb +++ b/instrumentation/aws_sdk/test/test_helper.rb @@ -10,6 +10,7 @@ require 'opentelemetry-instrumentation-aws_sdk' require 'minitest/autorun' +require 'webmock/minitest' require 'rspec/mocks/minitest_integration' # global opentelemetry-sdk setup: @@ -22,3 +23,19 @@ c.use 'OpenTelemetry::Instrumentation::AwsSdk' c.add_span_processor span_processor end + +class TestHelper + class << self + def telemetry_plugin?(service) + m = ::Aws.const_get(service).const_get(:Client) + Aws.const_defined?('Plugins::Telemetry') && + m.plugins.include?(Aws::Plugins::Telemetry) + end + + def match_span_attrs(expected_attrs, span, expect) + expected_attrs.each do |key, value| + expect._(span.attributes[key]).must_equal(value) + end + end + end +end