diff --git a/lib/new_relic/agent.rb b/lib/new_relic/agent.rb index cc93bdd18f..02a91595b0 100644 --- a/lib/new_relic/agent.rb +++ b/lib/new_relic/agent.rb @@ -62,6 +62,7 @@ module Agent require 'new_relic/agent/attribute_processing' require 'new_relic/agent/linking_metadata' require 'new_relic/agent/local_log_decorator' + require 'new_relic/agent/llm' require 'new_relic/agent/instrumentation/controller_instrumentation' diff --git a/lib/new_relic/agent/llm.rb b/lib/new_relic/agent/llm.rb new file mode 100644 index 0000000000..3057d635a1 --- /dev/null +++ b/lib/new_relic/agent/llm.rb @@ -0,0 +1,11 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative 'llm/llm_event' +require_relative 'llm/chat_completion' +require_relative 'llm/chat_completion_message' +require_relative 'llm/chat_completion_summary' +require_relative 'llm/embedding' +require_relative 'llm/feedback' +require_relative 'llm/response_headers' diff --git a/lib/new_relic/agent/llm/chat_completion.rb b/lib/new_relic/agent/llm/chat_completion.rb new file mode 100644 index 0000000000..ca24221135 --- /dev/null +++ b/lib/new_relic/agent/llm/chat_completion.rb @@ -0,0 +1,15 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +module NewRelic + module Agent + module Llm + module ChatCompletion + ATTRIBUTES = %i[conversation_id] + + attr_accessor(*ATTRIBUTES) + end + end + end +end diff --git a/lib/new_relic/agent/llm/chat_completion_message.rb b/lib/new_relic/agent/llm/chat_completion_message.rb new file mode 100644 index 0000000000..eca258e1ad --- /dev/null +++ b/lib/new_relic/agent/llm/chat_completion_message.rb @@ -0,0 +1,26 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +module NewRelic + module Agent + module Llm + class ChatCompletionMessage < LlmEvent + include ChatCompletion + + ATTRIBUTES = %i[content role sequence completion_id is_response] + EVENT_NAME = 'LlmChatCompletionMessage' + + attr_accessor(*ATTRIBUTES) + + def attributes + LlmEvent::ATTRIBUTES + ChatCompletion::ATTRIBUTES + ATTRIBUTES + end + + def event_name + EVENT_NAME + end + end + end + end +end diff --git a/lib/new_relic/agent/llm/chat_completion_summary.rb b/lib/new_relic/agent/llm/chat_completion_summary.rb new file mode 100644 index 0000000000..15fa09e493 --- /dev/null +++ b/lib/new_relic/agent/llm/chat_completion_summary.rb @@ -0,0 +1,33 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative 'response_headers' + +module NewRelic + module Agent + module Llm + class ChatCompletionSummary < LlmEvent + include ChatCompletion + include ResponseHeaders + + ATTRIBUTES = %i[api_key_last_four_digits request_max_tokens + response_number_of_messages request_model response_organization + response_usage_total_tokens response_usage_prompt_tokens + response_usage_completion_tokens response_choices_finish_reason + request_temperature duration error] + EVENT_NAME = 'LlmChatCompletionSummary' + + attr_accessor(*ATTRIBUTES) + + def attributes + LlmEvent::ATTRIBUTES + ChatCompletion::ATTRIBUTES + ResponseHeaders::ATTRIBUTES + ATTRIBUTES + end + + def event_name + EVENT_NAME + end + end + end + end +end diff --git a/lib/new_relic/agent/llm/embedding.rb b/lib/new_relic/agent/llm/embedding.rb new file mode 100644 index 0000000000..773831211f --- /dev/null +++ b/lib/new_relic/agent/llm/embedding.rb @@ -0,0 +1,28 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +module NewRelic + module Agent + module Llm + class Embedding < LlmEvent + include ResponseHeaders + + ATTRIBUTES = %i[input api_key_last_four_digits request_model + response_organization response_usage_total_tokens + response_usage_prompt_tokens duration error] + EVENT_NAME = 'LlmEmbedding' + + attr_accessor(*ATTRIBUTES) + + def attributes + LlmEvent::ATTRIBUTES + ResponseHeaders::ATTRIBUTES + ATTRIBUTES + end + + def event_name + EVENT_NAME + end + end + end + end +end diff --git a/lib/new_relic/agent/llm/feedback.rb b/lib/new_relic/agent/llm/feedback.rb new file mode 100644 index 0000000000..225a398eb0 --- /dev/null +++ b/lib/new_relic/agent/llm/feedback.rb @@ -0,0 +1,12 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +module NewRelic + module Agent + module Llm + class Feedback < LlmEvent + end + end + end +end diff --git a/lib/new_relic/agent/llm/llm_event.rb b/lib/new_relic/agent/llm/llm_event.rb new file mode 100644 index 0000000000..7869e7fa70 --- /dev/null +++ b/lib/new_relic/agent/llm/llm_event.rb @@ -0,0 +1,59 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +module NewRelic + module Agent + module Llm + class LlmEvent + # Every subclass must define its own ATTRIBUTES constant, an array of symbols representing + # that class's unique attributes + ATTRIBUTES = %i[id request_id span_id transaction_id + trace_id response_model vendor ingest_source] + # These attributes should not be passed as arguments to initialize and will be set by the agent + AGENT_DEFINED_ATTRIBUTES = %i[span_id transaction_id trace_id ingest_source] + INGEST_SOURCE = 'Ruby' + + attr_accessor(*ATTRIBUTES) + + # This initialize method is used for all subclasses. + # It leverages the subclass's `attributes` method to iterate through + # all the attributes for that subclass. + # It assigns instance variables for all arguments passed to the method. + # It also assigns agent-defined attributes. + def initialize(opts = {}) + (attributes - AGENT_DEFINED_ATTRIBUTES).each do |attr| + instance_variable_set(:"@#{attr}", opts[attr]) if opts.key?(attr) + end + + @span_id = NewRelic::Agent::Tracer.current_span_id + @transaction_id = NewRelic::Agent::Tracer.current_transaction&.guid + @trace_id = NewRelic::Agent::Tracer.current_trace_id + @ingest_source = INGEST_SOURCE + end + + # All subclasses use event_attributes to get a full hash of all + # attributes and their values + def event_attributes + attributes.each_with_object({}) do |attr, hash| + hash[attr] = instance_variable_get(:"@#{attr}") + end + end + + # Subclasses define an attributes method to concatenate attributes + # defined across their ancestors and other modules + def attributes + ATTRIBUTES + end + + # Subclasses that record events will override this method + def event_name + end + + def record + NewRelic::Agent.record_custom_event(event_name, event_attributes) + end + end + end + end +end diff --git a/lib/new_relic/agent/llm/response_headers.rb b/lib/new_relic/agent/llm/response_headers.rb new file mode 100644 index 0000000000..892b7a1cf4 --- /dev/null +++ b/lib/new_relic/agent/llm/response_headers.rb @@ -0,0 +1,36 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +module NewRelic + module Agent + module Llm + module ResponseHeaders + ATTRIBUTES = %i[llm_version rate_limit_requests rate_limit_tokens + rate_limit_remaining_requests rate_limit_remaining_tokens + rate_limit_reset_requests rate_limit_reset_tokens] + + OPENAI_VERSION = 'openai-version' + X_RATELIMIT_LIMIT_REQUESTS = 'x-ratelimit-limit-requests' + X_RATELIMIT_LIMIT_TOKENS = 'x-ratelimit-limit-tokens' + X_RATELIMIT_REMAINING_REQUESTS = 'x-ratelimit-remaining-requests' + X_RATELIMIT_REMAINING_TOKENS = 'x-ratelimit-remaining-tokens' + X_RATELIMIT_RESET_REQUESTS = 'x-ratelimit-reset-requests' + X_RATELIMIT_RESET_TOKENS = 'x-ratelimit-reset-tokens' + + attr_accessor(*ATTRIBUTES) + + # Headers is a hash of Net::HTTP response headers + def populate_openai_response_headers(headers) + self.llm_version = headers[OPENAI_VERSION]&.first + self.rate_limit_requests = headers[X_RATELIMIT_LIMIT_REQUESTS]&.first + self.rate_limit_tokens = headers[X_RATELIMIT_LIMIT_TOKENS]&.first + self.rate_limit_remaining_requests = headers[X_RATELIMIT_REMAINING_REQUESTS]&.first + self.rate_limit_remaining_tokens = headers[X_RATELIMIT_REMAINING_TOKENS]&.first + self.rate_limit_reset_requests = headers[X_RATELIMIT_RESET_REQUESTS]&.first + self.rate_limit_reset_tokens = headers[X_RATELIMIT_RESET_TOKENS]&.first + end + end + end + end +end diff --git a/test/new_relic/agent/llm/chat_completion_message_test.rb b/test/new_relic/agent/llm/chat_completion_message_test.rb new file mode 100644 index 0000000000..7695b6fc91 --- /dev/null +++ b/test/new_relic/agent/llm/chat_completion_message_test.rb @@ -0,0 +1,90 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class ChatCompletionMessageTest < Minitest::Test + def test_attributes_assigned_by_parent_present + assert_includes NewRelic::Agent::Llm::ChatCompletionMessage.ancestors, NewRelic::Agent::Llm::LlmEvent + assert_includes NewRelic::Agent::Llm::LlmEvent::AGENT_DEFINED_ATTRIBUTES, :transaction_id + + in_transaction do |txn| + event = NewRelic::Agent::Llm::ChatCompletionMessage.new + + assert_equal txn.guid, event.transaction_id + end + end + + def test_attributes_in_parent_list_can_be_assigned_on_init + assert_includes NewRelic::Agent::Llm::LlmEvent::ATTRIBUTES, :id + + event = NewRelic::Agent::Llm::ChatCompletionMessage.new(id: 123) + + assert_equal 123, event.id + end + + def test_included_module_attributes_list_can_be_assigned_on_init + assert_includes NewRelic::Agent::Llm::ChatCompletionMessage.ancestors, NewRelic::Agent::Llm::ChatCompletion + assert_includes NewRelic::Agent::Llm::ChatCompletion::ATTRIBUTES, :conversation_id + + conversation_id = '123abc' + event = NewRelic::Agent::Llm::ChatCompletionMessage.new(conversation_id: conversation_id) + + assert_equal conversation_id, event.conversation_id + end + + def test_attributes_constant_values_can_be_passed_as_args_and_set_on_init + assert_includes NewRelic::Agent::Llm::ChatCompletionMessage::ATTRIBUTES, :role + role = 'user' + event = NewRelic::Agent::Llm::ChatCompletionMessage.new(role: role) + + assert_equal role, event.role + end + + def test_args_passed_to_init_not_set_as_instance_vars_when_not_in_attributes_constant + event = NewRelic::Agent::Llm::ChatCompletionMessage.new(fake: 'fake') + + refute_includes event.attributes, :fake + refute event.instance_variable_defined?(:@fake) + end + + def test_record_creates_an_event + in_transaction do |txn| + message = NewRelic::Agent::Llm::ChatCompletionMessage.new( + id: 7, content: 'Red-Tailed Hawk' + ) + message.sequence = 2 + message.conversation_id = 25 + message.request_id = '789' + message.response_model = 'gpt-4' + message.vendor = 'OpenAI' + message.role = 'system' + message.completion_id = 123 + message.is_response = 'true' + + message.record + _, events = NewRelic::Agent.agent.custom_event_aggregator.harvest! + type, attributes = events[0] + + assert_equal 'LlmChatCompletionMessage', type['type'] + + assert_equal 7, attributes['id'] + assert_equal 25, attributes['conversation_id'] + assert_equal '789', attributes['request_id'] + assert_equal txn.current_segment.guid, attributes['span_id'] + assert_equal txn.guid, attributes['transaction_id'] + assert_equal txn.trace_id, attributes['trace_id'] + assert_equal 'gpt-4', attributes['response_model'] + assert_equal 'OpenAI', attributes['vendor'] + assert_equal 'Ruby', attributes['ingest_source'] + assert_equal 'Red-Tailed Hawk', attributes['content'] + assert_equal 'system', attributes['role'] + assert_equal 2, attributes['sequence'] + assert_equal 123, attributes['completion_id'] + assert_equal 'true', attributes['is_response'] + end + end + end +end diff --git a/test/new_relic/agent/llm/chat_completion_summary_test.rb b/test/new_relic/agent/llm/chat_completion_summary_test.rb new file mode 100644 index 0000000000..f496346a4e --- /dev/null +++ b/test/new_relic/agent/llm/chat_completion_summary_test.rb @@ -0,0 +1,119 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class ChatCompletionSummaryTest < Minitest::Test + def test_attributes_assigned_by_parent_present + assert_includes NewRelic::Agent::Llm::ChatCompletionSummary.ancestors, NewRelic::Agent::Llm::LlmEvent + assert_includes NewRelic::Agent::Llm::LlmEvent::AGENT_DEFINED_ATTRIBUTES, :transaction_id + + in_transaction do |txn| + event = NewRelic::Agent::Llm::ChatCompletionSummary.new + + assert_equal txn.guid, event.transaction_id + end + end + + def test_attributes_in_parent_list_can_be_assigned_on_init + assert_includes NewRelic::Agent::Llm::LlmEvent::ATTRIBUTES, :id + + event = NewRelic::Agent::Llm::ChatCompletionSummary.new(id: 123) + + assert_equal 123, event.id + end + + def test_included_module_attributes_list_can_be_assigned_on_init + assert_includes NewRelic::Agent::Llm::ChatCompletionSummary.ancestors, NewRelic::Agent::Llm::ChatCompletion + assert_includes NewRelic::Agent::Llm::ChatCompletion::ATTRIBUTES, :conversation_id + + conversation_id = '123abc' + event = NewRelic::Agent::Llm::ChatCompletionSummary.new(conversation_id: conversation_id) + + assert_equal conversation_id, event.conversation_id + end + + def test_attributes_constant_values_can_be_passed_as_args_and_set_on_init + assert_includes NewRelic::Agent::Llm::ChatCompletionSummary::ATTRIBUTES, :request_model + request_model = 'gpt-4-turbo-preview' + event = NewRelic::Agent::Llm::ChatCompletionSummary.new(request_model: request_model) + + assert_equal request_model, event.request_model + end + + def test_args_passed_to_init_not_set_as_instance_vars_when_not_in_attributes_constant + event = NewRelic::Agent::Llm::ChatCompletionSummary.new(fake: 'fake') + + refute_includes event.attributes, :fake + refute event.instance_variable_defined?(:@fake) + end + + def test_record_creates_an_event + in_transaction do |txn| + summary = NewRelic::Agent::Llm::ChatCompletionSummary.new( + id: 123, + request_model: 'gpt-4-turbo-preview', + api_key_last_four_digits: 'sk-0713' + ) + summary.request_id = '789' + summary.conversation_id = 456 + summary.response_usage_total_tokens = 20 + summary.request_max_tokens = 500 + summary.response_number_of_messages = 5 + summary.request_model = 'gpt-4-turbo-preview' + summary.response_model = 'gpt-4' + summary.response_organization = '98338' + summary.response_usage_total_tokens = 20 + summary.response_usage_prompt_tokens = '24' + summary.response_usage_completion_tokens = '26' + summary.response_choices_finish_reason = 'stop' + summary.vendor = 'OpenAI' + summary.duration = '500' + summary.error = 'true' + summary.llm_version = '2022-01-01' + summary.rate_limit_requests = '100' + summary.rate_limit_tokens = '101' + summary.rate_limit_reset_tokens = '102' + summary.rate_limit_reset_requests = '103' + summary.rate_limit_remaining_tokens = '104' + summary.rate_limit_remaining_requests = '105' + + summary.record + _, events = NewRelic::Agent.agent.custom_event_aggregator.harvest! + type, attributes = events[0] + + assert_equal 'LlmChatCompletionSummary', type['type'] + + assert_equal 123, attributes['id'] + assert_equal 456, attributes['conversation_id'] + assert_equal '789', attributes['request_id'] + assert_equal txn.current_segment.guid, attributes['span_id'] + assert_equal txn.guid, attributes['transaction_id'] + assert_equal txn.trace_id, attributes['trace_id'] + assert_equal 'sk-0713', attributes['api_key_last_four_digits'] + assert_equal 500, attributes['request_max_tokens'] + assert_equal 5, attributes['response_number_of_messages'] + assert_equal 'gpt-4-turbo-preview', attributes['request_model'] + assert_equal 'gpt-4', attributes['response_model'] + assert_equal '98338', attributes['response_organization'] + assert_equal 20, attributes['response_usage_total_tokens'] + assert_equal '24', attributes['response_usage_prompt_tokens'] + assert_equal '26', attributes['response_usage_completion_tokens'] + assert_equal 'stop', attributes['response_choices_finish_reason'] + assert_equal 'OpenAI', attributes['vendor'] + assert_equal 'Ruby', attributes['ingest_source'] + assert_equal '500', attributes['duration'] + assert_equal 'true', attributes['error'] + assert_equal '2022-01-01', attributes['llm_version'] + assert_equal '100', attributes['rate_limit_requests'] + assert_equal '101', attributes['rate_limit_tokens'] + assert_equal '102', attributes['rate_limit_reset_tokens'] + assert_equal '103', attributes['rate_limit_reset_requests'] + assert_equal '104', attributes['rate_limit_remaining_tokens'] + assert_equal '105', attributes['rate_limit_remaining_requests'] + end + end + end +end diff --git a/test/new_relic/agent/llm/chat_completion_test.rb b/test/new_relic/agent/llm/chat_completion_test.rb new file mode 100644 index 0000000000..0bfbb39b37 --- /dev/null +++ b/test/new_relic/agent/llm/chat_completion_test.rb @@ -0,0 +1,13 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class ChatCompletionTest < Minitest::Test + def test_attributes_include_conversation_id + assert_includes ChatCompletion::ATTRIBUTES, :conversation_id + end + end +end diff --git a/test/new_relic/agent/llm/embedding_test.rb b/test/new_relic/agent/llm/embedding_test.rb new file mode 100644 index 0000000000..e10eceadca --- /dev/null +++ b/test/new_relic/agent/llm/embedding_test.rb @@ -0,0 +1,95 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class EmbeddingTest < Minitest::Test + def test_attributes_assigned_by_parent_present + assert_includes NewRelic::Agent::Llm::Embedding.ancestors, NewRelic::Agent::Llm::LlmEvent + assert_includes NewRelic::Agent::Llm::LlmEvent::AGENT_DEFINED_ATTRIBUTES, :transaction_id + + in_transaction do |txn| + event = NewRelic::Agent::Llm::Embedding.new + + assert_equal txn.guid, event.transaction_id + end + end + + def test_attributes_in_parent_list_can_be_assigned_on_init + assert_includes NewRelic::Agent::Llm::LlmEvent::ATTRIBUTES, :id + + event = NewRelic::Agent::Llm::Embedding.new(id: 123) + + assert_equal 123, event.id + end + + def test_attributes_constant_values_can_be_passed_as_args_and_set_on_init + assert_includes NewRelic::Agent::Llm::Embedding::ATTRIBUTES, :input + input = 'Salut!' + event = NewRelic::Agent::Llm::Embedding.new(input: input) + + assert_equal input, event.input + end + + def test_args_passed_to_init_not_set_as_instance_vars_when_not_in_attributes_constant + event = NewRelic::Agent::Llm::Embedding.new(fake: 'fake') + + refute_includes event.attributes, :fake + refute event.instance_variable_defined?(:@fake) + end + + def test_record_creates_an_event + in_transaction do |txn| + embedding = NewRelic::Agent::Llm::Embedding.new(input: 'Bonjour', request_model: 'text-embedding-ada-002', id: 123) + embedding.request_id = '789' + embedding.api_key_last_four_digits = 'sk-0126' + embedding.response_model = 'text-embedding-3-large' + embedding.response_organization = '98338' + embedding.response_usage_total_tokens = '20' + embedding.response_usage_prompt_tokens = '24' + embedding.vendor = 'OpenAI' + embedding.duration = '500' + embedding.error = 'true' + embedding.llm_version = '2022-01-01' + embedding.rate_limit_requests = '100' + embedding.rate_limit_tokens = '101' + embedding.rate_limit_reset_tokens = '102' + embedding.rate_limit_reset_requests = '103' + embedding.rate_limit_remaining_tokens = '104' + embedding.rate_limit_remaining_requests = '105' + + embedding.record + _, events = NewRelic::Agent.agent.custom_event_aggregator.harvest! + type, attributes = events[0] + + assert_equal 'LlmEmbedding', type['type'] + + assert_equal 123, attributes['id'] + assert_equal '789', attributes['request_id'] + assert_equal txn.current_segment.guid, attributes['span_id'] + assert_equal txn.guid, attributes['transaction_id'] + assert_equal txn.trace_id, attributes['trace_id'] + assert_equal 'Bonjour', attributes['input'] + assert_equal 'sk-0126', attributes['api_key_last_four_digits'] + assert_equal 'text-embedding-ada-002', attributes['request_model'] + assert_equal 'text-embedding-3-large', attributes['response_model'] + assert_equal '98338', attributes['response_organization'] + assert_equal '20', attributes['response_usage_total_tokens'] + assert_equal '24', attributes['response_usage_prompt_tokens'] + assert_equal 'OpenAI', attributes['vendor'] + assert_equal 'Ruby', attributes['ingest_source'] + assert_equal '500', attributes['duration'] + assert_equal 'true', attributes['error'] + assert_equal '2022-01-01', attributes['llm_version'] + assert_equal '100', attributes['rate_limit_requests'] + assert_equal '101', attributes['rate_limit_tokens'] + assert_equal '102', attributes['rate_limit_reset_tokens'] + assert_equal '103', attributes['rate_limit_reset_requests'] + assert_equal '104', attributes['rate_limit_remaining_tokens'] + assert_equal '105', attributes['rate_limit_remaining_requests'] + end + end + end +end diff --git a/test/new_relic/agent/llm/feedback_test.rb b/test/new_relic/agent/llm/feedback_test.rb new file mode 100644 index 0000000000..680d94ecbb --- /dev/null +++ b/test/new_relic/agent/llm/feedback_test.rb @@ -0,0 +1,10 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class FeedbackTest < Minitest::Test + end +end diff --git a/test/new_relic/agent/llm/llm_event_test.rb b/test/new_relic/agent/llm/llm_event_test.rb new file mode 100644 index 0000000000..c69c121724 --- /dev/null +++ b/test/new_relic/agent/llm/llm_event_test.rb @@ -0,0 +1,51 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class LlmEventTest < Minitest::Test + def test_agent_defined_attributes_set + assert_includes NewRelic::Agent::Llm::LlmEvent::AGENT_DEFINED_ATTRIBUTES, :transaction_id + + in_transaction do |txn| + event = NewRelic::Agent::Llm::LlmEvent.new(transaction_id: 'fake') + + refute_equal 'fake', event.transaction_id + assert_equal txn.guid, event.transaction_id + end + end + + def test_attributes_constant_values_can_be_passed_as_args_and_set_on_init + id = 123 + event = NewRelic::Agent::Llm::LlmEvent.new(id: id) + + assert_equal id, event.id + end + + def test_args_passed_to_init_not_set_as_instance_vars_when_not_in_attributes_constant + event = NewRelic::Agent::Llm::LlmEvent.new(fake: 'fake') + + refute event.instance_variable_defined?(:@fake) + end + + def test_event_attributes_returns_a_hash_of_assigned_attributes_and_values + event = NewRelic::Agent::Llm::LlmEvent.new(id: 123) + event.vendor = 'OpenAI' + result = event.event_attributes + + assert_equal(result.keys, NewRelic::Agent::Llm::LlmEvent::ATTRIBUTES) + assert_equal(123, result[:id]) + assert_equal('OpenAI', result[:vendor]) + end + + def test_record_does_not_create_an_event + event = NewRelic::Agent::Llm::LlmEvent.new + event.record + _, events = NewRelic::Agent.agent.custom_event_aggregator.harvest! + + assert_empty events + end + end +end diff --git a/test/new_relic/agent/llm/response_headers_test.rb b/test/new_relic/agent/llm/response_headers_test.rb new file mode 100644 index 0000000000..478ca21f7c --- /dev/null +++ b/test/new_relic/agent/llm/response_headers_test.rb @@ -0,0 +1,54 @@ +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. +# frozen_string_literal: true + +require_relative '../../../test_helper' + +module NewRelic::Agent::Llm + class ResponseHeadersTest < Minitest::Test + def openai_response_headers_hash + # Response headers from a real OpenAI request + # rubocop:disable Style/StringLiterals, Style/WordArray + {"date" => ["Thu, 25 Jan 2024 23:16:44 GMT"], + "content-type" => ["application/json"], + "transfer-encoding" => ["chunked"], + "connection" => ["keep-alive"], + "access-control-allow-origin" => ["*"], + "openai-model" => ["text-embedding-ada-002"], + "openai-organization" => + ["user-whatever"], + "openai-processing-ms" => ["22"], + "openai-version" => ["2020-10-01"], + "strict-transport-security" => + ["max-age=15724800; includeSubDomains"], + "x-ratelimit-limit-requests" => ["200"], + "x-ratelimit-limit-tokens" => ["150000"], + "x-ratelimit-remaining-requests" => ["199"], + "x-ratelimit-remaining-tokens" => ["149990"], + "x-ratelimit-reset-requests" => ["7m12s"], + "x-ratelimit-reset-tokens" => ["4ms"], + "x-request-id" => ["123abc456"], + "cf-cache-status" => ["DYNAMIC"], + "set-cookie" => + ["we-dont-use-this-value", + "we-dont-use-this-value"], + "server" => ["cloudflare"], + "cf-ray" => ["123abc-SJC"], + "alt-svc" => ["h3=\":443\"; ma=86400"]} + # rubocop:enable Style/StringLiterals, Style/WordArray + end + + def test_populate_openai_response_headers + event = NewRelic::Agent::Llm::ChatCompletionSummary.new + event.populate_openai_response_headers(openai_response_headers_hash) + + assert_equal '2020-10-01', event.llm_version + assert_equal '200', event.rate_limit_requests + assert_equal '150000', event.rate_limit_tokens + assert_equal '199', event.rate_limit_remaining_requests + assert_equal '149990', event.rate_limit_remaining_tokens + assert_equal '7m12s', event.rate_limit_reset_requests + assert_equal '4ms', event.rate_limit_reset_tokens + end + end +end