chore: re-enable compilation checks, update otel and protobuf (#2254)

* chore: re-enable compilation checks, update otel and protobuf

* chore: fix failing tests

* fix: update grpc server startup config

* feat: consolidate otel trace transform module into a Logs.Processor

* chore: compilation warnings

* chore: fix failing tests

.github/workflows/elixir-ci.yml

@@ -33,8 +33,8 @@ jobs:
             run: mix lint --only warnings
           - name: Code Quality - Formatting
             run: mix test.format
-          # - name: Compilation Warnings
-          #   run: mix test.compile
+          - name: Compilation Warnings
+            run: mix test.compile
           - name: Tests
             run: mix do ecto.create, ecto.migrate, test
           # - name: Security - Sobelow Code Scan

.tool-versions

@@ -2,3 +2,4 @@ elixir 1.17.3-otp-27
 nodejs 18.13.0
 rust 1.64.0
 erlang 27.1.2
+protoc 28.0

DEVELOPMENT.md

@@ -2,7 +2,8 @@
 
 ## Dev Env Setup
 
-1. Install dependencies with `asdf` using `asdf install`
+1. Run `make setup`, which will:
+   1. Install dependencies with `asdf` using `asdf install`
 2. Decrypt secrets with `make decrypt.dev`. It will decrypt two files:
    1. Dev secrets - `.dev.env`
    2. Google JWT key - `.gcloud.json`

Makefile

@@ -23,6 +23,14 @@ test.only:
 .PHONY: test test.only
 
 setup: setup.node
+	# install dependencies
+	asdf install
+
+	# add protobuf install
+	mix escript.install hex protobuf
+
+	asdf reshim
+	# run elixir setup
 	mix setup
 
 setup.node:

config/config.exs

@@ -125,8 +125,6 @@ config :open_api_spex, :cache_adapter, OpenApiSpex.Plug.PersistentTermCache
 
 config :logflare, Logflare.Cluster.Utils, min_cluster_size: 1
 
-config :grpc, start_server: true
-
 config :logflare, Logflare.AlertsScheduler, init_task: {Logflare.Alerting, :init_alert_jobs, []}
 
 config :opentelemetry,

config/test.exs

@@ -34,7 +34,6 @@ config :logflare, Logflare.Repo,
   pool: Ecto.Adapters.SQL.Sandbox
 
 config :logflare, :postgres_backend_adapter, pool_size: 1
-config :grpc, start_server: false
 
 defmodule LogflareTest.LogFilters do
   def ignore_finch_disconnections(%{meta: %{mfa: {Finch.HTTP2.Pool, :disconnected, _}}}, _opts) do

lib/logflare/application.ex

@@ -104,7 +104,8 @@ defmodule Logflare.Application do
          name: Logflare.Source.V1SourceDynSup},
         LogflareWeb.Endpoint,
         # If we get a log event and the Source.Supervisor is not up it will 500
-        {GRPC.Server.Supervisor, {LogflareGrpc.Endpoint, grpc_port, cred: grpc_creds}},
+        {GRPC.Server.Supervisor,
+         endpoint: LogflareGrpc.Endpoint, port: grpc_port, cred: grpc_creds, start_server: true},
         # Monitor system level metrics
         SystemMetricsSup,
 

lib/logflare_grpc/trace/transform.ex → lib/logflare/logs/otel_trace.ex

@@ -1,35 +1,28 @@
-defmodule LogflareGrpc.Trace.Transform do
+defmodule Logflare.Logs.OtelTrace do
   @moduledoc """
-  Converts a GRPC Trace to a Logflare Event
-  """
-
-  alias Opentelemetry.Proto.Trace.V1.ResourceSpans
-  alias Opentelemetry.Proto.Common.V1.KeyValue
-  alias Logflare.LogEvent
-  alias Logflare.Source
+  Converts a GRPC Trace to event parameters
 
-  @doc """
   Converts a list of ResourceSpans to a list of Logflare Events
   Important details about the conversation:
   * One ResourceSpans can contain multiple ScopeSpans
   * One ScopeSpans can contain multiple Spans
   * One Span can contain multiple Events
   * A Log Event is created for each Span and each Event
+
   """
-  @spec to_log_events(list(ResourceSpans.t()), Source.t()) :: list(LogEvent.t())
-  def to_log_events(resource_spans, source) do
-    resource_spans
-    |> Enum.map(&to_log_event(&1, source))
-    |> List.flatten()
-  end
+  require Logger
+
+  alias Opentelemetry.Proto.Trace.V1.ResourceSpans
+  alias Opentelemetry.Proto.Common.V1.KeyValue
+
+  @behaviour Logflare.Logs.Processor
 
-  defp to_log_event(%ResourceSpans{} = span, source) do
-    span
-    |> handle_resource_span()
-    |> Enum.map(&LogEvent.make(&1, %{source: source}))
+  def handle_batch(resource_spans, _source) when is_list(resource_spans) do
+    Enum.map(resource_spans, &handle_event/1)
+    |> List.flatten()
   end
 
-  defp handle_resource_span(%{resource: resource, scope_spans: scope_spans}) do
+  defp handle_event(%ResourceSpans{resource: resource, scope_spans: scope_spans}) do
     resource = handle_resource(resource)
 
     scope_spans
@@ -64,7 +57,7 @@ defmodule LogflareGrpc.Trace.Transform do
 
     [
       %{
-        "message" => span.name,
+        "event_message" => span.name,
         "metadata" => metadata,
         "span_id" => Base.encode16(span.span_id),
         "parent_span_id" => Base.encode16(span.parent_span_id),
@@ -84,7 +77,7 @@ defmodule LogflareGrpc.Trace.Transform do
     metadata = Map.merge(metadata, resource)
 
     %{
-      "message" => event.name,
+      "event_message" => event.name,
       "metadata" => metadata,
       "parent_span_id" => Base.encode16(span_id),
       "trace_id" => Ecto.UUID.cast!(trace_id),

lib/logflare_grpc/opentelemetry/proto/collector/logs/v1/logs_service.pb.ex

@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Collector.Logs.V1.ExportLogsServiceRequest do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_logs, 1,
     repeated: true,
@@ -10,7 +11,8 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.ExportLogsServiceResponse do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :partial_success, 1,
     type: Opentelemetry.Proto.Collector.Logs.V1.ExportLogsPartialSuccess,
@@ -19,17 +21,19 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.ExportLogsPartialSuccess do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :rejected_log_records, 1, type: :int64, json_name: "rejectedLogRecords"
   field :error_message, 2, type: :string, json_name: "errorMessage"
 end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.LogsService.Service do
   @moduledoc false
+
   use GRPC.Service,
     name: "opentelemetry.proto.collector.logs.v1.LogsService",
-    protoc_gen_elixir_version: "0.11.0"
+    protoc_gen_elixir_version: "0.13.0"
 
   rpc(
     :Export,
@@ -40,5 +44,6 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Logs.V1.LogsService.Stub do
   @moduledoc false
+
   use GRPC.Stub, service: Opentelemetry.Proto.Collector.Logs.V1.LogsService.Service
 end

lib/logflare_grpc/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.ex

@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsServiceRequest do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :resource_metrics, 1,
     repeated: true,
@@ -10,7 +11,8 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsServiceResponse do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :partial_success, 1,
     type: Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsPartialSuccess,
@@ -19,17 +21,19 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.ExportMetricsPartialSuccess do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :rejected_data_points, 1, type: :int64, json_name: "rejectedDataPoints"
   field :error_message, 2, type: :string, json_name: "errorMessage"
 end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.MetricsService.Service do
   @moduledoc false
+
   use GRPC.Service,
     name: "opentelemetry.proto.collector.metrics.v1.MetricsService",
-    protoc_gen_elixir_version: "0.11.0"
+    protoc_gen_elixir_version: "0.13.0"
 
   rpc(
     :Export,
@@ -40,5 +44,6 @@ end
 
 defmodule Opentelemetry.Proto.Collector.Metrics.V1.MetricsService.Stub do
   @moduledoc false
+
   use GRPC.Stub, service: Opentelemetry.Proto.Collector.Metrics.V1.MetricsService.Service
 end

lib/logflare_grpc/opentelemetry/proto/collector/profiles/v1development/profiles_service.pb.ex

@@ -0,0 +1,50 @@
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceRequest do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :resource_profiles, 1,
+    repeated: true,
+    type: Opentelemetry.Proto.Profiles.V1development.ResourceProfiles,
+    json_name: "resourceProfiles"
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceResponse do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :partial_success, 1,
+    type: Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesPartialSuccess,
+    json_name: "partialSuccess"
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesPartialSuccess do
+  @moduledoc false
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :rejected_profiles, 1, type: :int64, json_name: "rejectedProfiles"
+  field :error_message, 2, type: :string, json_name: "errorMessage"
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ProfilesService.Service do
+  @moduledoc false
+
+  use GRPC.Service,
+    name: "opentelemetry.proto.collector.profiles.v1development.ProfilesService",
+    protoc_gen_elixir_version: "0.13.0"
+
+  rpc(
+    :Export,
+    Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceRequest,
+    Opentelemetry.Proto.Collector.Profiles.V1development.ExportProfilesServiceResponse
+  )
+end
+
+defmodule Opentelemetry.Proto.Collector.Profiles.V1development.ProfilesService.Stub do
+  @moduledoc false
+
+  use GRPC.Stub,
+    service: Opentelemetry.Proto.Collector.Profiles.V1development.ProfilesService.Service
+end

lib/logflare_grpc/opentelemetry/proto/collector/trace/v1/trace_service.pb.ex

@@ -1,46 +1,49 @@
 defmodule Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field(:resource_spans, 1,
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :resource_spans, 1,
     repeated: true,
     type: Opentelemetry.Proto.Trace.V1.ResourceSpans,
     json_name: "resourceSpans"
-  )
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field(:partial_success, 1,
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :partial_success, 1,
     type: Opentelemetry.Proto.Collector.Trace.V1.ExportTracePartialSuccess,
     json_name: "partialSuccess"
-  )
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.ExportTracePartialSuccess do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
 
-  field(:rejected_spans, 1, type: :int64, json_name: "rejectedSpans")
-  field(:error_message, 2, type: :string, json_name: "errorMessage")
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+
+  field :rejected_spans, 1, type: :int64, json_name: "rejectedSpans"
+  field :error_message, 2, type: :string, json_name: "errorMessage"
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.TraceService.Service do
   @moduledoc false
+
   use GRPC.Service,
     name: "opentelemetry.proto.collector.trace.v1.TraceService",
-    protoc_gen_elixir_version: "0.11.0"
+    protoc_gen_elixir_version: "0.13.0"
 
   rpc(
     :Export,
-    stream(Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest),
-    stream(Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse)
+    Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceRequest,
+    Opentelemetry.Proto.Collector.Trace.V1.ExportTraceServiceResponse
   )
 end
 
 defmodule Opentelemetry.Proto.Collector.Trace.V1.TraceService.Stub do
   @moduledoc false
+
   use GRPC.Stub, service: Opentelemetry.Proto.Collector.Trace.V1.TraceService.Service
 end

lib/logflare_grpc/opentelemetry/proto/common/v1/common.pb.ex

@@ -1,6 +1,7 @@
 defmodule Opentelemetry.Proto.Common.V1.AnyValue do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   oneof(:value, 0)
 
@@ -24,29 +25,33 @@ end
 
 defmodule Opentelemetry.Proto.Common.V1.ArrayValue do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :values, 1, repeated: true, type: Opentelemetry.Proto.Common.V1.AnyValue
 end
 
 defmodule Opentelemetry.Proto.Common.V1.KeyValueList do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :values, 1, repeated: true, type: Opentelemetry.Proto.Common.V1.KeyValue
 end
 
 defmodule Opentelemetry.Proto.Common.V1.KeyValue do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :key, 1, type: :string
   field :value, 2, type: Opentelemetry.Proto.Common.V1.AnyValue
 end
 
 defmodule Opentelemetry.Proto.Common.V1.InstrumentationScope do
   @moduledoc false
-  use Protobuf, protoc_gen_elixir_version: "0.11.0", syntax: :proto3
+
+  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
 
   field :name, 1, type: :string
   field :version, 2, type: :string