feat: added support for OTel-based distributed tracing (#20)

* feat: added support for distributed tracing

* refactor: moved all .proto in the Protos root folder

README.md

@@ -113,6 +113,13 @@ Start an empty PostgreSQL database for development (optionally add `-d` to run i
 docker compose -f docker-compose.yaml up
 ```
 
+To additionally start an instance of [Jaeger Tracing](https://www.jaegertracing.io/), you can specify the `jaeger`
+profile:
+
+```sh
+docker compose -f docker-compose.yaml --profile=jaeger up
+```
+
 Restore dependencies and run in Debug mode:
 
 ```sh
@@ -170,6 +177,34 @@ rm -rf coverage/
 docker build -t ghcr.io/chgl/vfps:latest .
 ```
 
+### Run iter8 SLO experiments locally
+
+```sh
+kind create cluster
+
+export IMAGE_TAG="iter8-test"
+
+docker build -t ghcr.io/chgl/vfps:${IMAGE_TAG} .
+
+kind load docker-image ghcr.io/chgl/vfps:${IMAGE_TAG}
+
+helm repo add chgl https://chgl.github.io/charts
+helm repo update
+
+helm install \
+  --set="image.tag=${IMAGE_TAG}" \
+  -f tests/iter8/values.yaml \
+  --wait \
+  --timeout=10m \
+  vfps chgl/vfps
+
+kubectl apply -f tests/iter8/experiment.yaml
+
+iter8 k assert -c completed --timeout 15m
+iter8 k assert -c nofailure,slos
+iter8 k report
+```
+
 ## Benchmarks
 
 ### Micro benchmarks

docker-compose.yaml

@@ -18,6 +18,27 @@ services:
     ports:
       - "35432:5432"
 
+  jaeger:
+    image: docker.io/jaegertracing/all-in-one:1.38@sha256:dc111a693eb22c38ea11a71b8303430a98efe04826049faab6f73dcf6ee9ad5d
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+        reservations:
+          memory: 1g
+    cap_drop:
+      - ALL
+    ipc: none
+    security_opt:
+      - "no-new-privileges:true"
+    privileged: false
+    profiles:
+      - jaeger
+    ports:
+      - "6831:6831/udp"
+      - "127.0.0.1:16686:16686"
+
   vfps:
     image: ghcr.io/chgl/vfps:${VFPS_IMAGE_TAG:-latest}
     restart: unless-stopped

grpc-utils.Dockerfile

@@ -23,7 +23,6 @@ chmod +x /usr/local/bin/ghz
 ghz --version
 EOF
 
-COPY src/Vfps/google /tmp/protos/google
 COPY src/Vfps/Protos /tmp/protos/Protos
 
 # grpcurl -import-path=/tmp/protos/ -proto=Protos/vfps/api/v1/namespaces.proto describe

src/Vfps.Tests/WebAppTests/TestFactory.cs

@@ -25,7 +25,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder)
             services.EnsureDbCreated<TDbContext>();
         });
 
-        builder.UseEnvironment("test");
+        builder.UseEnvironment("Test");
     }
 }
 

src/Vfps/Fhir/FhirController.cs

@@ -83,9 +83,15 @@ public async Task<ObjectResult> CreatePseudonym([FromBody] Parameters parameters
 
         var generator = Lookup[@namespace.PseudonymGenerationMethod];
 
-        var pseudonymValue = generator.GeneratePseudonym(originalValue, @namespace.PseudonymLength);
+        var pseudonymValue = string.Empty;
 
-        pseudonymValue = $"{@namespace.PseudonymPrefix}{pseudonymValue}{@namespace.PseudonymSuffix}";
+        using (var activity = Program.ActivitySource.StartActivity("GeneratePseudonym"))
+        {
+            activity?.SetTag("Method", generator.GetType().Name);
+
+            pseudonymValue = generator.GeneratePseudonym(originalValue, @namespace.PseudonymLength);
+            pseudonymValue = $"{@namespace.PseudonymPrefix}{pseudonymValue}{@namespace.PseudonymSuffix}";
+        }
 
         var now = DateTimeOffset.UtcNow;
         var pseudonym = new Data.Models.Pseudonym()

src/Vfps/Fhir/FhirFormatter.cs

@@ -16,18 +16,17 @@ public FhirOutputFormatter()
         SupportedEncodings.Add(Encoding.UTF8);
         SupportedEncodings.Add(Encoding.Unicode);
     }
+
     private JsonSerializerOptions FhirJsonOptions { get; } = new JsonSerializerOptions().ForFhir(typeof(Bundle).Assembly);
 
-    protected override bool CanWriteType(Type type)
+    protected override bool CanWriteType(Type? type)
     {
         return typeof(Resource).IsAssignableFrom(type);
     }
 
     public override async Task WriteResponseBodyAsync(
         OutputFormatterWriteContext context, Encoding selectedEncoding)
     {
-        //using var _ = Program.ActivitySource.StartActivity("SerializeFHIRToJSON");
-
         var resource = context.Object as Resource;
         var httpContext = context.HttpContext;
 
@@ -62,8 +61,6 @@ public FhirInputFormatter()
     public override async Task<InputFormatterResult> ReadRequestBodyAsync(InputFormatterContext context,
         Encoding encoding)
     {
-        //using var _ = Program.ActivitySource.StartActivity("DeserializeJSONToFHIR");
-
         var httpContext = context.HttpContext;
         using var reader = new StreamReader(httpContext.Request.Body, encoding);
         var json = await reader.ReadToEndAsync();

src/Vfps/Program.cs

@@ -7,8 +7,9 @@
 using Prometheus;
 using Vfps.Config;
 using Microsoft.Extensions.Caching.Memory;
-using Google.Api;
 using Vfps.Fhir;
+using Vfps.Tracing;
+using System.Diagnostics;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -90,6 +91,13 @@
     options.OutputFormatters.Insert(0, new FhirOutputFormatter());
 });
 
+// Tracing
+var isTracingEnabled = builder.Configuration.GetValue("Tracing:IsEnabled", false);
+if (isTracingEnabled)
+{
+    builder.AddTracing();
+}
+
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.
@@ -137,4 +145,7 @@
 
 app.Run();
 
-public partial class Program { }
+public partial class Program
+{
+    internal static readonly ActivitySource ActivitySource = new("Vfps");
+}

src/Vfps/google/api/annotations.proto → src/Vfps/Protos/google/api/annotations.proto

@@ -16,7 +16,7 @@ syntax = "proto3";
 
 package google.api;
 
-import "google/api/http.proto";
+import "Protos/google/api/http.proto";
 import "google/protobuf/descriptor.proto";
 
 option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";

src/Vfps/Protos/vfps/api/v1/namespaces.proto

@@ -2,8 +2,8 @@ syntax = "proto3";
 
 package vfps.api.v1;
 
+import "Protos/google/api/annotations.proto";
 import "Protos/vfps/api/v1/meta.proto";
-import "google/api/annotations.proto";
 
 option csharp_namespace = "Vfps.Protos";
 

src/Vfps/Protos/vfps/api/v1/pseudonyms.proto

@@ -2,8 +2,8 @@ syntax = "proto3";
 
 package vfps.api.v1;
 
+import "Protos/google/api/annotations.proto";
 import "Protos/vfps/api/v1/meta.proto";
-import "google/api/annotations.proto";
 
 option csharp_namespace = "Vfps.Protos";
 

src/Vfps/Services/PseudonymService.cs

@@ -42,10 +42,15 @@ public override async Task<PseudonymServiceCreateResponse> Create(PseudonymServi
         }
 
         var generator = Lookup[@namespace.PseudonymGenerationMethod];
+        var pseudonymValue = string.Empty;
 
-        var pseudonymValue = generator.GeneratePseudonym(request.OriginalValue, @namespace.PseudonymLength);
+        using (var activity = Program.ActivitySource.StartActivity("GeneratePseudonym"))
+        {
+            activity?.SetTag("Method", generator.GetType().Name);
 
-        pseudonymValue = $"{@namespace.PseudonymPrefix}{pseudonymValue}{@namespace.PseudonymSuffix}";
+            pseudonymValue = generator.GeneratePseudonym(request.OriginalValue, @namespace.PseudonymLength);
+            pseudonymValue = $"{@namespace.PseudonymPrefix}{pseudonymValue}{@namespace.PseudonymSuffix}";
+        }
 
         var pseudonym = new Data.Models.Pseudonym()
         {

src/Vfps/Tracing/TracingConfigurationExtensions.cs

@@ -0,0 +1,73 @@
+using System.Reflection;
+using OpenTelemetry.Trace;
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using Npgsql;
+
+namespace Vfps.Tracing
+{
+    public static class TracingConfigurationExtensions
+    {
+        public static WebApplicationBuilder AddTracing(this WebApplicationBuilder builder)
+        {
+            var assembly = Assembly.GetExecutingAssembly().GetName();
+            var assemblyVersion = assembly.Version?.ToString() ?? "unknown";
+            var tracingExporter = builder.Configuration.GetValue("Tracing:Exporter", "jaeger").ToLowerInvariant();
+            var serviceName = builder.Configuration.GetValue("Tracing:ServiceName", assembly.Name);
+
+            var rootSamplerType = builder.Configuration.GetValue("Tracing:RootSampler", "AlwaysOnSampler");
+            var samplingRatio = builder.Configuration.GetValue("Tracing:SamplingProbability", 0.1d);
+
+            Sampler rootSampler = rootSamplerType switch
+            {
+                nameof(AlwaysOnSampler) => new AlwaysOnSampler(),
+                nameof(AlwaysOffSampler) => new AlwaysOffSampler(),
+                nameof(TraceIdRatioBasedSampler) => new TraceIdRatioBasedSampler(samplingRatio),
+                _ => throw new ArgumentException($"Unsupported sampler type '{rootSamplerType}'"),
+            };
+
+            builder.Services.AddOpenTelemetryTracing(options =>
+            {
+                options
+                    .ConfigureResource(r => r.AddService(
+                        serviceName: serviceName,
+                        serviceVersion: assemblyVersion,
+                        serviceInstanceId: Environment.MachineName))
+                    .SetSampler(new ParentBasedSampler(rootSampler))
+                    .AddNpgsql()
+                    .AddSource(Program.ActivitySource.Name)
+                    .AddAspNetCoreInstrumentation(o =>
+                    {
+                        o.Filter = (r) =>
+                        {
+                            var ignoredPaths = new[]
+                            {
+                                "/healthz",
+                                "/readyz",
+                                "/livez"
+                            };
+
+                            var path = r.Request.Path.Value!;
+                            return !ignoredPaths.Any(path.Contains);
+                        };
+                    });
+
+                switch (tracingExporter)
+                {
+                    case "jaeger":
+                        options.AddJaegerExporter();
+                        builder.Services.Configure<JaegerExporterOptions>(builder.Configuration.GetSection("Tracing:Jaeger"));
+                        break;
+
+                    case "otlp":
+                        var endpoint = builder.Configuration.GetValue("Tracing:Otlp:Endpoint", "");
+                        options.AddOtlpExporter(otlpOptions => otlpOptions.Endpoint = new Uri(endpoint));
+                        break;
+                }
+            });
+
+            return builder;
+        }
+    }
+}

src/Vfps/Vfps.csproj

@@ -21,9 +21,14 @@
         <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="7.0.0-rc.1.22427.2" />
         <PackageReference Include="Npgsql" Version="7.0.0-rc.1" />
         <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.0-rc.1" />
-        <PackageReference Include="prometheus-net.AspNetCore" Version="7.0.0-pre-000282-d90ebf3" />
-        <PackageReference Include="prometheus-net.AspNetCore.Grpc" Version="7.0.0-pre-000277-6bc5023" />
-        <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="7.0.0-pre-000277-6bc5023" />
+        <PackageReference Include="Npgsql.OpenTelemetry" Version="7.0.0-rc.1" />
+        <PackageReference Include="OpenTelemetry.Exporter.Jaeger" Version="1.4.0-beta.1" />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.4.0-beta.1" />
+        <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.0.0-rc9.7" />
+        <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc9.7" />
+        <PackageReference Include="prometheus-net.AspNetCore" Version="7.0.0-pre-000288-4688bd3" />
+        <PackageReference Include="prometheus-net.AspNetCore.Grpc" Version="7.0.0-pre-000288-4688bd3" />
+        <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="7.0.0-pre-000288-4688bd3" />
     </ItemGroup>
     <ItemGroup>
         <Protobuf Include="Protos\vfps\api\v1\*.proto" GrpcServices="Server" />

src/Vfps/appsettings.Development.json

@@ -14,5 +14,13 @@
         "IsEnabled": false
       }
     }
+  },
+  "Tracing": {
+    "IsEnabled": true,
+    "Exporter": "jaeger",
+    "Jaeger": {
+      "AgentHost": "localhost",
+      "AgentPort": 6831
+    }
   }
 }

src/Vfps/appsettings.Test.json

@@ -0,0 +1,5 @@
+{
+  "Tracing": {
+    "IsEnabled": true
+  }
+}

src/Vfps/appsettings.json

@@ -35,5 +35,16 @@
       }
     }
   },
-  "ForceRunDatabaseMigrations": false
+  "ForceRunDatabaseMigrations": false,
+  "Tracing": {
+    "IsEnabled": false,
+    "Exporter": "jaeger",
+    "ServiceName": "vfps",
+    "RootSampler": "AlwaysOnSampler",
+    "SamplingProbability": 0.1,
+    "Jaeger": {},
+    "Otlp": {
+      "Endpoint": ""
+    }
+  }
 }

src/Vfps/buf.yaml

@@ -1,7 +1,7 @@
 version: v1
 lint:
   ignore:
-    - google/
+    - Protos/google/
   use:
     - DEFAULT
   except:

tests/iter8/experiment.yaml

@@ -39,15 +39,16 @@ stringData:
     # task: download proto files
     - run: |
         mkdir -p google/api
-        curl -o google/api/annotations.proto https://raw.githubusercontent.com/chgl/vfps/master/src/Vfps/google/api/annotations.proto
-        curl -o google/api/http.proto https://raw.githubusercontent.com/chgl/vfps/master/src/Vfps/google/api/http.proto
+        curl -o google/api/annotations.proto https://raw.githubusercontent.com/chgl/vfps/v0.4.0/src/Vfps/google/api/annotations.proto
+        curl -o google/api/http.proto https://raw.githubusercontent.com/chgl/vfps/v0.4.0/src/Vfps/google/api/http.proto
 
         mkdir -p Protos/vfps/api/v1
-        curl -o Protos/vfps/api/v1/pseudonyms.proto https://raw.githubusercontent.com/chgl/vfps/master/src/Vfps/Protos/vfps/api/v1/pseudonyms.proto
-        curl -o Protos/vfps/api/v1/meta.proto https://raw.githubusercontent.com/chgl/vfps/master/src/Vfps/Protos/vfps/api/v1/meta.proto
+        curl -o Protos/vfps/api/v1/pseudonyms.proto https://raw.githubusercontent.com/chgl/vfps/v0.4.0/src/Vfps/Protos/vfps/api/v1/pseudonyms.proto
+        curl -o Protos/vfps/api/v1/meta.proto https://raw.githubusercontent.com/chgl/vfps/v0.4.0/src/Vfps/Protos/vfps/api/v1/meta.proto
 
     # task: create a pseudonym namespace for this experiment
     - run: |
+        # using curl since grpcurl isn't installed by default
         curl -X 'POST' \
           'http://vfps:8080/v1/namespaces' \
           -H 'accept: application/json' \
@@ -64,10 +65,11 @@ stringData:
     - task: grpc
       with:
         call: vfps.api.v1.PseudonymService/Create
-        host: vfps:8081
+        host: dns:///vfps:8081
         proto: Protos/vfps/api/v1/pseudonyms.proto
         total: 50000
         insecure: true
+        lb-strategy: round_robin
         data:
           namespace: benchmark
           originalValue: "{{randomString 32}}"