feat: refine loki integration, switch to http1

Logflare · Oct 8, 2024 · d529c2d · d529c2d
1 parent e79a778
commit d529c2d
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 5 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -60,6 +60,39 @@ services:
       # - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-changeme}
     # disable strict permission checks
     command: ["--strict.perms=false"]
+  loki:
+    image: grafana/loki:2.9.2
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml -log.level=debug
+
+  grafana:
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    entrypoint:
+      - sh
+      - -euc
+      - |
+        mkdir -p /etc/grafana/provisioning/datasources
+        cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
+        apiVersion: 1
+        datasources:
+        - name: Loki
+          type: loki
+          access: proxy 
+          orgId: 1
+          url: http://loki:3100
+          basicAuth: false
+          isDefault: true
+          version: 1
+          editable: false
+        EOF
+        /run.sh
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
 volumes:
   pg-data:
 
diff --git a/lib/logflare/backends/adaptor/loki_adaptor.ex b/lib/logflare/backends/adaptor/loki_adaptor.ex
@@ -36,7 +36,8 @@ defmodule Logflare.Backends.Adaptor.LokiAdaptor do
           url: backend.config.url,
           headers: Map.get(backend.config, :headers, %{}),
           format_batch: &format_batch/1,
-          gzip: true
+          gzip: true,
+          http: "http1"
         }
     }
 
@@ -47,19 +48,25 @@ defmodule Logflare.Backends.Adaptor.LokiAdaptor do
   def format_batch(log_events) do
     # split events
     stream_map =
-      for %_{source: %_{name: name, id: source_id}} = event <- log_events, reduce: %{} do
+      for %_{source: %_{name: name, service_name: service_name, id: source_id}} = event <-
+            log_events,
+          reduce: %{} do
         acc ->
-          Map.update(acc, {source_id, name}, [event], fn prev -> [event | prev] end)
+          Map.update(acc, {source_id, service_name || name}, [event], fn prev ->
+            [event | prev]
+          end)
       end
 
     streams =
       for {{_source_id, name}, events} <- stream_map do
         formatted_events =
           Enum.map(events, fn event ->
-            ["#{event.body["timestamp"]}", Jason.encode!(event.body)]
+            formatted_ts = event.body["timestamp"] * 1000
+
+            ["#{formatted_ts}", Jason.encode!(event.body)]
           end)
 
-        %{stream: %{source: name}, values: formatted_events}
+        %{stream: %{source: "supabase", service: name}, values: formatted_events}
       end
 
     %{streams: streams}