perf: scaled partitioning of registries (#2130)

* chore: fix flaky schema check

* Revert "fix: reduce exit trapping and error logging on supervision tree termination"

This reverts commit e2fb447.

* chore: resolve flaky tests, remove unnecessary exit trapping

* perf: scaled partitioning of registries

lib/logflare/application.ex

@@ -63,7 +63,9 @@ defmodule Logflare.Application do
         Logflare.Repo,
         Logflare.Backends,
         {Registry,
-         name: Logflare.V1SourceRegistry, keys: :unique, partitions: System.schedulers_online()},
+         name: Logflare.V1SourceRegistry,
+         keys: :unique,
+         partitions: max(round(System.schedulers_online() / 8), 1)},
         {Task.Supervisor, name: Logflare.TaskSupervisor},
         {DynamicSupervisor, strategy: :one_for_one, name: Logflare.Endpoints.Cache},
         {DynamicSupervisor,
@@ -133,7 +135,9 @@ defmodule Logflare.Application do
         # Sources
         # v1 ingest pipline
         {Registry,
-         name: Logflare.V1SourceRegistry, keys: :unique, partitions: System.schedulers_online()},
+         name: Logflare.V1SourceRegistry,
+         keys: :unique,
+         partitions: max(round(System.schedulers_online() / 8), 1)},
         Logs.RejectedLogEvents,
         # init Counters before Supervisof as Supervisor calls Counters through table create
         Counters,

lib/logflare/backends/supervisor.ex

@@ -13,14 +13,18 @@ defmodule Logflare.Backends.Supervisor do
 
   @impl Supervisor
   def init(_) do
+    base = System.schedulers_online()
+
     children = [
       Backends.IngestEventQueue,
       Backends.IngestEventQueue.BroadcastWorker,
       Backends.IngestEventQueue.MapperJanitor,
       Backends.Adaptor.PostgresAdaptor.Supervisor,
       {DynamicSupervisor, strategy: :one_for_one, name: Backends.SourcesSup},
-      {Registry, name: Backends.SourceRegistry, keys: :unique},
-      {Registry, name: Backends.BackendRegistry, keys: :unique}
+      {Registry,
+       name: Backends.SourceRegistry, keys: :unique, partitions: max(round(base / 8), 1)},
+      {Registry,
+       name: Backends.BackendRegistry, keys: :unique, partitions: max(round(base / 8), 1)}
     ]
 
     opts = [strategy: :one_for_one]