Dynamically determine dimensionality of embedding field (#4073)

* Dynamically determine dimensionality of embedding field

* Use secrets to manage model dimension config because of Sagemaker cold starts

---------

Co-authored-by: Brendan Quinn <brendan-quinn@northwestern.edu>
Co-authored-by: Karen Shaw <karendid@gmail.com>

app/config/config.exs

@@ -75,10 +75,13 @@ config :meadow, Meadow.Search.Cluster,
   bulk_page_size: 200,
   bulk_wait_interval: 500,
   json_encoder: Ecto.Jason,
-  embedding_model_id: aws_secret("meadow",
-    dig: ["search", "embedding_model_id"],
-    default: nil
-  )
+  embedding_model_id:
+    aws_secret("meadow",
+      dig: ["search", "embedding_model_id"],
+      default: nil
+    ),
+  embedding_dimensions:
+    aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil)
 
 config :meadow,
   ark: %{

app/lib/meadow/search/config.ex

@@ -2,6 +2,7 @@ defmodule Meadow.Search.Config do
   @moduledoc """
   Convenience methods for retrieving search-specific configuration
   """
+  alias Meadow.Search.HTTP
 
   require Logger
 
@@ -38,6 +39,7 @@ defmodule Meadow.Search.Config do
         File.read!(settings)
         |> Jason.decode!()
         |> put_in(["settings", "default_pipeline"], pipeline)
+        |> add_embedding_dimension()
 
       {%{settings: settings}, _} ->
         File.read!(settings) |> Jason.decode!()
@@ -52,6 +54,28 @@ defmodule Meadow.Search.Config do
     |> Keyword.get(:embedding_model_id)
   end
 
+  def add_embedding_dimension(
+        %{"mappings" => %{"properties" => %{"embedding" => %{"dimension" => _}}}} = settings
+      ) do
+    case embedding_model_dimensions() do
+      nil -> settings
+      _ -> insert_embedding_dimension(settings)
+    end
+  end
+
+  def insert_embedding_dimension(settings),
+    do:
+      put_in(
+        settings,
+        ["mappings", "properties", "embedding", "dimension"],
+        embedding_model_dimensions()
+      )
+
+  def embedding_model_dimensions do
+    Application.get_env(:meadow, Meadow.Search.Cluster)
+    |> Keyword.get(:embedding_dimensions)
+  end
+
   def index_versions do
     index_configs()
     |> Enum.map(& &1.version)