From c71f2df9f6f83212788ff91a7968fa0b4b2195ce Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 31 Jul 2024 23:20:27 +0000
Subject: [PATCH 1/2] Bump version to 9.4.3

---
 app/mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/mix.exs b/app/mix.exs
index 0d336f0e3..17a132599 100644
--- a/app/mix.exs
+++ b/app/mix.exs
@@ -3,7 +3,7 @@ Code.require_file("lib/env.ex")
 defmodule Meadow.MixProject do
   use Mix.Project
 
-  @app_version "9.4.2"
+  @app_version "9.4.3"
 
   def project do
     [

From 98c80d48219258d5decbe5d60d19ca18bc66b6c0 Mon Sep 17 00:00:00 2001
From: "Michael B. Klein" <mbklein@gmail.com>
Date: Thu, 1 Aug 2024 13:35:11 -0500
Subject: [PATCH 2/2] Dynamically determine dimensionality of embedding field
 (#4073)

* Dynamically determine dimensionality of embedding field

* Use secrets to manage model dimension config because of Sagemaker cold starts

---------

Co-authored-by: Brendan Quinn <brendan-quinn@northwestern.edu>
Co-authored-by: Karen Shaw <karendid@gmail.com>
---
 app/config/config.exs           | 11 +++++++----
 app/lib/meadow/search/config.ex | 24 ++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/app/config/config.exs b/app/config/config.exs
index f9531e8ef..55aea6a1a 100644
--- a/app/config/config.exs
+++ b/app/config/config.exs
@@ -75,10 +75,13 @@ config :meadow, Meadow.Search.Cluster,
   bulk_page_size: 200,
   bulk_wait_interval: 500,
   json_encoder: Ecto.Jason,
-  embedding_model_id: aws_secret("meadow",
-    dig: ["search", "embedding_model_id"],
-    default: nil
-  )
+  embedding_model_id:
+    aws_secret("meadow",
+      dig: ["search", "embedding_model_id"],
+      default: nil
+    ),
+  embedding_dimensions:
+    aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil)
 
 config :meadow,
   ark: %{
diff --git a/app/lib/meadow/search/config.ex b/app/lib/meadow/search/config.ex
index eb2865c42..24315837b 100644
--- a/app/lib/meadow/search/config.ex
+++ b/app/lib/meadow/search/config.ex
@@ -2,6 +2,7 @@ defmodule Meadow.Search.Config do
   @moduledoc """
   Convenience methods for retrieving search-specific configuration
   """
+  alias Meadow.Search.HTTP
 
   require Logger
 
@@ -38,6 +39,7 @@ defmodule Meadow.Search.Config do
         File.read!(settings)
         |> Jason.decode!()
         |> put_in(["settings", "default_pipeline"], pipeline)
+        |> add_embedding_dimension()
 
       {%{settings: settings}, _} ->
         File.read!(settings) |> Jason.decode!()
@@ -52,6 +54,28 @@ defmodule Meadow.Search.Config do
     |> Keyword.get(:embedding_model_id)
   end
 
+  def add_embedding_dimension(
+        %{"mappings" => %{"properties" => %{"embedding" => %{"dimension" => _}}}} = settings
+      ) do
+    case embedding_model_dimensions() do
+      nil -> settings
+      _ -> insert_embedding_dimension(settings)
+    end
+  end
+
+  def insert_embedding_dimension(settings),
+    do:
+      put_in(
+        settings,
+        ["mappings", "properties", "embedding", "dimension"],
+        embedding_model_dimensions()
+      )
+
+  def embedding_model_dimensions do
+    Application.get_env(:meadow, Meadow.Search.Cluster)
+    |> Keyword.get(:embedding_dimensions)
+  end
+
   def index_versions do
     index_configs()
     |> Enum.map(& &1.version)