Merge pull request #4110 from nulib/livebook-storage-config

Improve Meadow Livebook user experience

bin/meadow-livebook

@@ -15,11 +15,13 @@ echo -n "   Starting Livebook..."
 docker buildx build -t nulib/meadow:livebook $ROOT/livebook > /dev/null 2>&1
 LIVEBOOK_CONTAINER=$(docker run --rm -d --network host \
   -v ${HOME}/.meadow_livebooks:/data/books \
+  -v ${HOME}/environment/meadow_kino:/meadow_kino \
   -e LB_MEADOW_COOKIE=${COOKIE} \
   -e LB_MEADOW_NODE=meadow@${HOST} \
   -e LIVEBOOK_NODE=livebook@${HOST} \
   -e LIVEBOOK_DISTRIBUTION=name \
   -e LIVEBOOK_COOKIE=${COOKIE} \
+  -e MEADOW_LIVEBOOK_BUCKET=${MEADOW_LIVEBOOK_BUCKET} \
   -e MEADOW_URL=https://${MEADOW_HOSTNAME}:3001/ \
   nulib/meadow:livebook)
 https-proxy start 8082 8080 > /dev/null 2>&1

infrastructure/deploy/modules/meadow_task/main.tf

@@ -26,6 +26,7 @@ locals {
       db_queue_interval     = var.db_queue_interval,
       db_queue_target       = var.db_queue_target,
       docker_repository     = data.aws_ecr_repository.meadow.repository_url,
+      livebook_bucket       = var.livebook_shared_bucket,
       name                  = var.name,
       processes             = var.meadow_processes
     }

infrastructure/deploy/task-definitions/meadow_app.json

@@ -187,6 +187,10 @@
       {
         "name": "LIVEBOOK_COOKIE",
         "value": "${secret_key_base}"
+      },
+      {
+        "name": "MEADOW_LIVEBOOK_BUCKET",
+        "value": "${livebook_bucket}"
       }
     ],
     "mountPoints": [],

infrastructure/deploy/variables.tf

@@ -208,3 +208,8 @@ variable "trusted_referers" {
   type    = string
   default = ""
 }
+
+variable "livebook_shared_bucket" {
+  type    = string
+  default = ""
+}

livebook/Dockerfile

@@ -7,4 +7,5 @@ ENV LIVEBOOK_IDENTITY_PROVIDER=custom:MeadowLivebookAuth
 ENV LIVEBOOK_IP=0.0.0.0
 ENV LIVEBOOK_TOKEN_ENABLED=false
 RUN mkdir -p ${LIVEBOOK_DATA_PATH}/books
-ADD ./meadow_livebook_auth.exs /app/user/extensions/meadow_livebook_auth.exs
+RUN git clone https://github.com/nulib/meadow_kino.git /meadow_kino
+ADD ./extensions/* /app/user/extensions/

livebook/meadow_livebook_auth.exs → livebook/extensions/meadow_livebook_auth.exs

@@ -23,6 +23,9 @@ defmodule MeadowLivebookAuth do
   @spec authenticate(GenServer.server(), Plug.Conn.t(), keyword()) ::
           {Plug.Conn.t(), map() | nil}
   def authenticate(server, conn, _) do
+    System.get_env("MEADOW_LIVEBOOK_BUCKET")
+    |> attach_storage()
+
     with url <- find_meadow_url(server),
          user <- meadow_auth(url, conn) do
       {conn, user}
@@ -90,4 +93,38 @@ defmodule MeadowLivebookAuth do
       map -> Map.put(map, key, value)
     end)
   end
+
+  defp attach_storage(bucket) when is_binary(bucket) and byte_size(bucket) > 0 do
+    url = "https://s3.amazonaws.com/#{bucket}"
+
+    Livebook.Hubs.get_file_systems()
+    |> Enum.any?(fn
+      %Livebook.FileSystem.S3{bucket_url: ^url} -> true
+      _ -> false
+    end)
+    |> attach_s3_storage(url)
+  end
+
+  defp attach_storage(_), do: :noop
+
+  defp attach_s3_storage(true, _), do: :noop
+
+  defp attach_s3_storage(false, url) do
+    hash = :crypto.hash(:sha256, url)
+    encrypted_hash = "s3-" <> Base.url_encode64(hash, padding: false)
+
+    [hub | _] = Livebook.Hubs.get_hubs()
+
+    file_system = %Livebook.FileSystem.S3{
+      id: encrypted_hash,
+      bucket_url: url,
+      external_id: nil,
+      region: Livebook.FileSystem.S3.region_from_url(url),
+      access_key_id: nil,
+      secret_access_key: nil,
+      hub_id: "personal-hub"
+    }
+
+    Livebook.Hubs.create_file_system(hub, file_system)
+  end
 end