Add copy_apps

lib/flame/code_sync.ex

@@ -22,7 +22,9 @@ defmodule FLAME.CodeSync do
   alias FLAME.CodeSync.PackagedStream
 
   defstruct id: nil,
+            get_path: nil,
             sync_beam_hashes: %{},
+            copy_apps: nil,
             copy_paths: nil,
             sync_beams: nil,
             extract_dir: nil,
@@ -38,8 +40,10 @@ defmodule FLAME.CodeSync do
 
   def new(opts \\ []) do
     Keyword.validate!(opts, [
+      :get_path,
       :tmp_dir,
       :extract_dir,
+      :copy_apps,
       :copy_paths,
       :sync_beams,
       :start_apps,
@@ -48,13 +52,17 @@ defmodule FLAME.CodeSync do
       :chunk_size
     ])
 
+    start_apps = Keyword.get(opts, :start_apps, true)
+
     compute_start_apps(%CodeSync{
       id: System.unique_integer([:positive]),
+      get_path: Keyword.get(opts, :get_path, &:code.get_path/0),
+      start_apps: start_apps,
+      copy_apps: Keyword.get(opts, :copy_apps, start_apps),
       copy_paths: Keyword.get(opts, :copy_paths, false),
       sync_beams: Keyword.get(opts, :sync_beams, []),
       tmp_dir: Keyword.get(opts, :tmp_dir, {System, :tmp_dir!, []}),
       extract_dir: Keyword.get(opts, :extract_dir, {Function, :identity, ["/"]}),
-      start_apps: Keyword.get(opts, :start_apps, true),
       verbose: Keyword.get(opts, :verbose, false),
       compress: Keyword.get(opts, :compress, true),
       chunk_size: Keyword.get(opts, :chunk_size, 64_000)
@@ -92,34 +100,30 @@ defmodule FLAME.CodeSync do
     }
   end
 
-  def compute_copy_paths(%CodeSync{} = code) do
-    copy_paths =
+  def compute_changed_paths(%CodeSync{} = code) do
+    copy_apps =
+      case code.copy_apps do
+        true -> lookup_apps_files(code)
+        false -> []
+      end
+
+    changed_paths =
       case code.copy_paths do
         paths when is_list(paths) ->
-          paths
+          Enum.uniq(lookup_copy_paths_files(paths) ++ copy_apps)
 
         false ->
-          []
+          copy_apps
 
         true ->
-          otp_lib = :code.lib_dir()
-
-          reject_apps =
-            for app <- [:flame, :eex, :elixir, :ex_unit, :iex, :logger, :mix],
-                lib_dir = :code.lib_dir(app),
-                is_list(lib_dir),
-                do: :filename.join(lib_dir, ~c"ebin")
-
-          :code.get_path()
-          |> Kernel.--([~c"." | reject_apps])
-          |> Enum.reject(fn path -> List.starts_with?(path, otp_lib) end)
-          |> Enum.map(&to_string/1)
+          IO.warn(
+            "copy_paths: true is deprecated, use starts_apps: true or copy_apps: true instead"
+          )
+
+          lookup_apps_files(code)
       end
 
-    %CodeSync{
-      code
-      | changed_paths: Enum.uniq(code.changed_paths ++ lookup_copy_files(copy_paths))
-    }
+    %CodeSync{code | changed_paths: Enum.uniq(code.changed_paths ++ changed_paths)}
   end
 
   def changed?(%CodeSync{} = code) do
@@ -270,20 +274,39 @@ defmodule FLAME.CodeSync do
     |> Enum.uniq()
   end
 
-  defp lookup_copy_files(paths) do
-    # include ebin's parent if basename is ebin (will include priv)
-    paths
+  defp lookup_apps_files(%CodeSync{get_path: get_path}) do
+    otp_lib = to_string(:code.lib_dir())
+
+    reject_apps =
+      for app <- [:flame, :eex, :elixir, :ex_unit, :iex, :logger, :mix],
+          lib_dir = :code.lib_dir(app),
+          is_list(lib_dir),
+          do: to_string(:filename.join(lib_dir, ~c"ebin"))
+
+    get_path.()
+    |> Enum.map(&to_string/1)
+    |> Kernel.--(["." | reject_apps])
+    |> Stream.reject(fn path -> String.starts_with?(path, otp_lib) end)
     |> Stream.map(fn parent_dir ->
+      # include ebin's parent if basename is ebin (will include priv)
       case Path.basename(parent_dir) do
-        "ebin" ->
-          Path.join(Path.dirname(parent_dir), "**/*")
-
-        _ ->
-          if File.regular?(parent_dir, [:raw]) do
-            parent_dir
-          else
-            Path.join(parent_dir, "*")
-          end
+        "ebin" -> Path.join(Path.dirname(parent_dir), "**/*")
+        _ -> Path.join(parent_dir, "*")
+      end
+    end)
+    |> Stream.uniq()
+    |> Stream.flat_map(fn glob -> Path.wildcard(glob) end)
+    |> Stream.uniq()
+    |> Enum.filter(fn path -> File.regular?(path, [:raw]) end)
+  end
+
+  defp lookup_copy_paths_files(paths) do
+    paths
+    |> Stream.map(fn parent_dir ->
+      if File.regular?(parent_dir, [:raw]) do
+        parent_dir
+      else
+        Path.join(parent_dir, "*")
       end
     end)
     |> Stream.uniq()

lib/flame/pool.ex

@@ -146,29 +146,33 @@ defmodule FLAME.Pool do
     * `:code_sync` – The optional list of options to enable copying and syncing code paths
       from the parent node to the runner node. Disabled by default. The options are:
 
-      * `:copy_paths` – If `true`, the pool will copy the code paths from the parent node
-        to the runner node on boot. Then any subsequent FLAME operation will sync code paths
-        from parent to child. Useful when you are starting an image that needs to run
-        dynamic code that is not available on the runner node. Defaults to `false`.
+      * `:start_apps` – Either a boolean or a list of specific OTP application names to start
+        when the runner boots. When `true`, all applications currently running on the parent node
+        are sent to the runner node to be started. Defaults to `false`. When set to `true`,
+        `copy_apps` will also be set to `true` if not explicitly set to `false`.
+
+      * `:copy_apps` – The boolean flag to copy all the application artifacts and their beam
+        files from the parent node to the runner node on boot. Defaults `false`.
+        When passing `start_apps: true`, automatically sets `copy_paths: true`.
+
+      * `:copy_paths` – The list of arbitrary paths to copy from the parent node to the runner
+        node on boot. Defaults to `[]`.
 
       * `:sync_beams` – A list of specific beam code paths to sync to the runner node. Useful
         when you want to sync specific beam code paths from the parent after sending all code
-        paths from `:copy_paths` on initial boot. For example, with `copy_paths: true`,
+        paths from `:copy_apps` on initial boot. For example, with `copy_apps: true`,
         and `sync_beams: ["/home/app/.cache/.../ebin"]`, all the code from the parent will be
         copied on boot, but only the specific beam files will be synced on subsequent calls.
-        With `copy_paths: false`, and `sync_beams: ["/home/app/.cache/.../ebin"]`,
+        With `copy_apps: false`, and `sync_beams: ["/home/app/.cache/.../ebin"]`,
         only the specific beam files will be synced on boot and for subsequent calls.
         Defaults to `[]`.
 
-      * `:start_apps` – Either a boolean or a list of specific OTP application names to start
-        when the runner boots. When `true`, all applications currently running on the parent node
-        are sent to the runner node to be started. Defaults to `false`.
-
       * `:verbose` – If `true`, the pool will log verbose information about the code sync process.
         Defaults to `false`.
 
-      * `:compress` – If `true`, the copy_paths and sync_beams will be compressed before sending.
-        Provides savings in network payload size at the cost of CPU time. Defaults to `false`.
+      * `:compress` – If `true`, the copy_apps, copy_paths, and sync_beams will be compressed
+        before sending. Provides savings in network payload size at the cost of CPU time.
+        Defaults to `true`.
 
       For example, in [Livebook](https://livebook.dev/), to start a pool with code sync enabled:
 
@@ -179,7 +183,6 @@ defmodule FLAME.Pool do
               name: :my_flame,
               code_sync: [
                 start_apps: true,
-                copy_paths: true,
                 sync_beams: [Path.join(System.tmp_dir!(), "livebook_runtime")]
               ],
               min: 1,
@@ -219,6 +222,7 @@ defmodule FLAME.Pool do
     ])
 
     Keyword.validate!(opts[:code_sync] || [], [
+      :copy_apps,
       :copy_paths,
       :sync_beams,
       :start_apps,
@@ -408,7 +412,7 @@ defmodule FLAME.Pool do
         code_sync =
           code_sync_opts
           |> CodeSync.new()
-          |> CodeSync.compute_copy_paths()
+          |> CodeSync.compute_changed_paths()
 
         %CodeSync.PackagedStream{} = parent_stream = CodeSync.package_to_stream(code_sync)
         parent_stream

lib/flame/runner.ex

@@ -348,6 +348,8 @@ defmodule FLAME.Runner do
       ])
 
     Keyword.validate!(opts[:code_sync] || [], [
+      :get_path,
+      :copy_apps,
       :copy_paths,
       :sync_beams,
       :start_apps,

test/code_sync_test.exs

@@ -22,7 +22,7 @@ defmodule FLAME.CodeSyncTest do
       code_sync =
         mock.opts
         |> CodeSync.new()
-        |> CodeSync.compute_copy_paths()
+        |> CodeSync.compute_changed_paths()
 
       assert %CodeSync{
                sync_beam_hashes: %{},
@@ -98,14 +98,14 @@ defmodule FLAME.CodeSyncTest do
     assert current.apps_to_start == []
   end
 
-  test "compute_copy_paths packages and extracts packaged code and starts apps" do
+  test "compute_changed_paths packages and extracts packaged code and starts apps" do
     assert :logger in started_apps()
     mock = CodeSyncMock.new()
 
     code =
       mock.opts
       |> CodeSync.new()
-      |> CodeSync.compute_copy_paths()
+      |> CodeSync.compute_changed_paths()
 
     assert %FLAME.CodeSync.PackagedStream{} = pkg = CodeSync.package_to_stream(code)
     assert File.exists?(pkg.stream.path)

test/support/code_sync_mock.ex

@@ -38,11 +38,19 @@ defmodule FLAME.Test.CodeSyncMock do
     extract_dir = Path.join([tmp_dir, "#{id}", "extracted_code"])
     File.mkdir_p!(extract_dir)
 
+    get_path =
+      fn ->
+        working_dir
+        |> Path.join("*/ebin")
+        |> Path.wildcard()
+      end
+
     default_opts = [
-      copy_paths: Path.wildcard(Path.join(working_dir, "*/ebin")),
+      start_apps: true,
       sync_beams: [working_dir],
       tmp_dir: {Function, :identity, [tmp_dir]},
-      extract_dir: {__MODULE__, :extract_dir, [id, test_pid, extract_dir]}
+      extract_dir: {__MODULE__, :extract_dir, [id, test_pid, extract_dir]},
+      get_path: get_path
     ]
 
     %CodeSyncMock{id: id, opts: Keyword.merge(default_opts, opts)}