feat: remove config field usage (#2168)

* feat: remove config field usage

* feat: nilify config columns migration

docs/docs.logflare.com/docs/self-hosting/index.md

@@ -67,6 +67,8 @@ Encryption keys must be Base64 encoded.
 
 Cipher used is AES with a 256-bit key in GCM mode.
 
+If `LOGFLARE_DB_ENCRYPTION_KEY` environement variable is not provided, a default hardcoded encryption key will be used.
+
 ### Rolling Encryption Keys
 
 In order to roll encryption keys and migrate existing encrypted data, use the `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED` environment variable.
@@ -77,7 +79,7 @@ Steps to perform the migration are:
 2. Generate a new encryption key and set it to `LOGFLARE_DB_ENCRYPTION_KEY`.
 3. Restart or deploy the server with the new environment variables.
 4. Upon successful server startup, an `info` log will be emitted that says that an retired encryption key is detected, and the migration will be initiated to transition all data encrypted with the retired key to be encrypted with the new key.
-5. Once the migration is complete, the retired encryption key can be safely removed.
+5. Once the migration is complete, the retired encryption key can be safely removed. There will be an `info` log that will be emitted once the migration is complete.
 
 ## BigQuery Setup
 

lib/logflare/backends/backend.ex

@@ -23,8 +23,7 @@ defmodule Logflare.Backends.Backend do
     field(:description, :string)
     field(:token, Ecto.UUID, autogenerate: true)
     field(:type, Ecto.Enum, values: Map.keys(@adaptor_mapping))
-    # TODO(Ziinc): make virtual once cluster is using encrypted fields fully
-    field(:config, :map)
+    field(:config, :map, virtual: true)
     field(:config_encrypted, Logflare.Ecto.EncryptedMap)
     many_to_many(:sources, Source, join_through: "sources_backends")
     belongs_to(:user, User)
@@ -41,17 +40,15 @@ defmodule Logflare.Backends.Backend do
     |> cast(attrs, [:type, :config, :user_id, :name, :description, :metadata])
     |> validate_required([:user_id, :type, :config, :name])
     |> validate_inclusion(:type, Map.keys(@adaptor_mapping))
-    |> do_config_change()
     |> validate_config()
+    |> do_config_change()
   end
 
   # temp function
   defp do_config_change(%Ecto.Changeset{changes: %{config: config}} = changeset) do
     changeset
     |> put_change(:config_encrypted, config)
-
-    # TODO(Ziinc): uncomment once cluster is using encrypted fields fully
-    # |> delete_change(:config)
+    |> delete_change(:config)
   end
 
   defp do_config_change(changeset), do: changeset
@@ -80,7 +77,9 @@ defmodule Logflare.Backends.Backend do
       type = value.type
 
       values =
-        Map.take(value, [
+        value
+        |> Map.put(:config, value.config_encrypted)
+        |> Map.take([
           :name,
           :token,
           :description,

priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs

@@ -0,0 +1,30 @@
+defmodule Logflare.Repo.Migrations.NilifyConfigColumnForBackendsTable do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Logflare.Ecto.EncryptedMap
+
+  def up do
+    from(b in "backends", update: [set: [config: nil]])
+    |> Logflare.Repo.update_all([])
+  end
+
+  def down do
+    {:ok, pid} = Logflare.Vault.start_link()
+
+    # copy configs over
+    Logflare.Repo.all(from b in "backends", select: [:id, :config_encrypted])
+    |> Enum.each(fn %{id: id} = backend ->
+      {:ok, config} = EncryptedMap.load(backend.config_encrypted)
+
+      from(b in "backends",
+        where: b.id == ^id,
+        update: [set: [config: ^config]]
+      )
+      |> Logflare.Repo.update_all([])
+    end)
+    # stop the vault
+    Process.unlink(pid)
+    Process.exit(pid, :kill)
+    :timer.sleep(100)
+  end
+end

test/logflare/backends_test.exs

@@ -24,8 +24,6 @@ defmodule Logflare.BackendsTest do
   end
 
   describe "encryption" do
-    # TODO(Ziinc): unskip once cluster is using encrypted fields fully
-    @tag :skip
     test "backend config is encrypted to the :config_encrypted field" do
       insert(:backend, config_encrypted: %{some_value: "testing"})
 

test/logflare/vault_test.exs

@@ -75,8 +75,7 @@ defmodule Logflare.VaultTest do
   defp get_config_encrypted() do
     [
       %{
-        # TODO(Ziinc): to uncomment once fully migrated over
-        # config: nil,
+        config: nil,
         config_encrypted: encrypted_str
       }
     ] = Repo.all(from b in "backends", select: [:config, :config_encrypted])