diff --git a/docs/docs.logflare.com/docs/self-hosting/index.md b/docs/docs.logflare.com/docs/self-hosting/index.md index 2ff8a9408..14d561d0e 100644 --- a/docs/docs.logflare.com/docs/self-hosting/index.md +++ b/docs/docs.logflare.com/docs/self-hosting/index.md @@ -65,6 +65,8 @@ Encryption keys must be Base64 encoded. Cipher used is AES with a 256-bit key in GCM mode. +If `LOGFLARE_DB_ENCRYPTION_KEY` environement variable is not provided, a default hardcoded encryption key will be used. + ### Rolling Encryption Keys In order to roll encryption keys and migrate existing encrypted data, use the `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED` environment variable. @@ -75,7 +77,7 @@ Steps to perform the migration are: 2. Generate a new encryption key and set it to `LOGFLARE_DB_ENCRYPTION_KEY`. 3. Restart or deploy the server with the new environment variables. 4. Upon successful server startup, an `info` log will be emitted that says that an retired encryption key is detected, and the migration will be initiated to transition all data encrypted with the retired key to be encrypted with the new key. -5. Once the migration is complete, the retired encryption key can be safely removed. +5. Once the migration is complete, the retired encryption key can be safely removed. There will be an `info` log that will be emitted once the migration is complete. ## BigQuery Setup diff --git a/lib/logflare/backends/backend.ex b/lib/logflare/backends/backend.ex index d7171c02a..358a1f31e 100644 --- a/lib/logflare/backends/backend.ex +++ b/lib/logflare/backends/backend.ex @@ -23,8 +23,7 @@ defmodule Logflare.Backends.Backend do field(:description, :string) field(:token, Ecto.UUID, autogenerate: true) field(:type, Ecto.Enum, values: Map.keys(@adaptor_mapping)) - # TODO(Ziinc): make virtual once cluster is using encrypted fields fully - field(:config, :map) + field(:config, :map, virtual: true) field(:config_encrypted, Logflare.Ecto.EncryptedMap) many_to_many(:sources, Source, join_through: "sources_backends") belongs_to(:user, User) @@ -41,17 +40,15 @@ defmodule Logflare.Backends.Backend do |> cast(attrs, [:type, :config, :user_id, :name, :description, :metadata]) |> validate_required([:user_id, :type, :config, :name]) |> validate_inclusion(:type, Map.keys(@adaptor_mapping)) - |> do_config_change() |> validate_config() + |> do_config_change() end # temp function defp do_config_change(%Ecto.Changeset{changes: %{config: config}} = changeset) do changeset |> put_change(:config_encrypted, config) - - # TODO(Ziinc): uncomment once cluster is using encrypted fields fully - # |> delete_change(:config) + |> delete_change(:config) end defp do_config_change(changeset), do: changeset @@ -80,7 +77,9 @@ defmodule Logflare.Backends.Backend do type = value.type values = - Map.take(value, [ + value + |> Map.put(:config, value.config_encrypted) + |> Map.take([ :name, :token, :description, diff --git a/priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs b/priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs new file mode 100644 index 000000000..b89e833a2 --- /dev/null +++ b/priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs @@ -0,0 +1,30 @@ +defmodule Logflare.Repo.Migrations.NilifyConfigColumnForBackendsTable do + use Ecto.Migration + import Ecto.Query + alias Logflare.Ecto.EncryptedMap + + def up do + from(b in "backends", update: [set: [config: nil]]) + |> Logflare.Repo.update_all([]) + end + + def down do + {:ok, pid} = Logflare.Vault.start_link() + + # copy configs over + Logflare.Repo.all(from b in "backends", select: [:id, :config_encrypted]) + |> Enum.each(fn %{id: id} = backend -> + {:ok, config} = EncryptedMap.load(backend.config_encrypted) + + from(b in "backends", + where: b.id == ^id, + update: [set: [config: ^config]] + ) + |> Logflare.Repo.update_all([]) + end) + # stop the vault + Process.unlink(pid) + Process.exit(pid, :kill) + :timer.sleep(100) + end +end diff --git a/test/logflare/backends_test.exs b/test/logflare/backends_test.exs index 033920570..78620dce0 100644 --- a/test/logflare/backends_test.exs +++ b/test/logflare/backends_test.exs @@ -24,8 +24,6 @@ defmodule Logflare.BackendsTest do end describe "encryption" do - # TODO(Ziinc): unskip once cluster is using encrypted fields fully - @tag :skip test "backend config is encrypted to the :config_encrypted field" do insert(:backend, config_encrypted: %{some_value: "testing"}) diff --git a/test/logflare/vault_test.exs b/test/logflare/vault_test.exs index 0d97aeedd..365526a7b 100644 --- a/test/logflare/vault_test.exs +++ b/test/logflare/vault_test.exs @@ -75,8 +75,7 @@ defmodule Logflare.VaultTest do defp get_config_encrypted() do [ %{ - # TODO(Ziinc): to uncomment once fully migrated over - # config: nil, + config: nil, config_encrypted: encrypted_str } ] = Repo.all(from b in "backends", select: [:config, :config_encrypted])