From c3b38e1dceda433c2d3a42907d4ea08888c2cba9 Mon Sep 17 00:00:00 2001 From: TzeYiing Date: Fri, 9 Aug 2024 01:21:40 +0800 Subject: [PATCH 1/2] feat: remove config field usage --- docs/docs.logflare.com/docs/self-hosting/index.md | 4 +++- lib/logflare/backends/backend.ex | 13 ++++++------- test/logflare/backends_test.exs | 2 -- test/logflare/vault_test.exs | 3 +-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/docs.logflare.com/docs/self-hosting/index.md b/docs/docs.logflare.com/docs/self-hosting/index.md index 2ff8a9408..14d561d0e 100644 --- a/docs/docs.logflare.com/docs/self-hosting/index.md +++ b/docs/docs.logflare.com/docs/self-hosting/index.md @@ -65,6 +65,8 @@ Encryption keys must be Base64 encoded. Cipher used is AES with a 256-bit key in GCM mode. +If `LOGFLARE_DB_ENCRYPTION_KEY` environement variable is not provided, a default hardcoded encryption key will be used. + ### Rolling Encryption Keys In order to roll encryption keys and migrate existing encrypted data, use the `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED` environment variable. @@ -75,7 +77,7 @@ Steps to perform the migration are: 2. Generate a new encryption key and set it to `LOGFLARE_DB_ENCRYPTION_KEY`. 3. Restart or deploy the server with the new environment variables. 4. Upon successful server startup, an `info` log will be emitted that says that an retired encryption key is detected, and the migration will be initiated to transition all data encrypted with the retired key to be encrypted with the new key. -5. Once the migration is complete, the retired encryption key can be safely removed. +5. Once the migration is complete, the retired encryption key can be safely removed. There will be an `info` log that will be emitted once the migration is complete. ## BigQuery Setup diff --git a/lib/logflare/backends/backend.ex b/lib/logflare/backends/backend.ex index d7171c02a..358a1f31e 100644 --- a/lib/logflare/backends/backend.ex +++ b/lib/logflare/backends/backend.ex @@ -23,8 +23,7 @@ defmodule Logflare.Backends.Backend do field(:description, :string) field(:token, Ecto.UUID, autogenerate: true) field(:type, Ecto.Enum, values: Map.keys(@adaptor_mapping)) - # TODO(Ziinc): make virtual once cluster is using encrypted fields fully - field(:config, :map) + field(:config, :map, virtual: true) field(:config_encrypted, Logflare.Ecto.EncryptedMap) many_to_many(:sources, Source, join_through: "sources_backends") belongs_to(:user, User) @@ -41,17 +40,15 @@ defmodule Logflare.Backends.Backend do |> cast(attrs, [:type, :config, :user_id, :name, :description, :metadata]) |> validate_required([:user_id, :type, :config, :name]) |> validate_inclusion(:type, Map.keys(@adaptor_mapping)) - |> do_config_change() |> validate_config() + |> do_config_change() end # temp function defp do_config_change(%Ecto.Changeset{changes: %{config: config}} = changeset) do changeset |> put_change(:config_encrypted, config) - - # TODO(Ziinc): uncomment once cluster is using encrypted fields fully - # |> delete_change(:config) + |> delete_change(:config) end defp do_config_change(changeset), do: changeset @@ -80,7 +77,9 @@ defmodule Logflare.Backends.Backend do type = value.type values = - Map.take(value, [ + value + |> Map.put(:config, value.config_encrypted) + |> Map.take([ :name, :token, :description, diff --git a/test/logflare/backends_test.exs b/test/logflare/backends_test.exs index 033920570..78620dce0 100644 --- a/test/logflare/backends_test.exs +++ b/test/logflare/backends_test.exs @@ -24,8 +24,6 @@ defmodule Logflare.BackendsTest do end describe "encryption" do - # TODO(Ziinc): unskip once cluster is using encrypted fields fully - @tag :skip test "backend config is encrypted to the :config_encrypted field" do insert(:backend, config_encrypted: %{some_value: "testing"}) diff --git a/test/logflare/vault_test.exs b/test/logflare/vault_test.exs index 0d97aeedd..365526a7b 100644 --- a/test/logflare/vault_test.exs +++ b/test/logflare/vault_test.exs @@ -75,8 +75,7 @@ defmodule Logflare.VaultTest do defp get_config_encrypted() do [ %{ - # TODO(Ziinc): to uncomment once fully migrated over - # config: nil, + config: nil, config_encrypted: encrypted_str } ] = Repo.all(from b in "backends", select: [:config, :config_encrypted]) From 39eabc0230d9ae10db408e0d3308a81b7b98fbcd Mon Sep 17 00:00:00 2001 From: TzeYiing Date: Fri, 9 Aug 2024 01:32:40 +0800 Subject: [PATCH 2/2] feat: nilify config columns migration --- ...ilify_config_column_for_backends_table.exs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs diff --git a/priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs b/priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs new file mode 100644 index 000000000..b89e833a2 --- /dev/null +++ b/priv/repo/migrations/20240808172408_nilify_config_column_for_backends_table.exs @@ -0,0 +1,30 @@ +defmodule Logflare.Repo.Migrations.NilifyConfigColumnForBackendsTable do + use Ecto.Migration + import Ecto.Query + alias Logflare.Ecto.EncryptedMap + + def up do + from(b in "backends", update: [set: [config: nil]]) + |> Logflare.Repo.update_all([]) + end + + def down do + {:ok, pid} = Logflare.Vault.start_link() + + # copy configs over + Logflare.Repo.all(from b in "backends", select: [:id, :config_encrypted]) + |> Enum.each(fn %{id: id} = backend -> + {:ok, config} = EncryptedMap.load(backend.config_encrypted) + + from(b in "backends", + where: b.id == ^id, + update: [set: [config: ^config]] + ) + |> Logflare.Repo.update_all([]) + end) + # stop the vault + Process.unlink(pid) + Process.exit(pid, :kill) + :timer.sleep(100) + end +end