From 3c91a839425b18a3e2d5f9756cc9e3e324596f7d Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Thu, 10 Oct 2024 21:41:09 -0400 Subject: [PATCH 1/4] feat(secrets): add Doppler adapter --- lib/kamal/secrets/adapters/doppler.rb | 28 ++++++++ test/secrets/doppler_adapter_test.rb | 100 ++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 lib/kamal/secrets/adapters/doppler.rb create mode 100644 test/secrets/doppler_adapter_test.rb diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb new file mode 100644 index 000000000..caa1833ed --- /dev/null +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -0,0 +1,28 @@ +class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base + private + def login(account) + unless loggedin?(account) + `doppler login -y` + raise RuntimeError, "Failed to login to Doppler" unless $?.success? + end + end + + def loggedin?(account) + `doppler me --json 2> /dev/null` + $?.success? + end + + def fetch_secrets(secrets, account:, session:) + project, config = account.split("/") + + raise RuntimeError, "Missing project or config from --acount=project/config option" unless project && config + raise RuntimeError, "Using --from option or FOLDER/SECRET is not supported by Doppler" if secrets.any?(/\//) + + items = `doppler secrets get #{secrets.map(&:shellescape).join(" ")} --json -p #{project} -c #{config}` + raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? + + items = JSON.parse(items) + + items.transform_values { |value| value["computed"] } + end +end diff --git a/test/secrets/doppler_adapter_test.rb b/test/secrets/doppler_adapter_test.rb new file mode 100644 index 000000000..c7cda4946 --- /dev/null +++ b/test/secrets/doppler_adapter_test.rb @@ -0,0 +1,100 @@ +require "test_helper" + +class DopplerAdapterTest < SecretAdapterTestCase + setup do + `true` # Ensure $? is 0 + end + + test "fetch" do + stub_ticks.with("doppler me --json 2> /dev/null") + + stub_ticks + .with("doppler secrets get SECRET1 FSECRET1 FSECRET2 --json -p my-project -c prd") + .returns(<<~JSON) + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET1": { + "computed":"fsecret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET2": { + "computed":"fsecret2", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "SECRET1", "FSECRET1", "FSECRET2"))) + + expected_json = { + "SECRET1"=>"secret1", + "FSECRET1"=>"fsecret1", + "FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + end + + test "fetch with from" do + stub_ticks.with("doppler me --json 2> /dev/null") + + error = assert_raises RuntimeError do + run_command("fetch", "--from", "FOLDER1", "FSECRET1", "FSECRET2") + end + + assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + end + + test "fetch with folder in secret" do + stub_ticks.with("doppler me --json 2> /dev/null") + + error = assert_raises RuntimeError do + run_command("fetch", "FOLDER1/FSECRET1", "SECRET2") + end + + assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + end + + test "fetch with signin" do + stub_ticks_with("doppler me --json 2> /dev/null", succeed: false) + stub_ticks_with("doppler login -y", succeed: true).returns("") + stub_ticks.with("doppler secrets get SECRET1 --json -p my-project -c prd").returns(single_item_json) + + json = JSON.parse(shellunescape(run_command("fetch", "SECRET1"))) + + expected_json = { + "SECRET1"=>"secret1" + } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "doppler", + "--account", "my-project/prd" ] + end + end + + def single_item_json + <<~JSON + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + end +end From 77cd29f5ad3abddbbe46faef19b01e89892a13c9 Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Mon, 4 Nov 2024 18:58:18 -0400 Subject: [PATCH 2/4] feat(cli): update secrets --account flag as optional depending on adapter --- lib/kamal/cli/secrets.rb | 12 +++++++++--- lib/kamal/secrets/adapters/base.rb | 9 ++++++++- .../secrets/adapters/test_optional_account.rb | 18 ++++++++++++++++++ test/cli/secrets_test.rb | 12 ++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 lib/kamal/secrets/adapters/test_optional_account.rb diff --git a/lib/kamal/cli/secrets.rb b/lib/kamal/cli/secrets.rb index b094be466..0cfcf628e 100644 --- a/lib/kamal/cli/secrets.rb +++ b/lib/kamal/cli/secrets.rb @@ -1,11 +1,17 @@ class Kamal::Cli::Secrets < Kamal::Cli::Base desc "fetch [SECRETS...]", "Fetch secrets from a vault" option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use" - option :account, type: :string, required: true, desc: "The account identifier or username" + option :account, type: :string, required: false, desc: "The account identifier or username" option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from" option :inline, type: :boolean, required: false, hidden: true def fetch(*secrets) - results = adapter(options[:adapter]).fetch(secrets, **options.slice(:account, :from).symbolize_keys) + adapter = initialize_adapter(options[:adapter]) + + if adapter.requires_account? && options[:account].blank? + return puts "No value provided for required options '--account'" + end + + results = adapter.fetch(secrets, **options.slice(:account, :from).symbolize_keys) return_or_puts JSON.dump(results).shellescape, inline: options[:inline] end @@ -29,7 +35,7 @@ def print end private - def adapter(adapter) + def initialize_adapter(adapter) Kamal::Secrets::Adapters.lookup(adapter) end diff --git a/lib/kamal/secrets/adapters/base.rb b/lib/kamal/secrets/adapters/base.rb index 579414aff..fc66bb34d 100644 --- a/lib/kamal/secrets/adapters/base.rb +++ b/lib/kamal/secrets/adapters/base.rb @@ -1,13 +1,20 @@ class Kamal::Secrets::Adapters::Base delegate :optionize, to: Kamal::Utils - def fetch(secrets, account:, from: nil) + def fetch(secrets, account: nil, from: nil) + raise RuntimeError, "Missing required option '--account'" if requires_account? && account.blank? + check_dependencies! + session = login(account) full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") } fetch_secrets(full_secrets, account: account, session: session) end + def requires_account? + true + end + private def login(...) raise NotImplementedError diff --git a/lib/kamal/secrets/adapters/test_optional_account.rb b/lib/kamal/secrets/adapters/test_optional_account.rb new file mode 100644 index 000000000..1f85302f7 --- /dev/null +++ b/lib/kamal/secrets/adapters/test_optional_account.rb @@ -0,0 +1,18 @@ +class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + + private + def login(account) + true + end + + def fetch_secrets(secrets, account:, session:) + secrets.to_h { |secret| [ secret, secret.reverse ] } + end + + def check_dependencies! + # no op + end +end diff --git a/test/cli/secrets_test.rb b/test/cli/secrets_test.rb index 6014a7e72..bd412862f 100644 --- a/test/cli/secrets_test.rb +++ b/test/cli/secrets_test.rb @@ -7,6 +7,18 @@ class CliSecretsTest < CliTestCase run_command("fetch", "foo", "bar", "baz", "--account", "myaccount", "--adapter", "test") end + test "fetch missing --acount" do + assert_equal \ + "No value provided for required options '--account'", + run_command("fetch", "foo", "bar", "baz", "--adapter", "test") + end + + test "fetch without required --account" do + assert_equal \ + "\\{\\\"foo\\\":\\\"oof\\\",\\\"bar\\\":\\\"rab\\\",\\\"baz\\\":\\\"zab\\\"\\}", + run_command("fetch", "foo", "bar", "baz", "--adapter", "test_optional_account") + end + test "extract" do assert_equal "oof", run_command("extract", "foo", "{\"foo\":\"oof\", \"bar\":\"rab\", \"baz\":\"zab\"}") end From 3069552315620af71b1540f63d093164346e2e0f Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Mon, 4 Nov 2024 19:00:38 -0400 Subject: [PATCH 3/4] feat(secrets): update doppler adapter to use --from option and DOPPLER_TOKEN env --- lib/kamal/secrets/adapters/doppler.rb | 41 ++++++++-- test/secrets/doppler_adapter_test.rb | 108 +++++++++++++++++++++++--- 2 files changed, 130 insertions(+), 19 deletions(-) diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb index caa1833ed..64d644f7e 100644 --- a/lib/kamal/secrets/adapters/doppler.rb +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -1,28 +1,53 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + private - def login(account) - unless loggedin?(account) + def login(*) + unless loggedin? `doppler login -y` raise RuntimeError, "Failed to login to Doppler" unless $?.success? end end - def loggedin?(account) + def loggedin? `doppler me --json 2> /dev/null` $?.success? end - def fetch_secrets(secrets, account:, session:) - project, config = account.split("/") + def fetch_secrets(secrets, **) + project_and_config_flags = "" + unless service_token_set? + project, config, _ = secrets.first.split("/") + + unless project && config + raise RuntimeError, "Missing project or config from '--from=project/config' option" + end + + project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}" + end - raise RuntimeError, "Missing project or config from --acount=project/config option" unless project && config - raise RuntimeError, "Using --from option or FOLDER/SECRET is not supported by Doppler" if secrets.any?(/\//) + secret_names = secrets.collect { |s| s.split("/").last } - items = `doppler secrets get #{secrets.map(&:shellescape).join(" ")} --json -p #{project} -c #{config}` + items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{project_and_config_flags}` raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? items = JSON.parse(items) items.transform_values { |value| value["computed"] } end + + def service_token_set? + ENV["DOPPLER_TOKEN"] && ENV["DOPPLER_TOKEN"][0, 5] == "dp.st" + end + + def check_dependencies! + raise RuntimeError, "Doppler CLI is not installed" unless cli_installed? + end + + def cli_installed? + `doppler --version 2> /dev/null` + $?.success? + end end diff --git a/test/secrets/doppler_adapter_test.rb b/test/secrets/doppler_adapter_test.rb index c7cda4946..c2b164682 100644 --- a/test/secrets/doppler_adapter_test.rb +++ b/test/secrets/doppler_adapter_test.rb @@ -6,6 +6,7 @@ class DopplerAdapterTest < SecretAdapterTestCase end test "fetch" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks.with("doppler me --json 2> /dev/null") stub_ticks @@ -30,7 +31,9 @@ class DopplerAdapterTest < SecretAdapterTestCase } JSON - json = JSON.parse(shellunescape(run_command("fetch", "SECRET1", "FSECRET1", "FSECRET2"))) + json = JSON.parse( + shellunescape run_command("fetch", "--from", "my-project/prd", "SECRET1", "FSECRET1", "FSECRET2") + ) expected_json = { "SECRET1"=>"secret1", @@ -41,32 +44,106 @@ class DopplerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end - test "fetch with from" do + test "fetch having DOPPLER_TOKEN" do + ENV["DOPPLER_TOKEN"] = "dp.st.xxxxxxxxxxxxxxxxxxxxxx" + + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks.with("doppler me --json 2> /dev/null") - error = assert_raises RuntimeError do - run_command("fetch", "--from", "FOLDER1", "FSECRET1", "FSECRET2") - end + stub_ticks + .with("doppler secrets get SECRET1 FSECRET1 FSECRET2 --json ") + .returns(<<~JSON) + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET1": { + "computed":"fsecret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET2": { + "computed":"fsecret2", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + + json = JSON.parse( + shellunescape run_command("fetch", "SECRET1", "FSECRET1", "FSECRET2") + ) + + expected_json = { + "SECRET1"=>"secret1", + "FSECRET1"=>"fsecret1", + "FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json - assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + ENV.delete("DOPPLER_TOKEN") end test "fetch with folder in secret" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) + stub_ticks.with("doppler me --json 2> /dev/null") + + stub_ticks + .with("doppler secrets get SECRET1 FSECRET1 FSECRET2 --json -p my-project -c prd") + .returns(<<~JSON) + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET1": { + "computed":"fsecret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET2": { + "computed":"fsecret2", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + + json = JSON.parse( + shellunescape run_command("fetch", "my-project/prd/SECRET1", "my-project/prd/FSECRET1", "my-project/prd/FSECRET2") + ) + + expected_json = { + "SECRET1"=>"secret1", + "FSECRET1"=>"fsecret1", + "FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + end + + test "fetch without --from" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks.with("doppler me --json 2> /dev/null") error = assert_raises RuntimeError do - run_command("fetch", "FOLDER1/FSECRET1", "SECRET2") + run_command("fetch", "FSECRET1", "FSECRET2") end - assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + assert_equal "Missing project or config from '--from=project/config' option", error.message end test "fetch with signin" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks_with("doppler me --json 2> /dev/null", succeed: false) stub_ticks_with("doppler login -y", succeed: true).returns("") stub_ticks.with("doppler secrets get SECRET1 --json -p my-project -c prd").returns(single_item_json) - json = JSON.parse(shellunescape(run_command("fetch", "SECRET1"))) + json = JSON.parse(shellunescape(run_command("fetch", "--from", "my-project/prd", "SECRET1"))) expected_json = { "SECRET1"=>"secret1" @@ -75,14 +152,23 @@ class DopplerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end + test "fetch without CLI installed" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "HOST", "PORT"))) + end + + assert_equal "Doppler CLI is not installed", error.message + end + private def run_command(*command) stdouted do Kamal::Cli::Secrets.start \ [ *command, "-c", "test/fixtures/deploy_with_accessories.yml", - "--adapter", "doppler", - "--account", "my-project/prd" ] + "--adapter", "doppler" ] end end From 8dd864af89e8e6875a783357310a4200b4a014ae Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Tue, 5 Nov 2024 14:14:18 -0400 Subject: [PATCH 4/4] refactor(secrets): adapter/test_optional_account inherit from adapter/test --- .../secrets/adapters/test_optional_account.rb | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/lib/kamal/secrets/adapters/test_optional_account.rb b/lib/kamal/secrets/adapters/test_optional_account.rb index 1f85302f7..3a252e682 100644 --- a/lib/kamal/secrets/adapters/test_optional_account.rb +++ b/lib/kamal/secrets/adapters/test_optional_account.rb @@ -1,18 +1,5 @@ -class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Base +class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Test def requires_account? false end - - private - def login(account) - true - end - - def fetch_secrets(secrets, account:, session:) - secrets.to_h { |secret| [ secret, secret.reverse ] } - end - - def check_dependencies! - # no op - end end