From 7f15fd143f7bae9298510d79ca9ca401fe4d7394 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 17 Sep 2024 12:55:11 +0100 Subject: [PATCH] Upgrade commands for Kamal 1.x -> 2.0 Adds: - `kamal upgrade` to upgrade all app hosts and accessory hosts - `kamal proxy upgrade` to upgrade the proxy on all hosts - `kamal accessory upgrade [name]` to upgrade accessories on all hosts Upgrade takes rolling and confirmed options and calls `proxy upgrade` and `accessory upgrade` in turn. To just upgrade a single host add -h [host] to the command. But the upgrade should run on all hosts, not just those running the proxy. Calling upgrade on a host that has already been upgraded should work ok. Upgrading hosts causes downtime but you can avoid if you run multiple hosts by: 1. Implementing the pre-proxy-reboot and post-proxy-reboot hooks to remove the host from external load balancers 2. Running the upgrade with the --rolling option **kamal proxy upgrade** 1. Creates a `kamal` network if required 2. Stops and removes the old proxy (whether Traefik or kamal-proxy) 3. Starts a kamal-proxy container in the `kamal` network 4. Reboots the app containers in the `kamal` network **kamal accessory upgrade [name]** 1. Creates a `kamal` network if required 2. Reboots the accessory containers in the `kamal` network A matching `downgrade` command will be added to Kamal 1.9. --- lib/kamal/cli/accessory.rb | 23 +++++++++++ lib/kamal/cli/base.rb | 4 ++ lib/kamal/cli/main.rb | 31 +++++++++++++++ lib/kamal/cli/proxy.rb | 13 ++---- lib/kamal/commander.rb | 7 ++++ test/cli/accessory_test.rb | 21 ++++++++++ test/cli/main_test.rb | 28 +++++++++++++ test/cli/proxy_test.rb | 61 +++++++++++++++++++++++++++++ test/fixtures/deploy_with_proxy.yml | 1 + 9 files changed, 180 insertions(+), 9 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index bc84fe522..e6dc78cc2 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -218,6 +218,29 @@ def remove_service_directory(name) end end + desc "upgrade", "Upgrade accessories from Kamal 1.x to 2.0 (restart them in 'kamal' network)" + option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time" + option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" + def upgrade(name) + confirming "This will restart all accessories" do + with_lock do + if options[:rolling] + KAMAL.accessory_hosts.each do |host| + say "Upgrading accessories on #{host}...", :magenta + KAMAL.with_specific_hosts(host) do + reboot name + end + say "Upgraded accessories on #{host}...", :magenta + end + else + say "Upgrading accessories on all hosts...", :magenta + reboot name + say "Upgraded accessories on all hosts", :magenta + end + end + end + end + private def with_accessory(name) if KAMAL.config.accessory(name) diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index b3af23cce..f594c54e1 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -174,6 +174,10 @@ def first_invocation instance_variable_get("@_invocations").first end + def reset_invocation(cli_class) + instance_variable_get("@_invocations")[cli_class].pop + end + def ensure_run_directory on(KAMAL.hosts) do execute(*KAMAL.server.ensure_run_directory) diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 46ce1583c..3a56be4c4 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -189,6 +189,37 @@ def remove end end + desc "upgrade", "Upgrade from Kamal 1.x to 2.0" + option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" + option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time" + def upgrade + confirming "This will replace Traefik with kamal-proxy and restart all accessories" do + with_lock do + if options[:rolling] + (KAMAL.hosts | KAMAL.accessory_hosts).each do |host| + KAMAL.with_specific_hosts(host) do + say "Upgrading #{host}...", :magenta + if KAMAL.hosts.include?(host) + invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false) + reset_invocation(Kamal::Cli::Proxy) + end + if KAMAL.accessory_hosts.include?(host) + invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true, rolling: false) + reset_invocation(Kamal::Cli::Accessory) + end + say "Upgraded #{host}", :magenta + end + end + else + say "Upgrading all hosts...", :magenta + invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true) + invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true) + say "Upgraded all hosts", :magenta + end + end + end + end + desc "version", "Show Kamal version" def version puts Kamal::VERSION diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index 108d78552..0e6a74aeb 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -62,7 +62,7 @@ def reboot end end - desc "upgrade", "Upgrade to correct proxy on servers (stop container, remove container, start new container)" + desc "upgrade", "Upgrade to kamal-proxy on servers (stop container, remove container, start new container, reboot app)" option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel" option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" def upgrade @@ -72,6 +72,7 @@ def upgrade host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ] host_groups.each do |hosts| host_list = Array(hosts).join(",") + say "Upgrading proxy on #{host_list}..." run_hook "pre-proxy-reboot", hosts: host_list on(hosts) do |host| execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug @@ -86,19 +87,17 @@ def upgrade execute *KAMAL.proxy.remove_image end - begin - old_hosts, KAMAL.specific_hosts = KAMAL.specific_hosts, hosts + KAMAL.with_specific_hosts(hosts) do invoke "kamal:cli:proxy:boot", [], invoke_options reset_invocation(Kamal::Cli::Proxy) invoke "kamal:cli:app:boot", [], invoke_options reset_invocation(Kamal::Cli::App) invoke "kamal:cli:prune:all", [], invoke_options reset_invocation(Kamal::Cli::Prune) - ensure - KAMAL.specific_hosts = old_hosts end run_hook "post-proxy-reboot", hosts: host_list + say "Upgraded proxy on #{host_list}" end end end @@ -204,10 +203,6 @@ def remove_host_directory end private - def reset_invocation(cli_class) - instance_variable_get("@_invocations")[cli_class].pop - end - def removal_allowed?(force) on(KAMAL.proxy_hosts) do |host| app_count = capture_with_info(*KAMAL.server.app_directory_count).chomp.to_i diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index c07e89331..1557df57b 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -65,6 +65,13 @@ def specific_hosts=(hosts) end end + def with_specific_hosts(hosts) + original_hosts, self.specific_hosts = specific_hosts, hosts + yield + ensure + self.specific_hosts = original_hosts + end + def accessory_names config.accessories&.collect(&:name) || [] end diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index 5bb8762a8..072962e17 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -220,6 +220,27 @@ class CliAccessoryTest < CliTestCase end end + test "upgrade" do + run_command("upgrade", "-y", "all").tap do |output| + assert_match "Upgrading accessories on all hosts...", output + assert_match "docker network create kamal on 1.1.1.3", output + assert_match "docker container stop app-mysql on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "Upgraded accessories on all hosts", output + end + end + + test "upgrade rolling" do + run_command("upgrade", "--rolling", "-y", "all").tap do |output| + assert_match "Upgrading accessories on 1.1.1.3...", output + assert_match "docker network create kamal on 1.1.1.3", output + assert_match "docker container stop app-mysql on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "Upgraded accessories on 1.1.1.3", output + end + end + + private def run_command(*command) stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 607fee21b..60fdf5c74 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -486,6 +486,34 @@ class CliMainTest < CliTestCase end end + test "upgrade" do + invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:upgrade", [ "all" ], invoke_options) + + run_command("upgrade", "-y", config_file: "deploy_with_accessories").tap do |output| + assert_match "Upgrading all hosts...", output + assert_match "Upgraded all hosts", output + end + end + + test "upgrade rolling" do + invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options).times(4) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:upgrade", [ "all" ], invoke_options).times(4) + + run_command("upgrade", "--rolling", "-y", config_file: "deploy_with_accessories").tap do |output| + assert_match "Upgrading 1.1.1.1...", output + assert_match "Upgraded 1.1.1.1", output + assert_match "Upgrading 1.1.1.2...", output + assert_match "Upgraded 1.1.1.2", output + assert_match "Upgrading 1.1.1.3...", output + assert_match "Upgraded 1.1.1.3", output + assert_match "Upgrading 1.1.1.4...", output + assert_match "Upgraded 1.1.1.4", output + end + end + private def run_command(*command, config_file: "deploy_simple") with_argv([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) do diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index 2a0834fc9..975200717 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -182,6 +182,67 @@ class CliProxyTest < CliTestCase end end + test "upgrade" do + Object.any_instance.stubs(:sleep) + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678") + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2") + .returns("v0.1.0") + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running").at_least_once # workers health check + + run_command("upgrade", "-y").tap do |output| + assert_match "Upgrading proxy on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4...", output + assert_match "docker login -u [REDACTED] -p [REDACTED]", output + assert_match "docker container stop traefik ; docker container prune --force --filter label=org.opencontainers.image.title=Traefik && docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik", output + assert_match "docker container stop kamal-proxy", output + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output + assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output + assert_match "/usr/bin/env mkdir -p .kamal", output + assert_match "docker network create kamal", output + assert_match "docker login -u [REDACTED] -p [REDACTED]", output + assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/proxy/config:/home/kamal-proxy/.config/kamal-proxy --log-opt max-size=\"10m\" basecamp/kamal-proxy:v0.1.0", output + assert_match "/usr/bin/env mkdir -p .kamal", output + assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output + assert_match "/usr/bin/env mkdir -p .kamal/apps/app/env/roles", output + assert_match %r{/usr/bin/env .* .kamal/apps/app/env/roles/web.env}, output + assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-.* -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size="10m" --label service="app" --label role="web" --label destination dhh/app:latest}, output + assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target \"12345678:80\" --deploy-timeout \"6s\" --buffer-requests --buffer-responses --log-request-header \"Cache-Control\" --log-request-header \"Last-Modified\" --log-request-header \"User-Agent\"", output + assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output + assert_match "docker tag dhh/app:latest dhh/app:latest", output + assert_match "/usr/bin/env mkdir -p .kamal", output + assert_match "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", output + assert_match "docker image prune --force --filter label=service=app", output + assert_match "Upgraded proxy on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", output + end + end + + test "upgrade rolling" do + Object.any_instance.stubs(:sleep) + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678") + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2") + .returns("v0.1.0") + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running").at_least_once # workers health check + + run_command("upgrade", "--rolling", "-y",).tap do |output| + %w[1.1.1.1 1.1.1.2 1.1.1.3 1.1.1.4].each do |host| + assert_match "Upgrading proxy on #{host}...", output + assert_match "docker container stop traefik ; docker container prune --force --filter label=org.opencontainers.image.title=Traefik && docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik on #{host}", output + assert_match "Upgraded proxy on #{host}", output + end + end + end + private def run_command(*command, fixture: :with_proxy) stdouted { Kamal::Cli::Proxy.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } diff --git a/test/fixtures/deploy_with_proxy.yml b/test/fixtures/deploy_with_proxy.yml index e91e9657a..a1f7a8537 100644 --- a/test/fixtures/deploy_with_proxy.yml +++ b/test/fixtures/deploy_with_proxy.yml @@ -39,3 +39,4 @@ accessories: - data:/data readiness_delay: 0 +readiness_timeout: 1