diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb index 4892acd24..9c34e8dd7 100644 --- a/lib/kamal/cli/app.rb +++ b/lib/kamal/cli/app.rb @@ -18,8 +18,9 @@ def boot roles.each do |role| app = KAMAL.app(role: role) auditor = KAMAL.auditor(role: role) + role_config = KAMAL.config.role(role) - if capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).present? + if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present? tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}" info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}" execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug @@ -29,11 +30,25 @@ def boot execute *auditor.record("Booted app version #{version}"), verbosity: :debug old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip - execute *app.start_or_run(hostname: "#{host}-#{SecureRandom.hex(6)}") + + if role_config.uses_cord? + execute *app.tie_cord(role_config.cord_host_file) + end + + execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}") Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) } - execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? + if old_version.present? + if role_config.uses_cord? + cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip + if cord.present? + execute *app.cut_cord(cord) + Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) } + end + end + execute *app.stop(version: old_version), raise_on_non_zero_exit: false + end end end end diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb index cb1f7091c..7fac2e170 100644 --- a/lib/kamal/commands/app.rb +++ b/lib/kamal/commands/app.rb @@ -1,34 +1,29 @@ class Kamal::Commands::App < Kamal::Commands::Base ACTIVE_DOCKER_STATUSES = [ :running, :restarting ] - attr_reader :role + attr_reader :role, :role_config def initialize(config, role: nil) super(config) @role = role - end - - def start_or_run(hostname: nil) - combine start, run(hostname: hostname), by: "||" + @role_config = config.role(self.role) end def run(hostname: nil) - role = config.role(self.role) - docker :run, "--detach", "--restart unless-stopped", "--name", container_name, *(["--hostname", hostname] if hostname), "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"", - *role.env_args, - *role.health_check_args, + *role_config.env_args, + *role_config.health_check_args, *config.logging_args, *config.volume_args, - *role.label_args, - *role.option_args, + *role_config.label_args, + *role_config.option_args, config.absolute_image, - role.cmd + role_config.cmd end def start @@ -76,14 +71,12 @@ def execute_in_existing_container(*command, interactive: false) end def execute_in_new_container(*command, interactive: false) - role = config.role(self.role) - docker :run, ("-it" if interactive), "--rm", - *role&.env_args, + *role_config&.env_args, *config.volume_args, - *role&.option_args, + *role_config&.option_args, config.absolute_image, *command end @@ -112,7 +105,7 @@ def current_running_version def list_versions(*docker_args, statuses: nil) pipe \ docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'), - %(while read line; do echo ${line##{service_role_dest}-}; done) # Extract SHA from "service-role-dest-SHA" + %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA" end def list_containers @@ -150,16 +143,30 @@ def tag_current_as_latest end def make_env_directory - make_directory config.role(role).host_env_directory + make_directory role_config.host_env_directory end def remove_env_file - [:rm, "-f", config.role(role).host_env_file_path] + [:rm, "-f", role_config.host_env_file_path] + end + + def cord(version:) + pipe \ + docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)), + [:awk, "'$2 == \"#{role_config.cord_container_directory}\" {print $1}'"] + end + + def tie_cord(cord) + create_empty_file(cord) + end + + def cut_cord(cord) + remove_directory(cord) end private def container_name(version = nil) - [ config.service, role, config.destination, version || config.version ].compact.join("-") + [ role_config.full_name, version || config.version ].compact.join("-") end def filter_args(statuses: nil) diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb index 3058df162..ff31c7475 100644 --- a/lib/kamal/commands/base.rb +++ b/lib/kamal/commands/base.rb @@ -34,6 +34,10 @@ def make_directory(path) [ :mkdir, "-p", path ] end + def remove_directory(path) + [ :rm, "-r", path ] + end + private def combine(*commands, by: "&&") commands @@ -69,5 +73,11 @@ def docker(*args) def tags(**details) Kamal::Tags.from_config(config, **details) end + + def create_empty_file(file) + chain \ + make_directory_for(file), + [:touch, file] + end end end diff --git a/lib/kamal/commands/healthcheck.rb b/lib/kamal/commands/healthcheck.rb index 4327087dd..fa050b9cb 100644 --- a/lib/kamal/commands/healthcheck.rb +++ b/lib/kamal/commands/healthcheck.rb @@ -10,7 +10,7 @@ def run "--label", "service=#{container_name}", "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"", *web.env_args, - *web.health_check_args, + *web.health_check_args(cord: false), *config.volume_args, *web.option_args, config.absolute_image, diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 8db892fc5..af6e46c5f 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -61,6 +61,14 @@ def run_directory raw_config.run_directory || ".kamal" end + def run_directory_as_docker_volume + if Pathname.new(run_directory).absolute? + run_directory + else + File.join "$(pwd)", run_directory + end + end + def roles @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) } @@ -141,7 +149,7 @@ def sshkit def healthcheck - { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999 }.merge(raw_config.healthcheck || {}) + { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {}) end def readiness_delay @@ -199,6 +207,10 @@ def host_env_directory "#{run_directory}/env" end + def run_id + @run_id ||= SecureRandom.hex(16) + end + private # Will raise ArgumentError if any required config keys are missing def ensure_required_keys_present diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb index f549d459b..6c57bc1dc 100644 --- a/lib/kamal/configuration/role.rb +++ b/lib/kamal/configuration/role.rb @@ -1,4 +1,5 @@ class Kamal::Configuration::Role + CORD_FILE = "cord" delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils attr_accessor :name @@ -47,28 +48,52 @@ def env_args argumentize "--env-file", host_env_file_path end - def health_check_args + def health_check_args(cord: true) if health_check_cmd.present? - optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval }) + if cord && uses_cord? + optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval }) + .concat(["--volume", "#{cord_host_directory}:#{cord_container_directory}"]) + else + optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval }) + end else [] end end def health_check_cmd - options = specializations["healthcheck"] || {} - options = config.healthcheck.merge(options) if running_traefik? + health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"]) + end - options["cmd"] || http_health_check(port: options["port"], path: options["path"]) + def health_check_cmd_with_cord + "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)" end def health_check_interval - options = specializations["healthcheck"] || {} - options = config.healthcheck.merge(options) if running_traefik? + health_check_options["interval"] || "1s" + end + + def uses_cord? + running_traefik? && cord_container_directory.present? && health_check_cmd.present? + end + + def cord_host_directory + File.join config.run_directory_as_docker_volume, "cords", [full_name, config.run_id].join("-") + end + + def cord_host_file + File.join cord_host_directory, CORD_FILE + end + + def cord_container_directory + health_check_options.fetch("cord", nil) + end - options["interval"] || "1s" + def cord_container_file + File.join cord_container_directory, CORD_FILE end + def cmd specializations["cmd"] end @@ -85,6 +110,10 @@ def running_traefik? name.web? || specializations["traefik"] end + def full_name + [ config.service, name, config.destination ].compact.join("-") + end + private attr_accessor :config @@ -164,4 +193,12 @@ def merged_env_with_secrets def http_health_check(port:, path:) "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present? end + + def health_check_options + @health_check_options ||= begin + options = specializations["healthcheck"] || {} + options = config.healthcheck.merge(options) if running_traefik? + options + end + end end diff --git a/lib/kamal/utils/healthcheck_poller.rb b/lib/kamal/utils/healthcheck_poller.rb index ddb09ec65..27a2ff7d5 100644 --- a/lib/kamal/utils/healthcheck_poller.rb +++ b/lib/kamal/utils/healthcheck_poller.rb @@ -1,5 +1,5 @@ class Kamal::Utils::HealthcheckPoller - TRAEFIK_HEALTHY_DELAY = 2 + TRAEFIK_UPDATE_DELAY = 2 class HealthcheckError < StandardError; end @@ -11,7 +11,7 @@ def wait_for_healthy(pause_after_ready: false, &block) begin case status = block.call when "healthy" - sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready + sleep TRAEFIK_UPDATE_DELAY if pause_after_ready when "running" # No health check configured sleep KAMAL.config.readiness_delay if pause_after_ready else @@ -31,6 +31,31 @@ def wait_for_healthy(pause_after_ready: false, &block) info "Container is healthy!" end + def wait_for_unhealthy(pause_after_ready: false, &block) + attempt = 1 + max_attempts = KAMAL.config.healthcheck["max_attempts"] + + begin + case status = block.call + when "unhealthy" + sleep TRAEFIK_UPDATE_DELAY if pause_after_ready + else + raise HealthcheckError, "container not unhealthy (#{status})" + end + rescue HealthcheckError => e + if attempt <= max_attempts + info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..." + sleep attempt + attempt += 1 + retry + else + raise + end + end + + info "Container is unhealthy!" + end + private def info(message) SSHKit.config.output.info(message) diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 8a90e90ab..d0e6268af 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -11,10 +11,11 @@ class CliAppTest < CliTestCase end test "boot will rename if same version is already running" do + Object.any_instance.stubs(:sleep) run_command("details") # Preheat Kamal const SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false) .returns("12345678") # running version SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) @@ -25,6 +26,14 @@ class CliAppTest < CliTestCase .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false) .returns("123") # old version + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false) + .returns("cordfile") # old version + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("unhealthy") # old version unhealthy + run_command("boot").tap do |output| assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output @@ -180,10 +189,16 @@ def run_command(*command, config: :with_accessories) end def stub_running + Object.any_instance.stubs(:sleep) + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") .returns("running") # health check + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("unhealthy") # health check end end diff --git a/test/cli/healthcheck_test.rb b/test/cli/healthcheck_test.rb index f9c3aa9c0..b07c4900c 100644 --- a/test/cli/healthcheck_test.rb +++ b/test/cli/healthcheck_test.rb @@ -6,6 +6,7 @@ class CliHealthcheckTest < CliTestCase Thread.report_on_exception = false Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying + Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012") SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 04f115750..28ef97ffc 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -176,9 +176,10 @@ class CliMainTest < CliTestCase end test "rollback good version" do + Object.any_instance.stubs(:sleep) [ "web", "workers" ].each do |role| SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false) .returns("").at_least_once SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet") @@ -191,14 +192,21 @@ class CliMainTest < CliTestCase .returns("running").at_least_once # health check end + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false) + .returns("corddirectory").at_least_once # health check + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("unhealthy").at_least_once # health check + Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" } run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output| - assert_match "Start container with version 123", output assert_hook_ran "pre-deploy", output, **hook_variables assert_match "docker tag dhh/app:123 dhh/app:latest", output - assert_match "docker start app-web-123", output + assert_match "docker run --detach --restart unless-stopped --name app-web-123", output assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running" assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0" end @@ -210,7 +218,7 @@ class CliMainTest < CliTestCase Kamal::Utils::HealthcheckPoller.stubs(:sleep) SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false) .returns("").at_least_once SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false) @@ -220,8 +228,7 @@ class CliMainTest < CliTestCase .returns("running").at_least_once # health check run_command("rollback", "123").tap do |output| - assert_match "Start container with version 123", output - assert_match "docker start app-web-123 || docker run --detach --restart unless-stopped --name app-web-123", output + assert_match "docker run --detach --restart unless-stopped --name app-web-123", output assert_no_match "docker stop", output end end diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb index 1b19b0331..6c6fbf641 100644 --- a/test/cli/traefik_test.rb +++ b/test/cli/traefik_test.rb @@ -19,6 +19,8 @@ class CliTraefikTest < CliTestCase end test "reboot --rolling" do + Object.any_instance.stubs(:sleep) + run_command("reboot", "--rolling").tap do |output| assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index ab3f6ace8..7b769c6df 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -3,6 +3,7 @@ class CommandsAppTest < ActiveSupport::TestCase setup do ENV["RAILS_MASTER_KEY"] = "456" + Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012") @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } } end @@ -13,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end test "run with hostname" do assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run(hostname: "myhost").join(" ") end @@ -27,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -35,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -43,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "cmd" => "/bin/up" } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -51,7 +52,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -66,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -83,18 +84,6 @@ class CommandsAppTest < ActiveSupport::TestCase new_command.start.join(" ") end - test "start_or_run" do - assert_equal \ - "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", - new_command.start_or_run.join(" ") - end - - test "start_or_run with hostname" do - assert_equal \ - "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", - new_command.start_or_run(hostname: "myhost").join(" ") - end - test "stop" do assert_equal \ "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop", @@ -342,6 +331,20 @@ class CommandsAppTest < ActiveSupport::TestCase assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ") end + test "cord" do + assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ") + end + + test "tie cord" do + assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ") + assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ") + assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ") + end + + test "cut cord" do + assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ") + end + private def new_command(role: "web") Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role) diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb index b29ac2b53..650039794 100644 --- a/test/configuration/role_test.rb +++ b/test/configuration/role_test.rb @@ -175,4 +175,24 @@ class ConfigurationRoleTest < ActiveSupport::TestCase assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path end + test "uses cord" do + assert @config_with_roles.role(:web).uses_cord? + assert !@config_with_roles.role(:workers).uses_cord? + end + + test "cord host directory" do + assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_host_directory + end + + test "cord host file" do + assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file + end + + test "cord container directory" do + assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_container_directory + end + + test "cord container file" do + assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file + end end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index d31e0dbdf..59046760c 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -224,7 +224,7 @@ class ConfigurationTest < ActiveSupport::TestCase :volume_args=>["--volume", "/local/path:/container/path"], :builder=>{}, :logging=>["--log-opt", "max-size=\"10m\""], - :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999 }} + :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }} assert_equal expected_config, @config.to_h end @@ -252,4 +252,17 @@ class ConfigurationTest < ActiveSupport::TestCase config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal")) assert_equal "/root/kamal", config.run_directory end + + test "run directory as docker volume" do + config = Kamal::Configuration.new(@deploy) + assert_equal "$(pwd)/.kamal", config.run_directory_as_docker_volume + + config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal")) + assert_equal "/root/kamal", config.run_directory_as_docker_volume + end + + test "run id" do + SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112") + assert_equal "09876543211234567890098765432112", @config.run_id + end end diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile index b964e0e3d..2b49fcb08 100644 --- a/test/integration/docker/deployer/Dockerfile +++ b/test/integration/docker/deployer/Dockerfile @@ -2,6 +2,8 @@ FROM ruby:3.2 WORKDIR /app +ENV VERBOSE=true + RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io RUN install -m 0755 -d /etc/apt/keyrings diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index 6ecb94b31..fe4a26ff7 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -18,7 +18,7 @@ builder: args: COMMIT_SHA: <%= `git rev-parse HEAD` %> healthcheck: - cmd: wget -qO- http://localhost > /dev/null + cmd: wget -qO- http://localhost > /dev/null || exit 1 traefik: args: accesslog: true diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index 08e2567f6..e10d9e526 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -54,7 +54,7 @@ class MainTest < IntegrationTest assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options]) assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder]) assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging] - assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck]) + assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck]) end private