Skip to content

Commit

Permalink
Prune healthcheck containers
Browse files Browse the repository at this point in the history
If a deployment is interrupted it could leave stale healthcheck
containers around that prevent dependent images from being pruned.
  • Loading branch information
djmb committed Sep 11, 2023
1 parent 9d35793 commit 718776e
Show file tree
Hide file tree
Showing 7 changed files with 33 additions and 13 deletions.
3 changes: 2 additions & 1 deletion lib/kamal/cli/prune.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ def containers
mutating do
on(KAMAL.hosts) do
execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
execute *KAMAL.prune.containers
execute *KAMAL.prune.app_containers
execute *KAMAL.prune.healthcheck_containers
end
end
end
Expand Down
10 changes: 3 additions & 7 deletions lib/kamal/commands/healthcheck.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ def run
"--detach",
"--name", container_name_with_version,
"--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
"--label", "service=#{container_name}",
"-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
"--label", "service=#{config.healthcheck_service}",
"-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
*web.env_args,
*web.health_check_args(cord: false),
*config.volume_args,
Expand Down Expand Up @@ -38,12 +38,8 @@ def remove
end

private
def container_name
[ "healthcheck", config.service, config.destination ].compact.join("-")
end

def container_name_with_version
"#{container_name}-#{config.version}"
"#{config.healthcheck_service}-#{config.version}"
end

def container_id
Expand Down
12 changes: 10 additions & 2 deletions lib/kamal/commands/prune.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,17 @@ def tagged_images
"while read image tag; do docker rmi $tag; done"
end

def containers(keep_last: 5)
def app_containers(keep_last: 5)
pipe \
docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
"tail -n +#{keep_last + 1}",
"while read container_id; do docker rm $container_id; done"
end

def healthcheck_containers
docker :container, :prune, "--force", *healthcheck_service_filter
end

private
def stopped_containers_filters
[ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
Expand All @@ -35,4 +39,8 @@ def active_image_list
def service_filter
[ "--filter", "label=service=#{config.service}" ]
end
end

def healthcheck_service_filter
[ "--filter", "label=service=#{config.healthcheck_service}" ]
end
end
4 changes: 4 additions & 0 deletions lib/kamal/configuration.rb
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,10 @@ def healthcheck
{ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
end

def healthcheck_service
[ "healthcheck", service, destination ].compact.join("-")
end

def readiness_delay
raw_config.readiness_delay || 7
end
Expand Down
3 changes: 2 additions & 1 deletion test/cli/prune_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ class CliPruneTest < CliTestCase
test "containers" do
run_command("containers").tap do |output|
assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
end
assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
end
end

private
Expand Down
10 changes: 8 additions & 2 deletions test/commands/prune_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,16 @@ class CommandsPruneTest < ActiveSupport::TestCase
new_command.tagged_images.join(" ")
end

test "containers" do
test "app containers" do
assert_equal \
"docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
new_command.containers.join(" ")
new_command.app_containers.join(" ")
end

test "healthcheck containers" do
assert_equal \
"docker container prune --force --filter label=service=healthcheck-app",
new_command.healthcheck_containers.join(" ")
end

private
Expand Down
4 changes: 4 additions & 0 deletions test/configuration_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ class ConfigurationTest < ActiveSupport::TestCase
assert_equal "app-missing", @config.service_with_version
end

test "healthcheck service" do
assert_equal "healthcheck-app", @config.healthcheck_service
end

test "env with missing secret" do
assert_raises(KeyError) do
config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
Expand Down

0 comments on commit 718776e

Please sign in to comment.