Wait for depends_on conditions before starting containers

This commit implements the long-syntax / conditional depends_on compose
mechanism which instructs the compose system to wait for a certain
condition before starting a container.

Currently available conditions are:

- service_started: same behavior as before this commit, the depending
  container will start as soon as the depended on container has started
- service_healthy: if the depended on container has a healthcheck, wait
  until said container is marked as healthy before starting the
  depending container
- service_completed_successfully: wait until the depended on container
  has exited and its exit code is 0, after which the depending container
  can be started

This mechanism is part of the v3 [1] compose spec and is useful for
controlling container startup based on other containers that can take a
certain amount of time to start or on containers that do complicated
setups and must exit before starting other containers.

[1] https://red.ht/conditional-depends

Signed-off-by: Adrian Torres <atorresj@redhat.com>

podman_compose.py

@@ -123,6 +123,14 @@ def to_enum(cls, condition):
  return cls.STARTED
 
 
+def wait(func):
+ def wrapper(*args, **kwargs):
+ while not func(*args, **kwargs):
+ time.sleep(0.5)
+
+ return wrapper
+
+
 def parse_short_mount(mount_str, basedir):
  mount_a = mount_str.split(":")
  mount_opt_dict = {}
@@ -1087,7 +1095,7 @@ def run(
  podman_args,
  cmd="",
  cmd_args=None,
- wait=True,
+ _wait=True,
  sleep=1,
  obj=None,
  log_formatter=None,
@@ -1113,7 +1121,7 @@ def run(
  else:
  p = subprocess.Popen(cmd_ls) # pylint: disable=consider-using-with
 
- if wait:
+ if _wait:
  exit_code = p.wait()
  log("exit code:", exit_code)
  if obj is not None:
@@ -1970,6 +1978,49 @@ def get_excluded(compose, args):
  return excluded
 
 
+@wait
+def wait_healthy(compose, container_name):
+ info = json.loads(compose.podman.output([], "inspect", [container_name]))[0]
+
+ if not info["Config"].get("Healthcheck"):
+ raise ValueError("Container %s does not define a health check" % container_name)
+
+ health = info["State"]["Healthcheck"]["Status"]
+ if health == "unhealthy":
+ raise RuntimeError(
+ "Container %s is in unhealthy state, aborting" % container_name
+ )
+ return health == "healthy"
+
+
+@wait
+def wait_completed(compose, container_name):
+ info = json.loads(compose.podman.output([], "inspect", [container_name]))[0]
+
+ if info["State"]["Status"] == "exited":
+ exit_code = info["State"]["ExitCode"]
+ if exit_code != 0:
+ raise RuntimeError(
+ "Container %s didn't complete successfully, exit code: %d"
+ % (container_name, exit_code)
+ )
+ return True
+ return False
+
+
+def wait_for_dependencies(compose, container):
+ for dep, condition in container["_deps"].items():
+ dep_container_name = compose.container_names_by_service[dep][0]
+ if condition == DependsCondition.STARTED:
+ # ignore -- will be handled by container order
+ continue
+ if condition == DependsCondition.HEALTHY:
+ wait_healthy(compose, dep_container_name)
+ else:
+ # implies DependsCondition.COMPLETED
+ wait_completed(compose, dep_container_name)
+
+
 @cmd_run(
  podman_compose, "up", "Create and start the entire stack or some of its services"
 )
@@ -2012,6 +2063,8 @@ def compose_up(compose, args):
  log("** skipping: ", cnt["name"])
  continue
  podman_args = container_to_args(compose, cnt, detached=args.detach)
+ if podman_command == "run":
+ wait_for_dependencies(compose, cnt)
  subproc = compose.podman.run([], podman_command, podman_args)
  if podman_command == "run" and subproc and subproc.returncode:
  compose.podman.run([], "start", [cnt["name"]])
@@ -2047,6 +2100,7 @@ def compose_up(compose, args):
  continue
  # TODO: remove sleep from podman.run
  obj = compose if exit_code_from == cnt["_service"] else None
+ wait_for_dependencies(compose, cnt)
  thread = Thread(
  target=compose.podman.run,
  args=[[], "start", ["-a", cnt["name"]]],

tests/deps/docker-compose.yaml

@@ -6,13 +6,26 @@ services:
  tmpfs:
  - /run
  - /tmp
+ healthcheck:
+ # test that httpd is running, the brackets [] thing is a trick
+ # to ignore the grep process returned by ps, meaning that this
+ # should only be true if httpd is currently running
+ test: ps | grep "[h]ttpd"
+ interval: 10s
+ timeout: 5s
+ retries: 5
  sleep:
  image: busybox
  command: ["/bin/busybox", "sh", "-c", "sleep 3600"]
  depends_on: "web"
  tmpfs:
  - /run
  - /tmp
+ healthcheck:
+ test: sleep 15
+ interval: 10s
+ timeout: 20s
+ retries: 5
  sleep2:
  image: busybox
  command: ["/bin/busybox", "sh", "-c", "sleep 3600"]
@@ -21,7 +34,13 @@ services:
  tmpfs:
  - /run
  - /tmp
- hello_world:
+ setup:
+ image: busybox
+ command: ["/bin/busybox", "sh", "-c", "sleep 30"]
+ tmpfs:
+ - /run
+ - /tmp
+ wait_started:
  image: busybox
  command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
  depends_on:
@@ -30,12 +49,16 @@ services:
  tmpfs:
  - /run
  - /tmp
- healthcheck:
- test: echo "hello world"
- interval: 10s
- timeout: 5s
- retries: 5
- hello_world_2:
+ wait_healthy:
+ image: busybox
+ command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
+ depends_on:
+ web:
+ condition: service_healthy
+ tmpfs:
+ - /run
+ - /tmp
+ wait_multiple_healthchecks:
  image: busybox
  command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
  depends_on:
@@ -44,3 +67,12 @@ services:
  tmpfs:
  - /run
  - /tmp
+ wait_completed_successfully:
+ image: busybox
+ command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
+ depends_on:
+ setup:
+ condition: service_completed_successfully
+ tmpfs:
+ - /run
+ - /tmp