Add support for cross-file service dependency

Signed-off-by: Szymon Niedzwiedz <git@hardcoded.info>

podman_compose.py

@@ -1013,19 +1013,13 @@ def rec_deps(services, service_name, start_point=None):
  return deps
 
 
-def flat_deps(services, with_extends=False):
+def flat_deps(services):
  """
  create dependencies "_deps" or update it recursively for all services
  """
  for name, srv in services.items():
  deps = set()
  srv["_deps"] = deps
- if with_extends:
- ext = srv.get("extends", {}).get("service", None)
- if ext:
- if ext != name:
- deps.add(ext)
- continue
  deps_ls = srv.get("depends_on", None) or []
  if is_str(deps_ls):
  deps_ls = [deps_ls]
@@ -1211,7 +1205,9 @@ def rec_merge_one(target, source):
  raise ValueError(
  f"can't merge value of {key} of type {value_type} and {value2_type}"
  )
- if is_list(value2):
+ if type(value) == set:
+ value = value.update(value2)
+ elif is_list(value2):
  if key == "volumes":
  # clean duplicate mount targets
  pts = {v.split(":", 1)[1] for v in value2 if ":" in v}
@@ -1241,36 +1237,179 @@ def rec_merge(target, *sources):
  return ret
 
 
-def resolve_extends(services, service_names, environ):
- for name in service_names:
- service = services[name]
- ext = service.get("extends", {})
- if is_str(ext):
- ext = {"service": ext}
- from_service_name = ext.get("service", None)
- if not from_service_name:
- continue
- filename = ext.get("file", None)
- if filename:
- if filename.startswith("./"):
- filename = filename[2:]
+class ComposeFileParsingException(Exception):
+ pass
+
+class ComposeFileParsingCircularDependencyException(ComposeFileParsingException):
+ pass
+
+def pretty_print_tuple(tup, data):
+ file, service = tup
+ if file is None:
+ return ",".join(data), service
+ return file, service
+
+class OrderedSet():
+ def __init__(self):
+ self.as_list = list()
+ self.as_set = set()
+
+ def pop(self):
+ r = self.as_list.pop()
+ self.as_set.remove(r)
+
+ def add(self, element):
+ self.as_list.append(element)
+ self.as_set.add(element)
+
+ def __contains__(self, element):
+ return element in self.as_set
+
+ def __str__(self):
+ return str(self.as_list)
+
+ def pretty_print(self, data):
+ new_list = []
+ for tup in self.as_list:
+ new_list.append(pretty_print_tuple(tup,data))
+ return new_list
+
+class CachedComposeFileParser():
+ """
+ This class handles preprocessed merged_yaml, and then recursively updates services
+ in order to remove extends field
+ # TODO: make all parsing recursive and move it to this class
+ """
+ def __init__(self, environ, files):
+ self.environ = environ
+ self.merged_yaml = {}
+ self.merged_files = [os.path.realpath(file) for file in files]
+ self.cache = dict()
+
+ def generate_compose(self):
+ for filename in self.merged_files:
  with open(filename, "r", encoding="utf-8") as f:
- content = yaml.safe_load(f) or {}
+ content = yaml.safe_load(f)
+ # log(filename, json.dumps(content, indent = 2))
+ if not isinstance(content, dict):
+ raise ComposeFileParsingException(
+ "Compose file does not contain a top level object: %s\n"
+ % filename
+ )
+ content = normalize(content)
+ # log(filename, json.dumps(content, indent = 2))
+ content = rec_subs(content, self.environ)
+ rec_merge(self.merged_yaml, content)
+ return self.merged_yaml
+
+ def read_file_and_cache_it(self, filename):
+ real_file = os.path.realpath(filename)
+ if real_file in self.merged_files:
+ return self.merged_yaml
+ if real_file not in self.cache:
+ with open(filename, "r", encoding="utf-8") as f:
+ content = yaml.safe_load(f) or dict()
  if "services" in content:
- content = content["services"]
- subdirectory = os.path.dirname(filename)
- content = rec_subs(content, environ)
- from_service = content.get(from_service_name, {})
- normalize_service(from_service, subdirectory)
- else:
- from_service = services.get(from_service_name, {}).copy()
- del from_service["_deps"]
- try:
- del from_service["extends"]
- except KeyError:
- pass
- new_service = rec_merge({}, from_service, service)
- services[name] = new_service
+ services = content["services"]
+ services = rec_subs(services, self.environ)
+ content['services'] = services
+ self.cache[real_file] = content
+ return self.cache[real_file]
+
+ def _pretty_file(self, name):
+ if name is None:
+ # merged docker-compose files passed to script
+ # are treated as a single file
+ return ",".join(self.merged_files)
+ return name
+
+ def _service_not_found(self, service_name, parent_service_name, parent_filename):
+ pretty_file = self._pretty_file(parent_filename)
+ msg = f"Service {service_name} has dependency of '{parent_service_name}" \
+ f"which does not exist in {pretty_file}"
+ raise ComposeFileParsingException(msg)
+
+ def resolve_extend(self, services, service, service_name, current_filename,
+ circular_dep_detector):
+ def patch_parent_filename_if_default(parent_filename):
+ if os.path.isabs(parent_filename):
+ return parent_filename
+ temp_name = current_filename
+ if temp_name is None:
+ temp_name = self.merged_files[0]
+ parent_filename = os.path.join(os.path.dirname(temp_name), parent_filename)
+ return parent_filename
+ if current_filename is not None:
+ current_filename = os.path.realpath(current_filename)
+ if current_filename in self.merged_files:
+ current_filename = None
+ service_unique_identifier = (current_filename, service_name,)
+ if service_unique_identifier in circular_dep_detector:
+ msg = f"Circular dependency to {pretty_print_tuple(service_unique_identifier, self.merged_files)} " \
+ f"detected: {circular_dep_detector.pretty_print(self.merged_files)}"
+ raise ComposeFileParsingCircularDependencyException(msg)
+ circular_dep_detector.add(service_unique_identifier)
+ try:
+ extends_section = service.get("extends")
+ if extends_section is None:
+ return service
+
+ if is_str(extends_section):
+ parent_service_name = extends_section
+ else:
+ parent_service_name = extends_section.get("service")
+
+ if parent_service_name is None:
+ pretty_name = self._pretty_file(current_filename)
+ raise ComposeFileParsingException(f"Service {service_name} in {pretty_name} has" \
+ f"extends field and no service name")
+ parent_filename = extends_section.get("file")
+ if parent_filename:
+ if parent_filename.startswith("./"):
+ parent_filename = parent_filename[2:]
+ subdirectory = os.path.dirname(parent_filename)
+ parent_filename = patch_parent_filename_if_default(parent_filename)
+ content = self.read_file_and_cache_it(parent_filename)
+ # ADDED: normalize each service later
+ from_service_ref = content['services'].get(parent_service_name)
+ if from_service_ref is None:
+ self._service_not_found(service_name, parent_service_name, parent_filename)
+ from_service_ref = self.resolve_extend(content['services'],from_service_ref, parent_service_name,
+ parent_filename, circular_dep_detector)
+ from_service_ref = normalize_service(from_service_ref, subdirectory)
+ content[parent_service_name] = from_service_ref
+ from_service = from_service_ref.copy()
+ else:
+ mutable_parent_service = services.get(parent_service_name)
+ if mutable_parent_service is None:
+ self._service_not_found(service_name, parent_service_name, current_filename)
+ from_service_ref = self.resolve_extend(services, mutable_parent_service,
+ parent_service_name, current_filename, circular_dep_detector)
+ from_service = from_service_ref.copy()
+ assert 'extends' in service # ensure, same service is not processed twice
+ del service['extends']
+ normalize_service(service)
+ services[service_name] = rec_merge(dict(), from_service, service)
+ assert services[service_name] is not None
+ finally:
+ circular_dep_detector.pop()
+ return services[service_name]
+
+ def parse_services(self):
+ # if current_filename is None it means
+ # we are handling merged compose file
+ current_filename = ",".join(self.merged_files)
+ services = self.merged_yaml.get("services", None)
+ if services is None:
+ services = {}
+ log(f"WARNING: No services defined in {current_filename}")
+ for service_name in services.keys():
+ services[service_name] = self.resolve_extend(services, services[service_name],
+ service_name, current_filename,
+ OrderedSet())
+ assert services[service_name] is not None
+ flat_deps(services)
+ return services
 
 
 def dotenv_to_dict(dotenv_path):
@@ -1444,21 +1583,10 @@ def _parse_compose_file(self):
  "COMPOSE_PATH_SEPARATOR": pathsep,
  }
  )
- compose = {}
- for filename in files:
- with open(filename, "r", encoding="utf-8") as f:
- content = yaml.safe_load(f)
- # log(filename, json.dumps(content, indent = 2))
- if not isinstance(content, dict):
- sys.stderr.write(
- "Compose file does not contain a top level object: %s\n"
- % filename
- )
- sys.exit(1)
- content = normalize(content)
- # log(filename, json.dumps(content, indent = 2))
- content = rec_subs(content, self.environ)
- rec_merge(compose, content)
+
+ parser = CachedComposeFileParser(self.environ, files)
+ compose = parser.generate_compose()
+
  self.merged_yaml = yaml.safe_dump(compose)
  merged_json_b = json.dumps(compose, separators=(",", ":")).encode("utf-8")
  self.yaml_hash = hashlib.sha256(merged_json_b).hexdigest()
@@ -1484,19 +1612,8 @@ def _parse_compose_file(self):
  self.project_name = project_name
  self.environ.update({"COMPOSE_PROJECT_NAME": self.project_name})
 
- services = compose.get("services", None)
- if services is None:
- services = {}
- log("WARNING: No services defined")
+ services = parser.parse_services()
 
- # NOTE: maybe add "extends.service" to _deps at this stage
- flat_deps(services, with_extends=True)
- service_names = sorted(
- [(len(srv["_deps"]), name) for name, srv in services.items()]
- )
- service_names = [name for _, name in service_names]
- resolve_extends(services, service_names, self.environ)
- flat_deps(services)
  service_names = sorted(
  [(len(srv["_deps"]), name) for name, srv in services.items()]
  )
@@ -1616,6 +1733,7 @@ def _parse_args(self):
  if not self.global_args.command or self.global_args.command == "help":
  parser.print_help()
  sys.exit(-1)
+ print(self.global_args)
  return self.global_args
 
  @staticmethod

pytests/test_recursive_parsing.py

@@ -0,0 +1,86 @@
+import argparse
+import pytest
+import subprocess
+import os
+from podman_compose import PodmanCompose, ComposeFileParsingCircularDependencyException
+from pathlib import Path
+
+
+NONEMPTY_STRING = "non_empty_string.yml"
+TESTS_PATH = Path(__file__).parent / '../tests/'
+
+def fake_podman_env():
+ env_with_fake_podman = os.environ.copy()
+ here = TESTS_PATH / 'fake_podman'
+ env_with_fake_podman.update({
+ 'PATH': f"{here}:{os.getenv('PATH','')}"
+ })
+ return env_with_fake_podman
+
+def MockedPodmanCompose(compose_file: Path, *args, **kwargs):
+ pc = PodmanCompose()
+ pc.global_args = argparse.Namespace(file=[str(compose_file)],
+ project_name=None,
+ env_file=NONEMPTY_STRING,
+ no_pod=True, *args, **kwargs)
+ return pc
+
+def test_given_compose_file_and_no_deps_arg_when_run_then_only_one_service_is_up():
+ # https://github.com/containers/podman-compose/issues/398
+
+ run = subprocess.run([str((Path(__file__).parent / "../podman_compose.py").absolute()),
+ '--dry-run', '-f', str(TESTS_PATH /
+ 'extends_to_be_run/docker-compose.yml'), 'run',
+ 'sh', 'sh'],
+ env=fake_podman_env(),
+ stderr=subprocess.PIPE,
+ universal_newlines=True
+ )
+ assert 'podman run' in run.stderr
+
+def test_given_compose_file_with_mounts_when_parsing_then_mounts_resolved_correctly():
+ # https://github.com/containers/podman-compose/issues/462
+ pc = MockedPodmanCompose(
+ TESTS_PATH / 'extends_valid_mounts_resolved/docker-compose.yml')
+ pc._parse_compose_file()
+ assert set([
+ '/tmp/service_other-bash:/tmp/service_other-bash:rw',
+ '/tmp/service_bash:/tmp/service_bash:rw']) == set(pc.services['other-bash']['volumes'])
+
+
+def test_given_cli_volume_and_compose_file_volume_when_parsing_then_both_are_used():
+ # https://github.com/containers/podman-compose/issues/464
+ run = subprocess.run([str((Path(__file__).parent / "../podman_compose.py").absolute()),
+ '--dry-run', '-f', str(TESTS_PATH /
+ 'file_and_cli_mounts/docker-compose.yml'), 'run',
+ '-v', '/tmp/test:/tmp/test', 'sh', 'sh'],
+ env=fake_podman_env(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ universal_newlines=True
+ )
+ assert '-v /tmp/test:/tmp/test' in run.stderr
+ assert '-v /tmp/service_sh:/tmp/service_sh' in run.stderr
+
+def test_given_ping_pong_dependencies_between_two_files_when_parsing_then_resolved_correctly():
+ # https://github.com/containers/podman-compose/issues/465
+ pc = MockedPodmanCompose(
+ TESTS_PATH / 'extends_recursive/docker-compose.yml')
+ pc._parse_compose_file()
+ assert pc.services['sh1'].items() >= {
+ 'image': 'busybox',
+ 'volumes': ['/host/7:/cnt/7:rw', '/host/4:/cnt/4:rw', '/host/1:/cnt/1:rw']}.items()
+ assert pc.services['sh2'].items() >= {
+ 'image': 'busybox',
+ 'volumes': ['/host/7:/cnt/7:rw']}.items()
+ assert pc.services['sh3'].items() >= {
+ 'image': 'busybox',
+ 'volumes': ['/host/7:/cnt/7:rw']}.items()
+
+
+def tests_given_compose_file_with_circular_dependency_when_parsing_then_raises_exception():
+ # https://github.com/containers/podman-compose/issues/465
+ pc = MockedPodmanCompose(
+ TESTS_PATH / 'extends_recursive_circular/docker-compose.yml')
+ with pytest.raises(ComposeFileParsingCircularDependencyException) as e:
+ pc._parse_compose_file()