debugged copy download-miniconda.sh issue and added support for multi…

…-platform build

ersilia/db/environments/managers.py

@@ -14,6 +14,8 @@
 
 from ...default import DOCKERHUB_ORG, DOCKERHUB_LATEST_TAG
 
+import subprocess
+import sys
 
 BENTOML_DOCKERPORT = 5000
 INTERNAL_DOCKERPORT = 80
@@ -123,7 +125,7 @@ def _build_ersilia_base(self):
  path = tempfile.mkdtemp(prefix="ersilia-")
  base_folder = os.path.join(path, "base")
  os.mkdir(base_folder)
- base_files = ["Dockerfile", "docker-entrypoint.sh", "nginx.conf"]
+ base_files = ["Dockerfile", "docker-entrypoint.sh", "nginx.conf", "download-miniconda.sh"]
  for f in base_files:
  cmd = "cd {0}; wget {2}/base/{1}".format(
  base_folder, f, self._model_deploy_dockerfiles_url
@@ -134,7 +136,7 @@ def _build_ersilia_base(self):
  )
  run_command(cmd)
 
- def build_with_ersilia(self, model_id):
+ def build_with_ersilia(self, model_id, docker_user, docker_pwd):
  self.logger.debug("Creating docker image with ersilia incorporated")
  if self.image_exists("base"):
  pass
@@ -153,16 +155,35 @@ def build_with_ersilia(self, model_id):
  text = text.replace("eos_identifier", model_id)
  with open(file_path, "w") as f:
  f.write(text)
- cmd = "cd {0}; docker build -t {1}/{2}:{3} .".format(
- model_folder, DOCKERHUB_ORG, model_id, DOCKERHUB_LATEST_TAG
- )
+ # login to docker so as to be able to push images to dockerhub
+ cmd = "docker login --password {0} --username {1}".format(
+ docker_pwd, docker_user
+ )
  run_command(cmd)
 
- def build(self, model_id, use_cache=True):
+ try:
+ # Attempt to build for linux/amd64 and linux/arm64
+ print('building for both linux/amd64,linux/arm64')
+ cmd = "cd {0}; docker buildx build --builder=container --platform linux/amd64,linux/arm64 -t {1}/{2}:{3} --push .".format(
+ model_folder, DOCKERHUB_ORG, model_id, DOCKERHUB_LATEST_TAG
+ )
+ run_command(cmd)
+ print('done building for linux/amd64,linux/arm64')
+ sys.exit() # This will terminate the program immediately only if build for linux/amd64,linux/arm64 complete successfully
+
+ except subprocess.CalledProcessError as e:
+ # Build failed for multi-platforms, now try building only for linux/amd64
+ self.logger.warning("Build failed for multi-platform, trying linux/amd64 only")
+ cmd = "cd {0}; docker build -t {1}/{2}:{3} .".format(
+ model_folder, DOCKERHUB_ORG, model_id, DOCKERHUB_LATEST_TAG
+ )
+ run_command(cmd)
+
+ def build(self, model_id, docker_user, docker_pwd, use_cache=True):
  if self.with_bentoml:
  self.build_with_bentoml(model_id=model_id, use_cache=use_cache)
  else:
- self.build_with_ersilia(model_id=model_id)
+ self.build_with_ersilia(model_id=model_id, docker_user=docker_user, docker_pwd=docker_pwd)
 
  def remove(self, model_id):
  self.docker.delete(org=DOCKERHUB_ORG, img=model_id, tag=DOCKERHUB_LATEST_TAG)

ersilia/publish/dockerhub.py

@@ -17,7 +17,7 @@ def set_credentials(self, docker_user, docker_pwd):
 
  def build_image(self):
  dm = DockerManager()
- dm.build(self.model_id)
+ dm.build(self.model_id, self.docker_user, self.docker_pwd)
 
  def upload(self):
  self.build_image()

ersilia/utils/terminal.py

@@ -31,21 +31,32 @@ def run_command(cmd, quiet=None):
  if type(cmd) == str:
  if quiet:
  with open(os.devnull, "w") as fp:
- subprocess.Popen(
+ return_code = subprocess.Popen(
  cmd, stdout=fp, stderr=fp, shell=True, env=os.environ
  ).wait()
  else:
- subprocess.Popen(cmd, shell=True, env=os.environ).wait()
+ return_code = subprocess.Popen(cmd, shell=True, env=os.environ).wait()
  else:
  if quiet:
- subprocess.check_call(
- cmd,
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- env=os.environ,
- )
+ try:
+ subprocess.check_call(
+ cmd,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ env=os.environ,
+ )
+ return_code = 0
+ except subprocess.CalledProcessError as e:
+ return_code = e.returncode
  else:
- subprocess.check_call(cmd, env=os.environ)
+ try:
+ subprocess.check_call(cmd, env=os.environ)
+ return_code = 0
+ except subprocess.CalledProcessError as e:
+ return_code = e.returncode
+
+ if return_code != 0:
+ raise subprocess.CalledProcessError(return_code, cmd)
 
 
 def run_command_check_output(cmd):