[k8s] Show hints when requested resources don't fit in Kubernetes cluster

sky/clouds/aws.py

@@ -425,7 +425,7 @@ def make_deploy_resources_variables(self,
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources_lib.Resources'
- ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+ ) -> Tuple[List['resources_lib.Resources'], List[str], Optional[str]]:
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
  # Check the instance type is valid in the cloud
@@ -436,10 +436,10 @@ def _get_feasible_launchable_resources(
  region=resources.region,
  zone=resources.zone)
  if not regions:
- return ([], [])
+ return ([], [], None)
  # Treat Resources(AWS, p3.2x, V100) as Resources(AWS, p3.2x).
  resources = resources.copy(accelerators=None)
- return ([resources], [])
+ return ([resources], [], None)
 
  def _make(instance_list):
  resource_list = []
@@ -465,9 +465,9 @@ def _make(instance_list):
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if default_instance_type is None:
- return ([], [])
+ return ([], [], None)
  else:
- return (_make([default_instance_type]), [])
+ return (_make([default_instance_type]), [], None)
 
  assert len(accelerators) == 1, resources
  acc, acc_count = list(accelerators.items())[0]
@@ -482,8 +482,8 @@ def _make(instance_list):
  zone=resources.zone,
  clouds='aws')
  if instance_list is None:
- return ([], fuzzy_candidate_list)
- return (_make(instance_list), fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
+ return (_make(instance_list), fuzzy_candidate_list, None)
 
  @classmethod
  @functools.lru_cache(maxsize=1) # Cache since getting identity is slow.

sky/clouds/azure.py

@@ -376,17 +376,17 @@ def _failover_disk_tier() -> Optional[resources_utils.DiskTier]:
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources.Resources'
- ) -> Tuple[List['resources.Resources'], List[str]]:
+ ) -> Tuple[List['resources.Resources'], List[str], Optional[str]]:
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
  ok, _ = Azure.check_disk_tier(resources.instance_type,
  resources.disk_tier)
  if not ok:
- return ([], [])
+ return ([], [], None)
  # Treat Resources(Azure, Standard_NC4as_T4_v3, T4) as
  # Resources(Azure, Standard_NC4as_T4_v3).
  resources = resources.copy(accelerators=None)
- return ([resources], [])
+ return ([resources], [], None)
 
  def _make(instance_list):
  resource_list = []
@@ -416,9 +416,9 @@ def _make(instance_list):
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if default_instance_type is None:
- return ([], [])
+ return ([], [], None)
  else:
- return (_make([default_instance_type]), [])
+ return (_make([default_instance_type]), [], None)
 
  assert len(accelerators) == 1, resources
  acc, acc_count = list(accelerators.items())[0]
@@ -433,8 +433,8 @@ def _make(instance_list):
  zone=resources.zone,
  clouds='azure')
  if instance_list is None:
- return ([], fuzzy_candidate_list)
- return (_make(instance_list), fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
+ return (_make(instance_list), fuzzy_candidate_list, None)
 
  @classmethod
  def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/cloud.py

@@ -344,15 +344,18 @@ def get_feasible_launchable_resources(
  self,
  resources: 'resources_lib.Resources',
  num_nodes: int = 1
- ) -> Tuple[List['resources_lib.Resources'], List[str]]:
- """Returns ([feasible and launchable resources], [fuzzy candidates]).
+ ) -> Tuple[List['resources_lib.Resources'], List[str], Optional[str]]:
+ """Returns ([feasible & launchable resources], [fuzzy candidates], hint)
 
  Feasible resources refer to an offering respecting the resource
  requirements. Currently, this function implements "filtering" the
  cloud's offerings only w.r.t. accelerators constraints.
 
  Launchable resources require a cloud and an instance type be assigned.
 
+ The cloud may optionally return a string hint to the user if no feasible
+ resources are found.
+
  Fuzzy candidates example: when the requested GPU is A100:1 but is not
  available in a cloud/region, the fuzzy candidates are results of a fuzzy
  search in the catalog that are offered in the location. E.g.,
@@ -372,12 +375,12 @@ def get_feasible_launchable_resources(
  # TODO(zhwu): The resources are now silently filtered out. We
  # should have some logging telling the user why the resources
  # are not considered.
- return ([], [])
+ return ([], [], None)
  return self._get_feasible_launchable_resources(resources)
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources_lib.Resources'
- ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+ ) -> Tuple[List['resources_lib.Resources'], List[str], Optional[str]]:
  """See get_feasible_launchable_resources()."""
  raise NotImplementedError
 

sky/clouds/cudo.py

@@ -203,11 +203,11 @@ def make_deploy_resources_variables(
  def _get_feasible_launchable_resources(
  self, resources: 'resources_lib.Resources'):
  if resources.use_spot:
- return ([], [])
+ return ([], [], None)
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
  resources = resources.copy(accelerators=None)
- return ([resources], [])
+ return ([resources], [], None)
 
  def _make(instance_list):
  resource_list = []
@@ -230,9 +230,9 @@ def _make(instance_list):
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if default_instance_type is None:
- return ([], [])
+ return ([], [], None)
  else:
- return (_make([default_instance_type]), [])
+ return (_make([default_instance_type]), [], None)
 
  assert len(accelerators) == 1, resources
  acc, acc_count = list(accelerators.items())[0]
@@ -247,8 +247,8 @@ def _make(instance_list):
  zone=resources.zone,
  clouds='cudo')
  if instance_list is None:
- return ([], fuzzy_candidate_list)
- return (_make(instance_list), fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
+ return (_make(instance_list), fuzzy_candidate_list, None)
 
  @classmethod
  def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/fluidstack.py

@@ -210,7 +210,7 @@ def _get_feasible_launchable_resources(
  assert resources.is_launchable(), resources
  # Accelerators are part of the instance type in Fluidstack Cloud
  resources = resources.copy(accelerators=None)
- return ([resources], [])
+ return ([resources], [], None)
 
  def _make(instance_list):
  resource_list = []
@@ -238,9 +238,9 @@ def _make(instance_list):
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if default_instance_type is None:
- return ([], [])
+ return ([], [], None)
  else:
- return (_make([default_instance_type]), [])
+ return (_make([default_instance_type]), [], None)
 
  assert len(accelerators) == 1, resources
  acc, acc_count = list(accelerators.items())[0]
@@ -255,8 +255,8 @@ def _make(instance_list):
  zone=resources.zone,
  clouds='fluidstack')
  if instance_list is None:
- return ([], fuzzy_candidate_list)
- return (_make(instance_list), fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
+ return (_make(instance_list), fuzzy_candidate_list, None)
 
  @classmethod
  def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/gcp.py

@@ -497,10 +497,10 @@ def make_deploy_resources_variables(
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources.Resources'
- ) -> Tuple[List['resources.Resources'], List[str]]:
+ ) -> Tuple[List['resources.Resources'], List[str], Optional[str]]:
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
- return ([resources], [])
+ return ([resources], [], None)
 
  if resources.accelerators is None:
  # Return a default instance type with the given number of vCPUs.
@@ -509,7 +509,7 @@ def _get_feasible_launchable_resources(
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if host_vm_type is None:
- return ([], [])
+ return ([], [], None)
  else:
  r = resources.copy(
  cloud=GCP(),
@@ -518,7 +518,7 @@ def _get_feasible_launchable_resources(
  cpus=None,
  memory=None,
  )
- return ([r], [])
+ return ([r], [], None)
 
  # Find instance candidates to meet user's requirements
  assert len(resources.accelerators.items()
@@ -540,7 +540,7 @@ def _get_feasible_launchable_resources(
  clouds='gcp')
 
  if instance_list is None:
- return ([], fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
  assert len(
  instance_list
  ) == 1, f'More than one instance type matched, {instance_list}'
@@ -555,11 +555,11 @@ def _get_feasible_launchable_resources(
  if resources.cpus.endswith('+'):
  cpus = float(resources.cpus[:-1])
  if cpus > num_cpus_in_tpu_vm:
- return ([], fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
  else:
  cpus = float(resources.cpus)
  if cpus != num_cpus_in_tpu_vm:
- return ([], fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
  # FIXME(woosuk, wei-lin): This leverages the fact that TPU VMs
  # have 334 GB RAM, and 400 GB RAM for tpu-v4. We need to move
  # this to service catalog, instead.
@@ -568,11 +568,11 @@ def _get_feasible_launchable_resources(
  if resources.memory.endswith('+'):
  memory = float(resources.memory[:-1])
  if memory > memory_in_tpu_vm:
- return ([], fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
  else:
  memory = float(resources.memory)
  if memory != memory_in_tpu_vm:
- return ([], fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
  else:
  host_vm_type = instance_list[0]
 
@@ -584,7 +584,7 @@ def _get_feasible_launchable_resources(
  cpus=None,
  memory=None,
  )
- return ([r], fuzzy_candidate_list)
+ return ([r], fuzzy_candidate_list, None)
 
  @classmethod
  def get_accelerators_from_instance_type(

sky/clouds/ibm.py

@@ -266,12 +266,12 @@ def get_default_instance_type(
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources_lib.Resources'
- ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+ ) -> Tuple[List['resources_lib.Resources'], List[str], Optional[str]]:
  fuzzy_candidate_list: List[str] = []
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
  resources = resources.copy(accelerators=None)
- return ([resources], fuzzy_candidate_list)
+ return ([resources], fuzzy_candidate_list, None)
 
  def _make(instance_list):
  resource_list = []
@@ -296,9 +296,9 @@ def _make(instance_list):
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if default_instance_type is None:
- return ([], [])
+ return ([], [], None)
  else:
- return (_make([default_instance_type]), [])
+ return (_make([default_instance_type]), [], None)
 
  assert len(accelerators) == 1, resources
  acc, acc_count = list(accelerators.items())[0]
@@ -312,8 +312,8 @@ def _make(instance_list):
  zone=resources.zone,
  clouds='ibm')
  if instance_list is None:
- return ([], fuzzy_candidate_list)
- return (_make(instance_list), fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
+ return (_make(instance_list), fuzzy_candidate_list, None)
 
  @classmethod
  def get_default_image(cls, region) -> str:

sky/clouds/kubernetes.py

@@ -318,12 +318,12 @@ def make_deploy_resources_variables(
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources_lib.Resources'
- ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+ ) -> Tuple[List['resources_lib.Resources'], List[str], Optional[str]]:
  fuzzy_candidate_list: List[str] = []
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
  resources = resources.copy(accelerators=None)
- return ([resources], fuzzy_candidate_list)
+ return ([resources], fuzzy_candidate_list, None)
 
  def _make(instance_list):
  resource_list = []
@@ -379,10 +379,10 @@ def _make(instance_list):
  logger.debug(f'Instance type {chosen_instance_type} does '
  'not fit in the Kubernetes cluster. '
  f'Reason: {reason}')
- return [], []
+ return [], [], reason
 
  # No fuzzy lists for Kubernetes
- return _make([chosen_instance_type]), []
+ return _make([chosen_instance_type]), [], None
 
  @classmethod
  def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/lambda_cloud.py

@@ -178,12 +178,12 @@ def make_deploy_resources_variables(
 
  def _get_feasible_launchable_resources(
  self, resources: 'resources_lib.Resources'
- ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+ ) -> Tuple[List['resources_lib.Resources'], List[str], Optional[str]]:
  if resources.instance_type is not None:
  assert resources.is_launchable(), resources
  # Accelerators are part of the instance type in Lambda Cloud
  resources = resources.copy(accelerators=None)
- return ([resources], [])
+ return ([resources], [], None)
 
  def _make(instance_list):
  resource_list = []
@@ -209,9 +209,9 @@ def _make(instance_list):
  memory=resources.memory,
  disk_tier=resources.disk_tier)
  if default_instance_type is None:
- return ([], [])
+ return ([], [], None)
  else:
- return (_make([default_instance_type]), [])
+ return (_make([default_instance_type]), [], None)
 
  assert len(accelerators) == 1, resources
  acc, acc_count = list(accelerators.items())[0]
@@ -226,8 +226,8 @@ def _make(instance_list):
  zone=resources.zone,
  clouds='lambda')
  if instance_list is None:
- return ([], fuzzy_candidate_list)
- return (_make(instance_list), fuzzy_candidate_list)
+ return ([], fuzzy_candidate_list, None)
+ return (_make(instance_list), fuzzy_candidate_list, None)
 
  @classmethod
  def check_credentials(cls) -> Tuple[bool, Optional[str]]: