Enable llama-405b - w/a for memory allocation error (#184)

Work around for allocation error while loading llama-405b.
HabanaAI · Sep 4, 2024 · 691255b · 691255b
1 parent b4f6a29
commit 691255b
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/vllm/model_executor/models/llama.py b/vllm/model_executor/models/llama.py
@@ -517,6 +517,8 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
  weight_loader = getattr(param, "weight_loader",
  default_weight_loader)
  weight_loader(param, loaded_weight)
+ if current_platform.is_hpu():
+ torch.hpu.synchronize()
 
  # If this function is called, it should always initialize KV cache scale
  # factors (or else raise an exception). Thus, handled exceptions should