diff --git a/vllm/worker/habana_model_runner.py b/vllm/worker/habana_model_runner.py
index c43acdf04923..f3bda39ec482 100644
--- a/vllm/worker/habana_model_runner.py
+++ b/vllm/worker/habana_model_runner.py
@@ -1790,6 +1790,7 @@ def make_model_input_from_broadcasted_tensor_dict(
                 attn_backend=self.attn_backend,
             ))
 
+    @torch.inference_mode()
     def prepare_model_input(
         self,
         seq_group_metadata_list: List[SequenceGroupMetadata],