set CUDA_DEVICE_MAX_CONNECTIONS=1 (#1113)

sgl-project · Aug 15, 2024 · 0cb099e · 0cb099e
1 parent 93d4e35
commit 0cb099e
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
@@ -360,6 +360,7 @@ def _set_envs_and_config(server_args: ServerArgs):
     os.environ["NCCL_CUMEM_ENABLE"] = "0"
     os.environ["NCCL_NVLS_ENABLE"] = "0"
     os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
+    os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"
 
     # Set ulimit
     set_ulimit()