diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py
index 869dce39a5a13..ca072b843d19f 100644
--- a/sycl/test-e2e/lit.cfg.py
+++ b/sycl/test-e2e/lit.cfg.py
@@ -80,12 +80,6 @@
 
 llvm_config.with_environment("PATH", config.lit_tools_dir, append_path=True)
 
-if "cuda:gpu" in config.sycl_devices:
-    llvm_config.with_system_environment("CUDA_PATH")
-
-if "hip:gpu" in config.sycl_devices:
-    llvm_config.with_system_environment("ROCM_PATH")
-
 # Configure LD_LIBRARY_PATH or corresponding os-specific alternatives
 if platform.system() == "Linux":
     config.available_features.add("linux")
@@ -456,6 +450,9 @@
         + ", ".join(supported_hip_platforms)
     )
 
+if "cuda:gpu" in config.sycl_devices:
+    llvm_config.with_system_environment("CUDA_PATH")
+
 # FIXME: This needs to be made per-device as well, possibly with a helper.
 if "hip:gpu" in config.sycl_devices and config.hip_platform == "AMD":
     llvm_config.with_system_environment("ROCM_PATH")