diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index b68baffb5a8565..94df3765a67e74 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -1334,8 +1334,9 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite(
     gpu::SetDefaultDeviceOp op, OpAdaptor adaptor,
     ConversionPatternRewriter &rewriter) const {
   Location loc = op.getLoc();
-  setDefaultDeviceCallBuilder.create(loc, rewriter, {adaptor.getDevIndex()});
-  rewriter.replaceOp(op, {});
+  auto call = setDefaultDeviceCallBuilder.create(loc, rewriter,
+                                                 {adaptor.getDevIndex()});
+  rewriter.replaceOp(op, call);
   return success();
 }
 
diff --git a/mlir/test/Conversion/GPUCommon/set-default-device.mlir b/mlir/test/Conversion/GPUCommon/set-default-device.mlir
new file mode 100644
index 00000000000000..c23d8a33d98803
--- /dev/null
+++ b/mlir/test/Conversion/GPUCommon/set-default-device.mlir
@@ -0,0 +1,10 @@
+// RUN: mlir-opt %s --gpu-to-llvm | FileCheck %s
+
+module attributes {gpu.container_module} {
+  // CHECK-LABEL: func @set_default_device
+  func.func @set_default_device(%arg0: i32) {
+    // CHECK: mgpuSetDefaultDevice
+    gpu.set_default_device %arg0
+    return
+  }
+}