NVIDIA · schweitzpgi · Dec 7, 2023 · Jun 29, 2024 · Jul 10, 2024
diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt
@@ -27,7 +27,7 @@ function(add_nvqpp_test TEST_NAME SOURCE_LOCATION)
  NAME
  nvqpp_${TEST_NAME}
  COMMAND
- bash -c "rm -f ${TEST_NAME}; ${CMAKE_BINARY_DIR}/bin/nvq++ ${NVQPP_COMPILE_ARGS} ${PARSED_ARGS_SOURCE_DIR}/${SOURCE_LOCATION} -o ${TEST_NAME} && \
+ bash -c "rm -f ${TEST_NAME}; ${CMAKE_BINARY_DIR}/bin/nvq++ ${NVQPP_COMPILE_ARGS} ${NVQPP_EXTRA_ARGS} ${PARSED_ARGS_SOURCE_DIR}/${SOURCE_LOCATION} -o ${TEST_NAME} && \
  ${PARSED_ARGS_LAUNCH_COMMAND} ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME} ${PARSED_ARGS_APPLICATION_ARGS}"
  )
  if(PARSED_ARGS_LABELS)
@@ -89,7 +89,7 @@ if (CUSTATEVEC_ROOT AND CUDA_FOUND)
  endif() 
 endif()
 
-add_nvqpp_test(photonics_sim providers/photonics.cpp TARGET photonics)
+add_nvqpp_test(photonics_sim providers/photonics.cpp TARGET photonics NVQPP_EXTRA_ARGS "--library-mode")
 add_nvqpp_test(SampleAsyncRemote using/cudaq/platform/sample_async_remote.cpp TARGET remote-mqpu SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/sphinx/snippets/cpp)
 set_tests_properties(
  nvqpp_SampleAsyncRemote

diff --git a/runtime/cudaq/platform/default/rest/helpers/ionq/ionq.config b/runtime/cudaq/platform/default/rest/helpers/ionq/ionq.config
@@ -25,10 +25,6 @@ CODEGEN_EMISSION=qir-base
 # This is required due to https://github.com/NVIDIA/cuda-quantum/issues/512
 POST_CODEGEN_PASSES="remove-measurements"
 
-# Library mode is only for simulators, 
-# and it is the default, physical backends must 
-# turn this off
-LIBRARY_MODE=false
 # Variables for validation
 # Default ionq-machine target if not specified.
 IONQ_MACHINE="simulator"

diff --git a/runtime/cudaq/platform/default/rest/helpers/iqm/iqm.config b/runtime/cudaq/platform/default/rest/helpers/iqm/iqm.config
@@ -23,9 +23,6 @@ PLATFORM_LOWERING_CONFIG="expand-measurements,unrolling-pipeline,decomposition{e
 # Tell the rest-qpu that we are generating IQM JSON.
 CODEGEN_EMISSION=iqm
 
-# Physical backends must set library mode to false.
-LIBRARY_MODE=false
-
 PLATFORM_EXTRA_ARGS=""
 # NB: extra arguments always take the form:
 # --<target>-<option> <value>

diff --git a/runtime/cudaq/platform/default/rest/helpers/oqc/oqc.config b/runtime/cudaq/platform/default/rest/helpers/oqc/oqc.config
@@ -24,11 +24,6 @@ PLATFORM_LOWERING_CONFIG="expand-measurements,unrolling-pipeline,decomposition{e
 # Tell the rest-qpu that we are generating QIR.
 CODEGEN_EMISSION=qir-base
 
-# Library mode is only for simulators, 
-# and it is the default, physical backends must 
-# turn this off
-LIBRARY_MODE=false
-
 # NB: extra arguments always take the form:
 # --<target>-<option> <value>
 # as in

diff --git a/runtime/cudaq/platform/default/rest/helpers/quantinuum/quantinuum.config b/runtime/cudaq/platform/default/rest/helpers/quantinuum/quantinuum.config
@@ -21,9 +21,6 @@ PLATFORM_LOWERING_CONFIG="expand-measurements,unrolling-pipeline,decomposition{e
 # Tell the rest-qpu that we are generating QIR.
 CODEGEN_EMISSION=qir-adaptive
 
-# Physical backends must set library mode to false.
-LIBRARY_MODE=false
-
 PLATFORM_EXTRA_ARGS=""
 # NB: extra arguments always take the form:
 # --<target>-<option> <value>

diff --git a/runtime/cudaq/platform/mqpu/remote-mqpu.config b/runtime/cudaq/platform/mqpu/remote-mqpu.config
@@ -22,18 +22,21 @@ PLATFORM_QPU=RemoteSimulatorQPU
 
 PREPROCESSOR_DEFINES="$PREPROCESSOR_DEFINES -D CUDAQ_REMOTE_SIM"
 
-if ${LIBRARY_MODE}; then
- # In library mode, we enforce that no custom fallback host compiler has been set via `cmake-host-compiler` option.
- # Note: even if another Clang-based compiler is set, we cannot guarantee that it is compatible with the LLVM version
- # that the CUDA-Q runtime is compiled against. Hence, we don't support custom host compiler for this platform.
+if ${LIBRARY_MODE} && !${VERIFY_ONLY}; then
+ # In library mode, we enforce that no custom fallback host compiler has
+ # been set via `cmake-host-compiler` option. Note: even if another
+ # Clang-based compiler is set, we cannot guarantee that it is compatible
+ # with the LLVM version that the CUDA-Q runtime is compiled against.
+ # Hence, we don't support custom host compiler for this platform.
  if [[ "$CMAKE_FALLBACK_HOST_CXX" != "$CXX" ]]; then 
  error_exit "Using remote simulation target in library mode with a custom cmake-host-compiler is unsupported. Please use the default nvq++ host compiler or enable MLIR mode."
  fi 
 
  COMPILER_FLAGS="$COMPILER_FLAGS -fembed-bitcode=all"
- # Clang made "out of line atomics" the default on arm64 targets (https://reviews.llvm.org/D93585);
- # requiring to link against `libclang_rt.builtins` on the server side when JIT'ing. 
- # Hence, disable it when using the remote platform. 
+ # Clang made "out of line atomics" the default on arm64 targets
+ # (https://reviews.llvm.org/D93585); requiring to link against
+ # `libclang_rt.builtins` on the server side when JIT'ing. Hence,
+ # disable it when using the remote platform.
  if [ "$(uname -m)" == "aarch64" ]; then
  COMPILER_FLAGS="$COMPILER_FLAGS -mno-outline-atomics"
  fi
@@ -53,7 +56,8 @@ while [ $# -gt 1 ]; do
  --remote-mqpu-backend)
  PLATFORM_EXTRA_ARGS="$PLATFORM_EXTRA_ARGS;backend;$2"
  ;;
- # Auto launch (and shutdown) server processes (providing the number of instances)
+ # Auto launch (and shutdown) server processes (providing the number of
+ # instances).
  --remote-mqpu-auto-launch)
  PLATFORM_EXTRA_ARGS="$PLATFORM_EXTRA_ARGS;auto_launch;$2"
  ;;

diff --git a/runtime/cudaq/platform/orca/orca.config b/runtime/cudaq/platform/orca/orca.config
@@ -9,11 +9,6 @@ GEN_TARGET_BACKEND=true
 # Add the rest-qpu library to the link list
 LINKLIBS="${LINKLIBS} -lcudaq-orca-qpu"
 
-# Library mode is only for simulators, 
-# and it is the default, physical backends must 
-# turn this off
-LIBRARY_MODE=false
-
 PLATFORM_EXTRA_ARGS=""
 # NB: extra arguments always take the form:
 # --<target>-<option> <value>

diff --git a/runtime/cudaq/qis/managers/photonics/PhotonicsExecutionManager.cpp b/runtime/cudaq/qis/managers/photonics/PhotonicsExecutionManager.cpp
@@ -5,6 +5,7 @@
  * This source code and the accompanying materials are made available under *
  * the terms of the Apache License 2.0 which accompanies this distribution. *
  ******************************************************************************/
+
 #include "common/Logger.h"
 #include "cudaq/qis/managers/BasicExecutionManager.h"
 #include "cudaq/spin_op.h"

diff --git a/runtime/cudaq/qis/managers/photonics/photonics.config b/runtime/cudaq/qis/managers/photonics/photonics.config
@@ -6,4 +6,5 @@
 # the terms of the Apache License 2.0 which accompanies this distribution. #
 # ============================================================================ #
 
-LIBRARY_MODE_EXECUTION_MANAGER=photonics
+LIBRARY_MODE_EXECUTION_MANAGER=photonics
+LIBRARY_MODE=true
diff --git a/targettests/Remote-Sim/unsupport_args.cpp b/targettests/Remote-Sim/unsupport_args.cpp
@@ -7,10 +7,9 @@
  ******************************************************************************/
 
 // REQUIRES: remote-sim
+// RUN: nvq++ %cpp_std --library-mode -verify --target remote-mqpu %s
 
-// clang-format off
-// RUN: nvq++ %cpp_std --target remote-mqpu %s 2>&1 | FileCheck %s
-// clang-format on
+// Tests for some of the errors that will be emitted in library-mode.
 
 #include <cudaq.h>
 
@@ -42,6 +41,25 @@ struct run_grover {
  }
 };
 
+// expected-error@* {{Callable entry-point kernel arguments are not supported for the remote simulator platform in library mode. Please rewrite the entry point kernel or use MLIR mode.}}
+// expected-error@* {{reference to overloaded function could not be resolved; did you mean to call it?}}
+// expected-error@* {{no matching function for call to 'forward'}}
+
+// 13 notes get generated.
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+// expected-note@* {{}}
+
 struct oracle {
  void operator()(cudaq::qvector<> &q) __qpu__ {
  z<cudaq::ctrl>(q[0], q[2]);
@@ -58,16 +76,10 @@ __qpu__ void oracle_func(cudaq::qvector<> &q) {
 int main() {
  {
  auto counts = cudaq::sample(run_grover{}, 3, 1, oracle{});
- // CHECK: Callable entry-point kernel arguments are not supported for the
- // CHECK: remote simulator platform in library mode. Please rewrite the
- // entry CHECK: point kernel or use MLIR mode.
  counts.dump();
  }
  {
  auto counts = cudaq::sample(run_grover{}, 3, 1, oracle_func);
- // CHECK NEXT: Callable entry-point kernel arguments are not supported for
- // CHECK NEXT: the remote simulator platform in library mode. Please
- // CHECK NEXT: rewrite the entry point kernel or use MLIR mode.
  counts.dump();
  }
 }
diff --git a/tools/nvqpp/nvq++.in b/tools/nvqpp/nvq++.in
@@ -351,6 +351,7 @@ DISABLE_QUBIT_MAPPING=false
 NVQIR_LIBS="-lnvqir -lnvqir-"
 CPPSTD=-std=c++20
 CUDAQ_OPT_EXTRA_PASSES=
+VERIFY_ONLY=false
 
 # Provide a default backend, user can override
 NVQIR_SIMULATION_BACKEND="qpp"
@@ -375,7 +376,7 @@ fi
 # We default to LIBRARY_MODE, physical
 # Quantum Targets can override this to turn on
 # our MLIR compilation workflow.
-LIBRARY_MODE=true
+LIBRARY_MODE=false
 
 CXX=${LLVMBIN}clang++${llvm_suffix}
 LINKER_CXX=${CXX}
@@ -503,6 +504,15 @@ while [ $# -ne 0 ]; do
  --enable-mlir)
  LIBRARY_MODE=false
  ;;
+ --library-mode)
+ LIBRARY_MODE=true
+ ;;
+ -verify)
+ VERIFY_ONLY=true
+ DO_LINK=false
+ ARGS="${ARGS} -Xclang -verify"
+ CUDAQ_QUAKE_ARGS="${CUDAQ_QUAKE_ARGS} -verify"
+ ;;
  --clang-verbose | -clang-verbose)
  CLANG_VERBOSE="-v"
  ;;
@@ -725,28 +735,32 @@ fi
 OPT_PASSES="builtin.module(${OPT_PASSES})"
 
 if ${SHOW_VERSION} && [ -z "$SRCS" ] && [ -z "$OBJS" ]; then
- # If version is requested and no source files were provided, log the version string.
+ # If version is requested and no source files were provided, log the
+ # version string.
  echo "$NVQPP_VERSION_STRING"
  DO_LINK=false
 fi
 
 for i in ${SRCS}; do
  file=$(basename -s .cc -s .cpp $i)
 
- # If LIBRARY_MODE explicitly requested, then
- # simply compile with the classical compiler.
+ # If LIBRARY_MODE explicitly requested, then simply compile with the
+ # classical compiler.
  if ${LIBRARY_MODE}; then
  run ${CMAKE_FALLBACK_HOST_CXX} ${CLANG_VERBOSE} ${CLANG_RESOURCE_DIR} ${COMPILER_FLAGS} ${PREPROCESSOR_DEFINES} ${INCLUDES} ${ARGS} -o ${file}.o -c $i
  OBJS="${OBJS} ${file}.o"
- # Go to the next iteration, maybe there
- # will be cudaq kernels there
+ # Go to the next iteration, maybe there will be cudaq kernels
+ # there
  continue
  fi
 
- # If we make it here, we have CUDA-Q kernels, need
- # to map to MLIR and output an LLVM file for the classical code
+ # If we make it here, we have CUDA-Q kernels, need to map to MLIR and
+ # output an LLVM file for the classical code
  run ${TOOLBIN}cudaq-quake ${CLANG_VERBOSE} ${CLANG_RESOURCE_DIR} ${PREPROCESSOR_DEFINES} ${INCLUDES} ${CUDAQ_QUAKE_ARGS} --emit-llvm-file $i -o ${file}.qke
  TMPFILES="${TMPFILES} ${file}.ll ${file}.qke"
+ if ${VERIFY_ONLY}; then
+ continue
+ fi
 
  # Run the MLIR passes
  QUAKE_IN=${file}.qke