Address MPI tracing issues on Darwin.

esmf-org · Oct 23, 2023 · 1bed7e4 · 1bed7e4
1 parent 53028df
commit 1bed7e4
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 14 deletions.
diff --git a/build/common.mk b/build/common.mk
@@ -2022,10 +2022,14 @@ ifeq ($(ESMF_TRACE_BUILD_SHARED),ON)
 ESMF_TRACE_LDPRELOAD := $(ESMF_LIBDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)
 ESMF_PRELOADSCRIPT = $(ESMF_LIBDIR)/preload.sh
 
-ifneq ($(ESMF_OS),Darwin)
-ESMF_ENV_PRELOAD = LD_PRELOAD
+ifeq ($(ESMF_OS),Darwin)
+ESMF_ENV_PRELOAD          = DYLD_INSERT_LIBRARIES
+ESMF_ENV_PRELOAD_DELIMIT  = ':'
+ESMF_SL_PRELOAD_LIBLINKER = $(ESMF_F90LINKER)
 else
-ESMF_ENV_PRELOAD = DYLD_INSERT_LIBRARIES
+ESMF_ENV_PRELOAD          = LD_PRELOAD
+ESMF_ENV_PRELOAD_DELIMIT  = ' '
+ESMF_SL_PRELOAD_LIBLINKER = $(ESMF_SL_LIBLINKER)
 endif
 
 # MPI implementations do not pick up LD_PRELOAD
@@ -2048,7 +2052,11 @@ endif
 build_preload_script:
 	-@echo "#!/bin/sh" > $(ESMF_PRELOADDIR)/preload.sh
 	-@echo "# Script to preload ESMF dynamic trace library" >> $(ESMF_PRELOADDIR)/preload.sh
-	-@echo 'env LD_PRELOAD="$$LD_PRELOAD $(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'if [ "$$$(ESMF_ENV_PRELOAD)" != "" ]; then' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'env $(ESMF_ENV_PRELOAD)="$$$(ESMF_ENV_PRELOAD)$(ESMF_ENV_PRELOAD_DELIMIT)$(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'else' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'env $(ESMF_ENV_PRELOAD)="$(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh
+	-@echo 'fi' >> $(ESMF_PRELOADDIR)/preload.sh
 	chmod 755 $(ESMF_PRELOADDIR)/preload.sh
 
 ESMF_TRACE_STATICLINKLIBS := -lesmftrace_static

diff --git a/src/Infrastructure/Trace/preload/gen_wrappers_mpi.py b/src/Infrastructure/Trace/preload/gen_wrappers_mpi.py
@@ -139,7 +139,8 @@
  *  This file was generated by gen_wrappers_mpi.py.
  *  !! DO NOT EDIT !!
  *
- * Functions that will be preloaded with LD_PRELOAD, thereby
+ * Functions that will be preloaded with LD_PRELOAD
+ * (or DYLD_INSERT_LIBRARIES on Darwin), thereby
  * overriding system library functions so we can call into our
  * wrapper function.
  *

diff --git a/src/Infrastructure/Trace/preload/makefile b/src/Infrastructure/Trace/preload/makefile
@@ -32,7 +32,7 @@ ifeq  ($(ESMF_OS),MinGW)
 endif
 
 tracelib_preload: preload.o preload_mpi.o wrappers.o wrappers_mpi.o
-	$(ESMF_SL_LIBLINKER) $(ESMF_SL_LIBOPTS) -o $(ESMF_LDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX) $^ $(ESMF_CXXLINKOPTS) $(ESMF_CXXLINKPATHS) $(ESMF_CXXLINKRPATHS) $(ESMF_CXXLINKLIBS) $(ESMF_TRACE_ESMFLIB)
+	$(ESMF_SL_PRELOAD_LIBLINKER) $(ESMF_SL_LIBOPTS) -o $(ESMF_LDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX) $^ $(ESMF_CXXLINKOPTS) $(ESMF_CXXLINKPATHS) $(ESMF_CXXLINKRPATHS) $(ESMF_CXXLINKLIBS) $(ESMF_TRACE_ESMFLIB)
 	$(MAKE) ESMF_PRELOADDIR=$(ESMF_LIBDIR) build_preload_script
 
 tracelib_static: wrappers_io.o wrappers_mpi.o wrappers.o

diff --git a/src/Infrastructure/Trace/preload/preload.C b/src/Infrastructure/Trace/preload/preload.C
@@ -14,8 +14,8 @@ extern "C" {
   /**
    * Called into from ESMCI_Trace.C when the trace
    * is opened and can receive events.  The linker will preload
-   * this symbol when LD_PRELOAD is specified so we can
-   * catch this notification.
+   * this symbol when LD_PRELOAD (or DYLD_INSERT_LIBRARIES on Darwin)
+   * is specified so we can catch this notification.
    */
   int c_esmftrace_notify_wrappers(int initialized) {
     if (initialized == 1) {

diff --git a/src/Infrastructure/Trace/preload/preload_io.C b/src/Infrastructure/Trace/preload/preload_io.C
@@ -2,7 +2,8 @@
  *
  * preload_io.c
  *
- * Functions that will be preloaded with LD_PRELOAD, thereby
+ * Functions that will be preloaded with LD_PRELOAD,
+ * (or DYLD_INSERT_LIBRARIES on Darwin), thereby
  * overriding system library functions so we can call into our
  * wrapper function.
  *

diff --git a/src/Infrastructure/Trace/preload/preload_mpi.C b/src/Infrastructure/Trace/preload/preload_mpi.C
@@ -7,7 +7,8 @@
  *  This file was generated by gen_wrappers_mpi.py.
  *  !! DO NOT EDIT !!
  *
- * Functions that will be preloaded with LD_PRELOAD, thereby
+ * Functions that will be preloaded with LD_PRELOAD,
+ * (or DYLD_INSERT_LIBRARIES on Darwin), thereby
  * overriding system library functions so we can call into our
  * wrapper function.
  *

diff --git a/src/Infrastructure/Trace/tests/ESMF_TraceMPIUTest.F90 b/src/Infrastructure/Trace/tests/ESMF_TraceMPIUTest.F90
@@ -109,7 +109,8 @@ program ESMF_TraceMPIUTest
 
   !------------------------------------------------------------------------
   !NEX_UTest
-  write(failMsg, *) "MPI_ALLREDUCE failed with ierr =", ierr
+  write(name, *) "Test MPI_ALLREDUCE ierr==0"
+  write(failMsg, *) "MPI_ALLREDUCE failed with ierr=", ierr
 #if (!defined ESMF_MPIUNI && defined ESMF_TESTTRACE)
   call ESMF_Test((ierr==0), name, failMsg, result, ESMF_SRCLINE)
 #else
@@ -119,8 +120,8 @@ program ESMF_TraceMPIUTest
 
   !------------------------------------------------------------------------
   !NEX_UTest
-  print *, "MPI_ALLREDUCE returned ", recv
-  write(failMsg, *) "MPI_ALLREDUCE produced unexpected result. Expected 4, got ", recv
+  write(name, *) "Test MPI_ALLREDUCE recv==4"
+  write(failMsg, *) "MPI_ALLREDUCE produced unexpected result. recv=", recv
 #if (!defined ESMF_MPIUNI && defined ESMF_TESTTRACE)
   call ESMF_Test((recv==petCount), name, failMsg, result, ESMF_SRCLINE)
 #else
@@ -133,7 +134,8 @@ program ESMF_TraceMPIUTest
 
   !------------------------------------------------------------------------
   !NEX_UTest
-  write(failMsg, *) "MPI call not profiled"
+  write(name, *) "Test MPI_ALLREDUCE call recorded as profiled"
+  write(failMsg, *) "MPI_ALLREDUCE call not profiled"
 #if (!defined ESMF_MPIUNI && defined ESMF_TESTTRACE)
   call ESMF_Test((mpicheck==1), name, failMsg, result, ESMF_SRCLINE)
 #else