From 1bed7e4d494ed0077920801b9a624c2ded8404e7 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Sun, 22 Oct 2023 23:01:00 -0700 Subject: [PATCH] Address MPI tracing issues on Darwin. --- build/common.mk | 16 ++++++++++++---- .../Trace/preload/gen_wrappers_mpi.py | 3 ++- src/Infrastructure/Trace/preload/makefile | 2 +- src/Infrastructure/Trace/preload/preload.C | 4 ++-- src/Infrastructure/Trace/preload/preload_io.C | 3 ++- src/Infrastructure/Trace/preload/preload_mpi.C | 3 ++- .../Trace/tests/ESMF_TraceMPIUTest.F90 | 10 ++++++---- 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/build/common.mk b/build/common.mk index 3f457a63d6..b717595bbf 100644 --- a/build/common.mk +++ b/build/common.mk @@ -2022,10 +2022,14 @@ ifeq ($(ESMF_TRACE_BUILD_SHARED),ON) ESMF_TRACE_LDPRELOAD := $(ESMF_LIBDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX) ESMF_PRELOADSCRIPT = $(ESMF_LIBDIR)/preload.sh -ifneq ($(ESMF_OS),Darwin) -ESMF_ENV_PRELOAD = LD_PRELOAD +ifeq ($(ESMF_OS),Darwin) +ESMF_ENV_PRELOAD = DYLD_INSERT_LIBRARIES +ESMF_ENV_PRELOAD_DELIMIT = ':' +ESMF_SL_PRELOAD_LIBLINKER = $(ESMF_F90LINKER) else -ESMF_ENV_PRELOAD = DYLD_INSERT_LIBRARIES +ESMF_ENV_PRELOAD = LD_PRELOAD +ESMF_ENV_PRELOAD_DELIMIT = ' ' +ESMF_SL_PRELOAD_LIBLINKER = $(ESMF_SL_LIBLINKER) endif # MPI implementations do not pick up LD_PRELOAD @@ -2048,7 +2052,11 @@ endif build_preload_script: -@echo "#!/bin/sh" > $(ESMF_PRELOADDIR)/preload.sh -@echo "# Script to preload ESMF dynamic trace library" >> $(ESMF_PRELOADDIR)/preload.sh - -@echo 'env LD_PRELOAD="$$LD_PRELOAD $(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh + -@echo 'if [ "$$$(ESMF_ENV_PRELOAD)" != "" ]; then' >> $(ESMF_PRELOADDIR)/preload.sh + -@echo 'env $(ESMF_ENV_PRELOAD)="$$$(ESMF_ENV_PRELOAD)$(ESMF_ENV_PRELOAD_DELIMIT)$(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh + -@echo 'else' >> $(ESMF_PRELOADDIR)/preload.sh + -@echo 'env $(ESMF_ENV_PRELOAD)="$(ESMF_PRELOADDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX)" $$*' >> $(ESMF_PRELOADDIR)/preload.sh + -@echo 'fi' >> $(ESMF_PRELOADDIR)/preload.sh chmod 755 $(ESMF_PRELOADDIR)/preload.sh ESMF_TRACE_STATICLINKLIBS := -lesmftrace_static diff --git a/src/Infrastructure/Trace/preload/gen_wrappers_mpi.py b/src/Infrastructure/Trace/preload/gen_wrappers_mpi.py index e861fa0cfc..1901f2c12e 100644 --- a/src/Infrastructure/Trace/preload/gen_wrappers_mpi.py +++ b/src/Infrastructure/Trace/preload/gen_wrappers_mpi.py @@ -139,7 +139,8 @@ * This file was generated by gen_wrappers_mpi.py. * !! DO NOT EDIT !! * - * Functions that will be preloaded with LD_PRELOAD, thereby + * Functions that will be preloaded with LD_PRELOAD + * (or DYLD_INSERT_LIBRARIES on Darwin), thereby * overriding system library functions so we can call into our * wrapper function. * diff --git a/src/Infrastructure/Trace/preload/makefile b/src/Infrastructure/Trace/preload/makefile index a4c27ca8e5..cdcdc28707 100644 --- a/src/Infrastructure/Trace/preload/makefile +++ b/src/Infrastructure/Trace/preload/makefile @@ -32,7 +32,7 @@ ifeq ($(ESMF_OS),MinGW) endif tracelib_preload: preload.o preload_mpi.o wrappers.o wrappers_mpi.o - $(ESMF_SL_LIBLINKER) $(ESMF_SL_LIBOPTS) -o $(ESMF_LDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX) $^ $(ESMF_CXXLINKOPTS) $(ESMF_CXXLINKPATHS) $(ESMF_CXXLINKRPATHS) $(ESMF_CXXLINKLIBS) $(ESMF_TRACE_ESMFLIB) + $(ESMF_SL_PRELOAD_LIBLINKER) $(ESMF_SL_LIBOPTS) -o $(ESMF_LDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX) $^ $(ESMF_CXXLINKOPTS) $(ESMF_CXXLINKPATHS) $(ESMF_CXXLINKRPATHS) $(ESMF_CXXLINKLIBS) $(ESMF_TRACE_ESMFLIB) $(MAKE) ESMF_PRELOADDIR=$(ESMF_LIBDIR) build_preload_script tracelib_static: wrappers_io.o wrappers_mpi.o wrappers.o diff --git a/src/Infrastructure/Trace/preload/preload.C b/src/Infrastructure/Trace/preload/preload.C index e085ad722e..2f74cdf735 100644 --- a/src/Infrastructure/Trace/preload/preload.C +++ b/src/Infrastructure/Trace/preload/preload.C @@ -14,8 +14,8 @@ extern "C" { /** * Called into from ESMCI_Trace.C when the trace * is opened and can receive events. The linker will preload - * this symbol when LD_PRELOAD is specified so we can - * catch this notification. + * this symbol when LD_PRELOAD (or DYLD_INSERT_LIBRARIES on Darwin) + * is specified so we can catch this notification. */ int c_esmftrace_notify_wrappers(int initialized) { if (initialized == 1) { diff --git a/src/Infrastructure/Trace/preload/preload_io.C b/src/Infrastructure/Trace/preload/preload_io.C index 270fa32f10..3ba6b165c4 100644 --- a/src/Infrastructure/Trace/preload/preload_io.C +++ b/src/Infrastructure/Trace/preload/preload_io.C @@ -2,7 +2,8 @@ * * preload_io.c * - * Functions that will be preloaded with LD_PRELOAD, thereby + * Functions that will be preloaded with LD_PRELOAD, + * (or DYLD_INSERT_LIBRARIES on Darwin), thereby * overriding system library functions so we can call into our * wrapper function. * diff --git a/src/Infrastructure/Trace/preload/preload_mpi.C b/src/Infrastructure/Trace/preload/preload_mpi.C index 8e1c9715c9..86c0f3d4cf 100644 --- a/src/Infrastructure/Trace/preload/preload_mpi.C +++ b/src/Infrastructure/Trace/preload/preload_mpi.C @@ -7,7 +7,8 @@ * This file was generated by gen_wrappers_mpi.py. * !! DO NOT EDIT !! * - * Functions that will be preloaded with LD_PRELOAD, thereby + * Functions that will be preloaded with LD_PRELOAD, + * (or DYLD_INSERT_LIBRARIES on Darwin), thereby * overriding system library functions so we can call into our * wrapper function. * diff --git a/src/Infrastructure/Trace/tests/ESMF_TraceMPIUTest.F90 b/src/Infrastructure/Trace/tests/ESMF_TraceMPIUTest.F90 index fb170979c0..7ce212727e 100644 --- a/src/Infrastructure/Trace/tests/ESMF_TraceMPIUTest.F90 +++ b/src/Infrastructure/Trace/tests/ESMF_TraceMPIUTest.F90 @@ -109,7 +109,8 @@ program ESMF_TraceMPIUTest !------------------------------------------------------------------------ !NEX_UTest - write(failMsg, *) "MPI_ALLREDUCE failed with ierr =", ierr + write(name, *) "Test MPI_ALLREDUCE ierr==0" + write(failMsg, *) "MPI_ALLREDUCE failed with ierr=", ierr #if (!defined ESMF_MPIUNI && defined ESMF_TESTTRACE) call ESMF_Test((ierr==0), name, failMsg, result, ESMF_SRCLINE) #else @@ -119,8 +120,8 @@ program ESMF_TraceMPIUTest !------------------------------------------------------------------------ !NEX_UTest - print *, "MPI_ALLREDUCE returned ", recv - write(failMsg, *) "MPI_ALLREDUCE produced unexpected result. Expected 4, got ", recv + write(name, *) "Test MPI_ALLREDUCE recv==4" + write(failMsg, *) "MPI_ALLREDUCE produced unexpected result. recv=", recv #if (!defined ESMF_MPIUNI && defined ESMF_TESTTRACE) call ESMF_Test((recv==petCount), name, failMsg, result, ESMF_SRCLINE) #else @@ -133,7 +134,8 @@ program ESMF_TraceMPIUTest !------------------------------------------------------------------------ !NEX_UTest - write(failMsg, *) "MPI call not profiled" + write(name, *) "Test MPI_ALLREDUCE call recorded as profiled" + write(failMsg, *) "MPI_ALLREDUCE call not profiled" #if (!defined ESMF_MPIUNI && defined ESMF_TESTTRACE) call ESMF_Test((mpicheck==1), name, failMsg, result, ESMF_SRCLINE) #else