From c20127097894992dc7e8d300e9795710926ce3ea Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 6 Mar 2023 16:41:33 -0700 Subject: [PATCH 01/10] redist_arraycreate1de: Remove error check for > 1 tile --- src/Infrastructure/IO/src/ESMCI_IO.C | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index 4004e7364f..329ac2db70 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1197,10 +1197,6 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet DistGrid *dg_orig = src_array_p->getDistGrid(); - localrc = (dg_orig->getTileCount() == 1) ? ESMF_SUCCESS : ESMF_RC_NOT_IMPL; - if (ESMC_LogDefault.MsgFoundError(localrc, "Tile count != 1 is not supported", ESMC_CONTEXT, rc)) - return; - const int *minIndexTile = dg_orig->getMinIndexPDimPTile(); const int *maxIndexTile = dg_orig->getMaxIndexPDimPTile(); const int *distgridToArrayMap = src_array_p->getDistGridToArrayMap(); From 538079a933bc9e3278189a24d5fec62c4201ec89 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 6 Mar 2023 16:42:40 -0700 Subject: [PATCH 02/10] Rename variables to be more precisely named --- src/Infrastructure/IO/src/ESMCI_IO.C | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index 329ac2db70..089d10cfd3 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1197,8 +1197,8 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet DistGrid *dg_orig = src_array_p->getDistGrid(); - const int *minIndexTile = dg_orig->getMinIndexPDimPTile(); - const int *maxIndexTile = dg_orig->getMaxIndexPDimPTile(); + const int *minIndexPDimPTile = dg_orig->getMinIndexPDimPTile(); + const int *maxIndexPDimPTile = dg_orig->getMaxIndexPDimPTile(); const int *distgridToArrayMap = src_array_p->getDistGridToArrayMap(); int ndims = dg_orig->getDimCount(); @@ -1208,42 +1208,42 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet for (int i=0; i minIndexTileVec; - std::vector maxIndexTileVec; + std::vector minIndexPDimPTileVec; + std::vector maxIndexPDimPTileVec; std::vector distgridToArrayMapVec; if (replicatedDims>0){ // eliminate replicated dimensions from the destination for (int i=0; i keep - minIndexTileVec.push_back(minIndexTile[i]); - maxIndexTileVec.push_back(maxIndexTile[i]); + minIndexPDimPTileVec.push_back(minIndexPDimPTile[i]); + maxIndexPDimPTileVec.push_back(maxIndexPDimPTile[i]); distgridToArrayMapVec.push_back(distgridToArrayMap[i]); } } - if (minIndexTileVec.size()<1){ + if (minIndexPDimPTileVec.size()<1){ ESMC_LogDefault.MsgFoundError(ESMF_RC_INTNRL_BAD, "Not enough distributed dimensions", ESMC_CONTEXT, rc); return; // bail out } // now point to the set of reduced lists - minIndexTile = &(minIndexTileVec[0]); - maxIndexTile = &(maxIndexTileVec[0]); + minIndexPDimPTile = &(minIndexPDimPTileVec[0]); + maxIndexPDimPTile = &(maxIndexPDimPTileVec[0]); distgridToArrayMap = &(distgridToArrayMapVec[0]); } - if ((maxIndexTile[0]-minIndexTile[0]+1) minIndexInterface((int*)minIndexTile, ndims-replicatedDims); - ESMCI::InterArray maxIndexInterface((int*)maxIndexTile, ndims-replicatedDims); + ESMCI::InterArray minIndexInterface((int*)minIndexPDimPTile, ndims-replicatedDims); + ESMCI::InterArray maxIndexInterface((int*)maxIndexPDimPTile, ndims-replicatedDims); #if 0 std::cout << ESMC_METHOD << "[" << me << "]: setting maxindex to: ("; for (int i=0; i Date: Fri, 26 May 2023 16:25:20 -0600 Subject: [PATCH 03/10] Add unit tests of multi-tile I/O with other than 1 DE per PET These currently fail as expected --- .../IO/tests/ESMF_IO_MultitileUTest.F90 | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 index 6ad10ee173..39b8edfdc3 100644 --- a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 +++ b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 @@ -47,6 +47,9 @@ program ESMF_IO_MultitileUTest type(ESMF_VM) :: vm integer :: localPet type(ESMF_Grid) :: grid6tile + type(ESMF_Grid) :: grid6tileUnevenDEs + integer :: grid6tileUnevenDEsLdeCount + integer :: lde type(ESMF_DistGrid) :: distgrid3tile ! Fields used for writing: @@ -58,6 +61,9 @@ program ESMF_IO_MultitileUTest ! This field is not in the field bundle: type(ESMF_Field) :: field3 real(ESMF_KIND_R8), pointer :: field3Data(:,:) + ! This field is for tests with something other than 1 DE per PET: + type(ESMF_Field) :: field1UnevenDEs + real(ESMF_KIND_R8), pointer :: field1UnevenDEsData(:,:) ! Fields used for reading: ! @@ -68,6 +74,9 @@ program ESMF_IO_MultitileUTest ! This field is not in the field bundle: type(ESMF_Field) :: field3Read real(ESMF_KIND_R8), pointer :: field3ReadData(:,:) + ! This field is for tests with something other than 1 DE per PET: + type(ESMF_Field) :: field1UnevenDEsRead + real(ESMF_KIND_R8), pointer :: field1UnevenDEsReadData(:,:) ! This is used for error testing: type(ESMF_Grid) :: gridSingleTile @@ -99,6 +108,7 @@ program ESMF_IO_MultitileUTest character(len=*), parameter :: fileNameFields = "ESMF_IO_MultitileUTestFields*.nc" character(len=*), parameter :: fileNameArrays = "ESMF_IO_MultitileUTestArrays*.nc" character(len=*), parameter :: fileNameFail = "ESMF_IO_MultitileUTestFail*.nc" + character(len=*), parameter :: fileNameUnevenDEs = "ESMF_IO_MultitileUTestUnevenDEs*.nc" !------------------------------------------------------------------------ call ESMF_TestStart(ESMF_SRCLINE, rc=rc) ! calls ESMF_Initialize() internally @@ -340,6 +350,55 @@ program ESMF_IO_MultitileUTest #endif !------------------------------------------------------------------------ + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Write a multi-tile Field with uneven DEs per PET" + write(failMsg, *) "Did not return ESMF_SUCCESS" + call ESMF_FieldWrite(field1UnevenDEs, fileName=fileNameUnevenDEs, overwrite=.true., rc=rc) +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" + call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Read a multi-tile Field with uneven DEs per PET" + write(failMsg, *) "Did not return ESMF_SUCCESS" + call ESMF_FieldRead(field1UnevenDEsRead, fileName=fileNameUnevenDEs, rc=rc) +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" + call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Confirm that Field-read field matches original with uneven DEs per PET" + write(failMsg, *) "Read-in field differs from original" + allEqual = .true. + do lde = 0, grid6tileUnevenDEsLdeCount - 1 + ! For simplicity, bail out if the following FieldGets fail rather than calling them their own unit test + call ESMF_FieldGet(field1UnevenDEs, localDe=lde, farrayPtr=field1UnevenDEsData, rc=rc) + if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_FieldGet(field1UnevenDEsRead, localDe=lde, farrayPtr=field1UnevenDEsReadData, rc=rc) + if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (.not. all(field1UnevenDEsReadData == field1UnevenDEsData)) then + allEqual = .false. + end if + end do +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test(allEqual, name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Comparison did not fail as expected" + call ESMF_Test(.not. allEqual, name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + #endif ! ESMF_TESTEXHAUSTIVE !------------------------------------------------------------------------ @@ -353,9 +412,11 @@ subroutine createFields(rc) integer, intent(out) :: rc integer :: decompPTile(2,6) + integer :: decompPTileUnevenDEs(2,6) type(ESMF_ArraySpec) :: arraySpec type(ESMF_ArraySpec) :: arraySpec_w_ungridded type(ESMF_Array) :: array1 + type(ESMF_DELayout) :: delayout real(ESMF_KIND_R8), pointer :: coordPtrX(:,:), coordPtrY(:,:) integer :: u1, u2, i, j real :: multiplier @@ -475,6 +536,44 @@ subroutine createFields(rc) call ESMF_FieldBundleAdd(fieldBundleRead, [field1Read, field2Read, field1CopyRead, field4dRead], rc=rc) if (rc /= ESMF_SUCCESS) return + !------------------------------------------------------------------------ + ! Set up a 6-tile grid with an uneven distribution of DEs to PETs, and create a field + ! on this grid + !------------------------------------------------------------------------ + + ! Decomposition for 8 PEs but 16 DEs + ! + ! The number of DEs per tile is: + ! Tile : 1 2 3 4 5 6 + ! # DEs: 2 1 6 1 3 3 + ! + ! The DEs are scattered in a disorganized fashion across PETs. We have the following + ! number of DEs on each PET: + ! PET #: 0 1 2 3 4 5 6 7 + ! # DEs: 1 2 3 4 0 3 0 3 + decompPTileUnevenDEs(1,:) = [2,1,3,1,1,3] + decompPTileUnevenDEs(2,:) = [1,1,2,1,3,1] + delayout = ESMF_DELayoutCreate(petMap=[3,2,5,5,1,3,2,1,7,3,0,7,2,7,3,5]) + grid6tileUnevenDEs = ESMF_GridCreateCubedSphere( & + tilesize = 6, & + regDecompPTile = decompPTileUnevenDEs, & + delayout = delayout, & + staggerLocList = [ESMF_STAGGERLOC_CENTER], & + rc = rc) + if (rc /= ESMF_SUCCESS) return + call ESMF_GridGet(grid6tileUnevenDEs, localDECount=grid6tileUnevenDEsLdeCount, rc=rc) + if (rc /= ESMF_SUCCESS) return + + field1UnevenDEs = ESMF_FieldCreate(grid6tileUnevenDEs, arraySpec, name="field1UnevenDEs", rc=rc) + if (rc /= ESMF_SUCCESS) return + call ESMF_FieldFill(field1UnevenDEs, dataFillScheme='sincos', member=1, rc=rc) + if (rc /= ESMF_SUCCESS) return + ! Note that we can't get farrayPtr here because we'll need to do that in a loop over DEs + + field1UnevenDEsRead = ESMF_FieldCreate(grid6tileUnevenDEs, arraySpec, name="field1UnevenDEs", rc=rc) + if (rc /= ESMF_SUCCESS) return + ! Note that we can't get farrayPtr here because we'll need to do that in a loop over DEs + end subroutine createFields subroutine createSingleTileField(rc) From 0cd0c57d0a388d62647280451acb87c0f0a3eee4 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 26 May 2023 18:19:05 -0600 Subject: [PATCH 04/10] First-order changes needed for redist_arraycreate1de with multi-tile Partially addresses esmf-org/esmf#84 Some things still needed: - Combining undistributed dimensions with something other than 1 DE per PET leads to failures in the multi-tile case - Need to extend code for handling replicatedDims > 0 with the multi-tile case - Need to generalize an error-check --- src/Infrastructure/IO/src/ESMCI_IO.C | 57 ++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index 089d10cfd3..ffad7dffe7 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1203,6 +1203,7 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet int ndims = dg_orig->getDimCount(); int rank = src_array_p->getRank(); + int tileCount = dg_orig->getTileCount(); int replicatedDims=0; for (int i=0; i maxIndexPDimPTileVec; std::vector distgridToArrayMapVec; if (replicatedDims>0){ + // FIXME(wjs, 2023-05-26) Still need to fix this loop to work with the multi-tile + // case; then remove the following error check + if (tileCount > 1) { + ESMC_LogDefault.MsgFoundError(ESMF_RC_NOT_IMPL, + "Multi-tile with > 1 DE per PET and replicated dims not yet implemented", ESMC_CONTEXT, rc); + return; + } + // eliminate replicated dimensions from the destination for (int i=0; i minIndexInterface((int*)minIndexPDimPTile, ndims-replicatedDims); - ESMCI::InterArray maxIndexInterface((int*)maxIndexPDimPTile, ndims-replicatedDims); + DistGrid *distgrid; + if (tileCount == 1) { + ESMCI::InterArray minIndexInterface((int*)minIndexPDimPTile, ndims-replicatedDims); + ESMCI::InterArray maxIndexInterface((int*)maxIndexPDimPTile, ndims-replicatedDims); #if 0 - std::cout << ESMC_METHOD << "[" << me << "]: setting maxindex to: ("; - for (int i=0; i minIndexInterface((int*)minIndexPDimPTile, 2, dummyLen); + ESMCI::InterArray maxIndexInterface((int*)maxIndexPDimPTile, 2, dummyLen); + // create default DistGrid, which means 1DE per PET + distgrid = DistGrid::create(&minIndexInterface, &maxIndexInterface, NULL, + NULL, 0, 0, NULL, NULL, NULL, NULL, NULL, (ESMCI::DELayout*)NULL, NULL, + &localrc); + if (ESMC_LogDefault.MsgFoundError(localrc, ESMCI_ERR_PASSTHRU, ESMC_CONTEXT, rc)) + return; + } // std::cout << ESMC_METHOD << ": creating temp Array for redistribution" << std::endl; ESMCI::ArraySpec arrayspec; From 53a4ecb75b46c329604641c4b06eafb1490c3899 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 20 Sep 2023 09:09:31 -0600 Subject: [PATCH 05/10] Add unit test of multi-tile I/O with both uneven DEs and ungridded dims --- .../IO/tests/ESMF_IO_MultitileUTest.F90 | 93 ++++++++++++++++++- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 index 39b8edfdc3..a58ef41760 100644 --- a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 +++ b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 @@ -61,9 +61,10 @@ program ESMF_IO_MultitileUTest ! This field is not in the field bundle: type(ESMF_Field) :: field3 real(ESMF_KIND_R8), pointer :: field3Data(:,:) - ! This field is for tests with something other than 1 DE per PET: - type(ESMF_Field) :: field1UnevenDEs + ! These fields are for tests with something other than 1 DE per PET: + type(ESMF_Field) :: field1UnevenDEs, field4dUnevenDEs real(ESMF_KIND_R8), pointer :: field1UnevenDEsData(:,:) + real(ESMF_KIND_R8), pointer :: field4dUnevenDEsData(:,:,:,:) ! Fields used for reading: ! @@ -74,9 +75,10 @@ program ESMF_IO_MultitileUTest ! This field is not in the field bundle: type(ESMF_Field) :: field3Read real(ESMF_KIND_R8), pointer :: field3ReadData(:,:) - ! This field is for tests with something other than 1 DE per PET: - type(ESMF_Field) :: field1UnevenDEsRead + ! These fields are for tests with something other than 1 DE per PET: + type(ESMF_Field) :: field1UnevenDEsRead, field4dUnevenDEsRead real(ESMF_KIND_R8), pointer :: field1UnevenDEsReadData(:,:) + real(ESMF_KIND_R8), pointer :: field4dUnevenDEsReadData(:,:,:,:) ! This is used for error testing: type(ESMF_Grid) :: gridSingleTile @@ -399,6 +401,55 @@ program ESMF_IO_MultitileUTest #endif !------------------------------------------------------------------------ + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Write a multi-tile Field with uneven DEs per PET and ungridded dims" + write(failMsg, *) "Did not return ESMF_SUCCESS" + call ESMF_FieldWrite(field4dUnevenDEs, fileName=fileNameUnevenDEs, overwrite=.true., rc=rc) +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" + call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Read a multi-tile Field with uneven DEs per PET and ungridded dims" + write(failMsg, *) "Did not return ESMF_SUCCESS" + call ESMF_FieldRead(field4dUnevenDEsRead, fileName=fileNameUnevenDEs, rc=rc) +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" + call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Confirm that Field-read field matches original with uneven DEs per PET and ungridded dims" + write(failMsg, *) "Read-in field differs from original" + allEqual = .true. + do lde = 0, grid6tileUnevenDEsLdeCount - 1 + ! For simplicity, bail out if the following FieldGets fail rather than calling them their own unit test + call ESMF_FieldGet(field4dUnevenDEs, localDe=lde, farrayPtr=field4dUnevenDEsData, rc=rc) + if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_FieldGet(field4dUnevenDEsRead, localDe=lde, farrayPtr=field4dUnevenDEsReadData, rc=rc) + if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (.not. all(field4dUnevenDEsReadData == field4dUnevenDEsData)) then + allEqual = .false. + end if + end do +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test(allEqual, name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Comparison did not fail as expected" + call ESMF_Test(.not. allEqual, name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + #endif ! ESMF_TESTEXHAUSTIVE !------------------------------------------------------------------------ @@ -537,7 +588,7 @@ subroutine createFields(rc) if (rc /= ESMF_SUCCESS) return !------------------------------------------------------------------------ - ! Set up a 6-tile grid with an uneven distribution of DEs to PETs, and create a field + ! Set up a 6-tile grid with an uneven distribution of DEs to PETs, and create fields ! on this grid !------------------------------------------------------------------------ @@ -574,6 +625,38 @@ subroutine createFields(rc) if (rc /= ESMF_SUCCESS) return ! Note that we can't get farrayPtr here because we'll need to do that in a loop over DEs + field4dUnevenDEs = ESMF_FieldCreate(grid6tileUnevenDEs, arraySpec_w_ungridded, name="field4dUnevenDEs", & + ungriddedLBound=[2,15], ungriddedUBound=[4,18], & + ! 2nd and 4th dimensions are ungridded dimensions + gridToFieldMap=[1,3], & + rc=rc) + if (rc /= ESMF_SUCCESS) return + do lde = 0, grid6tileUnevenDEsLdeCount-1 + call ESMF_FieldGet(field4dUnevenDEs, localDe=lde, farrayPtr=field4dUnevenDEsData, rc=rc) + if (rc /= ESMF_SUCCESS) return + call ESMF_GridGetCoord(grid6tileUnevenDEs, coordDim=1, localDe=lde, farrayPtr=coordPtrX, rc=rc) + if (rc /= ESMF_SUCCESS) return + call ESMF_GridGetCoord(grid6tileUnevenDEs, coordDim=2, localDe=lde, farrayPtr=coordPtrY, rc=rc) + if (rc /= ESMF_SUCCESS) return + do u1 = 2,4 + do u2 = 15,18 + do i = lbound(field4dUnevenDEsData, 1), ubound(field4dUnevenDEsData, 1) + do j = lbound(field4dUnevenDEsData, 3), ubound(field4dUnevenDEsData, 3) + multiplier = 5.**(u2-15) + field4dUnevenDEsData(i,u1,j,u2) = u1*multiplier*(coordPtrX(i,j) - coordPtrY(i,j)) + end do + end do + end do + end do + end do + + field4dUnevenDEsRead = ESMF_FieldCreate(grid6tileUnevenDEs, arraySpec_w_ungridded, name="field4dUnevenDEs", & + ungriddedLBound=[2,15], ungriddedUBound=[4,18], & + ! 2nd and 4th dimensions are ungridded dimensions + gridToFieldMap=[1,3], & + rc=rc) + if (rc /= ESMF_SUCCESS) return + end subroutine createFields subroutine createSingleTileField(rc) From 28f951b333b460b60526523ecba088c8548386a3 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 26 Sep 2023 11:03:42 -0600 Subject: [PATCH 06/10] Multi-tile I/O with replicated dims: add unit tests & try making it work I think this should work, but it currently fails in the read. I want to save this work, though, so I'm committing despite the failures. --- src/Infrastructure/IO/src/ESMCI_IO.C | 19 ++- .../IO/tests/ESMF_IO_MultitileUTest.F90 | 137 +++++++++++++++++- 2 files changed, 139 insertions(+), 17 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index ceed5caf28..35afc7a9f0 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1262,23 +1262,22 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet std::vector maxIndexPDimPTileVec; std::vector distgridToArrayMapVec; if (replicatedDims>0){ - // FIXME(wjs, 2023-05-26) Still need to fix this loop to work with the multi-tile - // case; then remove the following error check - if (tileCount > 1) { - ESMC_LogDefault.MsgFoundError(ESMF_RC_NOT_IMPL, - "Multi-tile with > 1 DE per PET and replicated dims not yet implemented", ESMC_CONTEXT, rc); - return; - } - // eliminate replicated dimensions from the destination for (int i=0; i keep - minIndexPDimPTileVec.push_back(minIndexPDimPTile[i]); - maxIndexPDimPTileVec.push_back(maxIndexPDimPTile[i]); distgridToArrayMapVec.push_back(distgridToArrayMap[i]); } } + for (int tile=0; tile keep + minIndexPDimPTileVec.push_back(minIndexPDimPTile[tile*ndims + i]); + maxIndexPDimPTileVec.push_back(maxIndexPDimPTile[tile*ndims + i]); + } + } + } if (minIndexPDimPTileVec.size()<1){ ESMC_LogDefault.MsgFoundError(ESMF_RC_INTNRL_BAD, "Not enough distributed dimensions", ESMC_CONTEXT, rc); diff --git a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 index a58ef41760..cb112da2e9 100644 --- a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 +++ b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 @@ -49,8 +49,8 @@ program ESMF_IO_MultitileUTest type(ESMF_Grid) :: grid6tile type(ESMF_Grid) :: grid6tileUnevenDEs integer :: grid6tileUnevenDEsLdeCount + integer :: arrayWithReplicatedDimsLdeCount integer :: lde - type(ESMF_DistGrid) :: distgrid3tile ! Fields used for writing: ! @@ -94,6 +94,9 @@ program ESMF_IO_MultitileUTest ! This array is not in the array bundle: type(ESMF_Array) :: array3 real(ESMF_KIND_R8), pointer :: array3Data(:,:) + ! This array is for tests with something other than 1 DE per PET: + type(ESMF_Array) :: arrayWithReplicatedDims + real(ESMF_KIND_R8), pointer :: arrayWithReplicatedDimsData(:,:) ! Arrays used for reading: ! @@ -104,13 +107,17 @@ program ESMF_IO_MultitileUTest ! This array is not in the array bundle: type(ESMF_Array) :: array3Read real(ESMF_KIND_R8), pointer :: array3ReadData(:,:) + ! This array is for tests with something other than 1 DE per PET: + type(ESMF_Array) :: arrayWithReplicatedDimsRead + real(ESMF_KIND_R8), pointer :: arrayWithReplicatedDimsReadData(:,:) logical :: allEqual character(len=*), parameter :: fileNameFields = "ESMF_IO_MultitileUTestFields*.nc" character(len=*), parameter :: fileNameArrays = "ESMF_IO_MultitileUTestArrays*.nc" character(len=*), parameter :: fileNameFail = "ESMF_IO_MultitileUTestFail*.nc" - character(len=*), parameter :: fileNameUnevenDEs = "ESMF_IO_MultitileUTestUnevenDEs*.nc" + character(len=*), parameter :: fileNameUnevenDEFields = "ESMF_IO_MultitileUTestUnevenDEFields*.nc" + character(len=*), parameter :: fileNameUnevenDEArrays = "ESMF_IO_MultitileUTestUnevenDEArrays*.nc" !------------------------------------------------------------------------ call ESMF_TestStart(ESMF_SRCLINE, rc=rc) ! calls ESMF_Initialize() internally @@ -356,7 +363,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Write a multi-tile Field with uneven DEs per PET" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldWrite(field1UnevenDEs, fileName=fileNameUnevenDEs, overwrite=.true., rc=rc) + call ESMF_FieldWrite(field1UnevenDEs, fileName=fileNameUnevenDEFields, overwrite=.true., rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -369,7 +376,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Read a multi-tile Field with uneven DEs per PET" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldRead(field1UnevenDEsRead, fileName=fileNameUnevenDEs, rc=rc) + call ESMF_FieldRead(field1UnevenDEsRead, fileName=fileNameUnevenDEFields, rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -405,7 +412,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Write a multi-tile Field with uneven DEs per PET and ungridded dims" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldWrite(field4dUnevenDEs, fileName=fileNameUnevenDEs, overwrite=.true., rc=rc) + call ESMF_FieldWrite(field4dUnevenDEs, fileName=fileNameUnevenDEFields, overwrite=.true., rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -418,7 +425,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Read a multi-tile Field with uneven DEs per PET and ungridded dims" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldRead(field4dUnevenDEsRead, fileName=fileNameUnevenDEs, rc=rc) + call ESMF_FieldRead(field4dUnevenDEsRead, fileName=fileNameUnevenDEFields, rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -450,6 +457,55 @@ program ESMF_IO_MultitileUTest #endif !------------------------------------------------------------------------ + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Write a multi-tile Array with uneven DEs per PET and replicated dims" + write(failMsg, *) "Did not return ESMF_SUCCESS" + call ESMF_ArrayWrite(arrayWithReplicatedDims, fileName=fileNameUnevenDEArrays, overwrite=.true., rc=rc) +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" + call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Read a multi-tile Array with uneven DEs per PET and replicated dims" + write(failMsg, *) "Did not return ESMF_SUCCESS" + call ESMF_ArrayRead(arrayWithReplicatedDimsRead, fileName=fileNameUnevenDEArrays, rc=rc) +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" + call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + + !------------------------------------------------------------------------ + !EX_UTest_Multi_Proc_Only + write(name,*) "Confirm that Array-read array matches original with uneven DEs per PET and replicated dims" + write(failMsg, *) "Read-in array differs from original" + allEqual = .true. + do lde = 0, arrayWithReplicatedDimsLdeCount - 1 + ! For simplicity, bail out if the following ArrayGets fail rather than calling them their own unit test + call ESMF_ArrayGet(arrayWithReplicatedDims, localDe=lde, farrayPtr=arrayWithReplicatedDimsData, rc=rc) + if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_ArrayGet(arrayWithReplicatedDimsRead, localDe=lde, farrayPtr=arrayWithReplicatedDimsReadData, rc=rc) + if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (.not. all(arrayWithReplicatedDimsReadData == arrayWithReplicatedDimsData)) then + allEqual = .false. + end if + end do +#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) + call ESMF_Test(allEqual, name, failMsg, result, ESMF_SRCLINE) +#else + write(failMsg, *) "Comparison did not fail as expected" + call ESMF_Test(.not. allEqual, name, failMsg, result, ESMF_SRCLINE) +#endif + !------------------------------------------------------------------------ + #endif ! ESMF_TESTEXHAUSTIVE !------------------------------------------------------------------------ @@ -604,7 +660,8 @@ subroutine createFields(rc) ! # DEs: 1 2 3 4 0 3 0 3 decompPTileUnevenDEs(1,:) = [2,1,3,1,1,3] decompPTileUnevenDEs(2,:) = [1,1,2,1,3,1] - delayout = ESMF_DELayoutCreate(petMap=[3,2,5,5,1,3,2,1,7,3,0,7,2,7,3,5]) + delayout = ESMF_DELayoutCreate(petMap=[3,2,5,5,1,3,2,1,7,3,0,7,2,7,3,5], rc=rc) + if (rc /= ESMF_SUCCESS) return grid6tileUnevenDEs = ESMF_GridCreateCubedSphere( & tilesize = 6, & regDecompPTile = decompPTileUnevenDEs, & @@ -704,6 +761,14 @@ subroutine createArrays(rc) integer :: minIndexPTile(2,3) integer :: maxIndexPTile(2,3) integer :: decompPTile(2,3) + type(ESMF_DistGrid) :: distgrid3tile + + ! Information for a 4-d, 3-tile DistGrid + integer :: minIndexPTile4d(4,3) + integer :: maxIndexPTile4d(4,3) + integer :: decompPTile4d(4,3) + type(ESMF_DELayout) :: delayout4d + type(ESMF_DistGrid) :: distgrid3tile4d type(ESMF_ArraySpec) :: arraySpec @@ -777,6 +842,64 @@ subroutine createArrays(rc) arrayBundleRead = ESMF_ArrayBundleCreate(arrayList=[array1Read, array2Read], name="abRead", rc=rc) if (rc /= ESMF_SUCCESS) return + !------------------------------------------------------------------------ + ! Set up a 3-tile, 4-d distgrid with an uneven distribution of DEs to PETs (but still 8 PEs) + !------------------------------------------------------------------------ + + minIndexPTile4d(:,1) = [11,1 ,21,51] + maxIndexPTile4d(:,1) = [20,10,23,55] + minIndexPTile4d(:,2) = [11,11,31,61] + maxIndexPTile4d(:,2) = [20,20,34,64] + minIndexPTile4d(:,3) = [1 ,11,41,71] + maxIndexPTile4d(:,3) = [10,20,45,73] + + ! Tiles 1 and 3 each have 6 DEs (along different dimensions); tile 2 has 16 DEs + ! (2x2x2x2) + decompPTile4d(:,1) = [1,3,1,2] + decompPTile4d(:,2) = [2,2,2,2] + decompPTile4d(:,3) = [3,1,2,1] + + ! The DEs are scattered in a disorganized fashion across PETs. We have the following + ! number of DEs on each PET: + ! PET #: 0 1 2 3 4 5 6 7 + ! # DEs: 4 1 3 8 0 5 0 7 + delayout4d = ESMF_DELayoutCreate(petMap=[3,7,2,7,5,0,3,3,5,5,1,3,0,7,2,3,5,7,3,0,7,3,2,7,0,3,5,7], rc=rc) + if (rc /= ESMF_SUCCESS) return + + distgrid3tile4d = ESMF_DistGridCreate( & + minIndexPTile = minIndexPTile4d, & + maxIndexPTile = maxIndexPTile4d, & + regDecompPTile = decompPTile4d, & + delayout = delayout4d, & + rc=rc) + if (rc /= ESMF_SUCCESS) return + + !------------------------------------------------------------------------ + ! Create arrays on the 3-tile, 4-d distgrid + !------------------------------------------------------------------------ + + ! Create a 2-d array... so there are 2 replicated dimensions + arrayWithReplicatedDims = ESMF_ArrayCreate(distgrid3tile4d, arraySpec, name="arrayWithReplicatedDims", & + ! Dimensions 2 and 4 of the DistGrid will be replicated dimensions; note also + ! that we rearrange dimensions 1 and 3: + distgridToArrayMap = [2,0,1,0], & + rc=rc) + if (rc /= ESMF_SUCCESS) return + call ESMF_ArrayGet(arrayWithReplicatedDims, localDECount=arrayWithReplicatedDimsLdeCount, rc=rc) + if (rc /= ESMF_SUCCESS) return + do lde = 0, arrayWithReplicatedDimsLdeCount-1 + call ESMF_ArrayGet(array1, farrayPtr=arrayWithReplicatedDimsData, rc=rc) + if (rc /= ESMF_SUCCESS) return + call fillArray(arrayWithReplicatedDimsData, (10+lde)) + end do + + arrayWithReplicatedDimsRead = ESMF_ArrayCreate(distgrid3tile4d, arraySpec, name="arrayWithReplicatedDims", & + ! Dimensions 2 and 4 of the DistGrid will be replicated dimensions; note also + ! that we rearrange dimensions 1 and 3: + distgridToArrayMap = [2,0,1,0], & + rc=rc) + if (rc /= ESMF_SUCCESS) return + end subroutine createArrays subroutine fillArray(array, multiplier) From c5f596127eea76d46ac9138e7ce22d5d5e30bf31 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 10 Oct 2023 20:11:59 -0600 Subject: [PATCH 07/10] Revert "Multi-tile I/O with replicated dims: add unit tests & try making it work" This reverts commit 28f951b333b460b60526523ecba088c8548386a3. There seems to be trouble with the I/O of Arrays with replicated dimensions in general - see https://github.com/esmf-org/esmf/issues/184. So I'm backing this out rather than trying to build on top of something that isn't completely working. --- src/Infrastructure/IO/src/ESMCI_IO.C | 19 +-- .../IO/tests/ESMF_IO_MultitileUTest.F90 | 137 +----------------- 2 files changed, 17 insertions(+), 139 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index 35afc7a9f0..ceed5caf28 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1262,22 +1262,23 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet std::vector maxIndexPDimPTileVec; std::vector distgridToArrayMapVec; if (replicatedDims>0){ + // FIXME(wjs, 2023-05-26) Still need to fix this loop to work with the multi-tile + // case; then remove the following error check + if (tileCount > 1) { + ESMC_LogDefault.MsgFoundError(ESMF_RC_NOT_IMPL, + "Multi-tile with > 1 DE per PET and replicated dims not yet implemented", ESMC_CONTEXT, rc); + return; + } + // eliminate replicated dimensions from the destination for (int i=0; i keep + minIndexPDimPTileVec.push_back(minIndexPDimPTile[i]); + maxIndexPDimPTileVec.push_back(maxIndexPDimPTile[i]); distgridToArrayMapVec.push_back(distgridToArrayMap[i]); } } - for (int tile=0; tile keep - minIndexPDimPTileVec.push_back(minIndexPDimPTile[tile*ndims + i]); - maxIndexPDimPTileVec.push_back(maxIndexPDimPTile[tile*ndims + i]); - } - } - } if (minIndexPDimPTileVec.size()<1){ ESMC_LogDefault.MsgFoundError(ESMF_RC_INTNRL_BAD, "Not enough distributed dimensions", ESMC_CONTEXT, rc); diff --git a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 index cb112da2e9..a58ef41760 100644 --- a/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 +++ b/src/Infrastructure/IO/tests/ESMF_IO_MultitileUTest.F90 @@ -49,8 +49,8 @@ program ESMF_IO_MultitileUTest type(ESMF_Grid) :: grid6tile type(ESMF_Grid) :: grid6tileUnevenDEs integer :: grid6tileUnevenDEsLdeCount - integer :: arrayWithReplicatedDimsLdeCount integer :: lde + type(ESMF_DistGrid) :: distgrid3tile ! Fields used for writing: ! @@ -94,9 +94,6 @@ program ESMF_IO_MultitileUTest ! This array is not in the array bundle: type(ESMF_Array) :: array3 real(ESMF_KIND_R8), pointer :: array3Data(:,:) - ! This array is for tests with something other than 1 DE per PET: - type(ESMF_Array) :: arrayWithReplicatedDims - real(ESMF_KIND_R8), pointer :: arrayWithReplicatedDimsData(:,:) ! Arrays used for reading: ! @@ -107,17 +104,13 @@ program ESMF_IO_MultitileUTest ! This array is not in the array bundle: type(ESMF_Array) :: array3Read real(ESMF_KIND_R8), pointer :: array3ReadData(:,:) - ! This array is for tests with something other than 1 DE per PET: - type(ESMF_Array) :: arrayWithReplicatedDimsRead - real(ESMF_KIND_R8), pointer :: arrayWithReplicatedDimsReadData(:,:) logical :: allEqual character(len=*), parameter :: fileNameFields = "ESMF_IO_MultitileUTestFields*.nc" character(len=*), parameter :: fileNameArrays = "ESMF_IO_MultitileUTestArrays*.nc" character(len=*), parameter :: fileNameFail = "ESMF_IO_MultitileUTestFail*.nc" - character(len=*), parameter :: fileNameUnevenDEFields = "ESMF_IO_MultitileUTestUnevenDEFields*.nc" - character(len=*), parameter :: fileNameUnevenDEArrays = "ESMF_IO_MultitileUTestUnevenDEArrays*.nc" + character(len=*), parameter :: fileNameUnevenDEs = "ESMF_IO_MultitileUTestUnevenDEs*.nc" !------------------------------------------------------------------------ call ESMF_TestStart(ESMF_SRCLINE, rc=rc) ! calls ESMF_Initialize() internally @@ -363,7 +356,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Write a multi-tile Field with uneven DEs per PET" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldWrite(field1UnevenDEs, fileName=fileNameUnevenDEFields, overwrite=.true., rc=rc) + call ESMF_FieldWrite(field1UnevenDEs, fileName=fileNameUnevenDEs, overwrite=.true., rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -376,7 +369,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Read a multi-tile Field with uneven DEs per PET" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldRead(field1UnevenDEsRead, fileName=fileNameUnevenDEFields, rc=rc) + call ESMF_FieldRead(field1UnevenDEsRead, fileName=fileNameUnevenDEs, rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -412,7 +405,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Write a multi-tile Field with uneven DEs per PET and ungridded dims" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldWrite(field4dUnevenDEs, fileName=fileNameUnevenDEFields, overwrite=.true., rc=rc) + call ESMF_FieldWrite(field4dUnevenDEs, fileName=fileNameUnevenDEs, overwrite=.true., rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -425,7 +418,7 @@ program ESMF_IO_MultitileUTest !EX_UTest_Multi_Proc_Only write(name,*) "Read a multi-tile Field with uneven DEs per PET and ungridded dims" write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_FieldRead(field4dUnevenDEsRead, fileName=fileNameUnevenDEFields, rc=rc) + call ESMF_FieldRead(field4dUnevenDEsRead, fileName=fileNameUnevenDEs, rc=rc) #if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) #else @@ -457,55 +450,6 @@ program ESMF_IO_MultitileUTest #endif !------------------------------------------------------------------------ - !------------------------------------------------------------------------ - !EX_UTest_Multi_Proc_Only - write(name,*) "Write a multi-tile Array with uneven DEs per PET and replicated dims" - write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_ArrayWrite(arrayWithReplicatedDims, fileName=fileNameUnevenDEArrays, overwrite=.true., rc=rc) -#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) - call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) -#else - write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" - call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) -#endif - !------------------------------------------------------------------------ - - !------------------------------------------------------------------------ - !EX_UTest_Multi_Proc_Only - write(name,*) "Read a multi-tile Array with uneven DEs per PET and replicated dims" - write(failMsg, *) "Did not return ESMF_SUCCESS" - call ESMF_ArrayRead(arrayWithReplicatedDimsRead, fileName=fileNameUnevenDEArrays, rc=rc) -#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) - call ESMF_Test((rc == ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE) -#else - write(failMsg, *) "Did not return ESMF_RC_LIB_NOT_PRESENT" - call ESMF_Test((rc == ESMF_RC_LIB_NOT_PRESENT), name, failMsg, result, ESMF_SRCLINE) -#endif - !------------------------------------------------------------------------ - - !------------------------------------------------------------------------ - !EX_UTest_Multi_Proc_Only - write(name,*) "Confirm that Array-read array matches original with uneven DEs per PET and replicated dims" - write(failMsg, *) "Read-in array differs from original" - allEqual = .true. - do lde = 0, arrayWithReplicatedDimsLdeCount - 1 - ! For simplicity, bail out if the following ArrayGets fail rather than calling them their own unit test - call ESMF_ArrayGet(arrayWithReplicatedDims, localDe=lde, farrayPtr=arrayWithReplicatedDimsData, rc=rc) - if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) - call ESMF_ArrayGet(arrayWithReplicatedDimsRead, localDe=lde, farrayPtr=arrayWithReplicatedDimsReadData, rc=rc) - if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT) - if (.not. all(arrayWithReplicatedDimsReadData == arrayWithReplicatedDimsData)) then - allEqual = .false. - end if - end do -#if (defined ESMF_PIO && (defined ESMF_NETCDF || defined ESMF_PNETCDF)) - call ESMF_Test(allEqual, name, failMsg, result, ESMF_SRCLINE) -#else - write(failMsg, *) "Comparison did not fail as expected" - call ESMF_Test(.not. allEqual, name, failMsg, result, ESMF_SRCLINE) -#endif - !------------------------------------------------------------------------ - #endif ! ESMF_TESTEXHAUSTIVE !------------------------------------------------------------------------ @@ -660,8 +604,7 @@ subroutine createFields(rc) ! # DEs: 1 2 3 4 0 3 0 3 decompPTileUnevenDEs(1,:) = [2,1,3,1,1,3] decompPTileUnevenDEs(2,:) = [1,1,2,1,3,1] - delayout = ESMF_DELayoutCreate(petMap=[3,2,5,5,1,3,2,1,7,3,0,7,2,7,3,5], rc=rc) - if (rc /= ESMF_SUCCESS) return + delayout = ESMF_DELayoutCreate(petMap=[3,2,5,5,1,3,2,1,7,3,0,7,2,7,3,5]) grid6tileUnevenDEs = ESMF_GridCreateCubedSphere( & tilesize = 6, & regDecompPTile = decompPTileUnevenDEs, & @@ -761,14 +704,6 @@ subroutine createArrays(rc) integer :: minIndexPTile(2,3) integer :: maxIndexPTile(2,3) integer :: decompPTile(2,3) - type(ESMF_DistGrid) :: distgrid3tile - - ! Information for a 4-d, 3-tile DistGrid - integer :: minIndexPTile4d(4,3) - integer :: maxIndexPTile4d(4,3) - integer :: decompPTile4d(4,3) - type(ESMF_DELayout) :: delayout4d - type(ESMF_DistGrid) :: distgrid3tile4d type(ESMF_ArraySpec) :: arraySpec @@ -842,64 +777,6 @@ subroutine createArrays(rc) arrayBundleRead = ESMF_ArrayBundleCreate(arrayList=[array1Read, array2Read], name="abRead", rc=rc) if (rc /= ESMF_SUCCESS) return - !------------------------------------------------------------------------ - ! Set up a 3-tile, 4-d distgrid with an uneven distribution of DEs to PETs (but still 8 PEs) - !------------------------------------------------------------------------ - - minIndexPTile4d(:,1) = [11,1 ,21,51] - maxIndexPTile4d(:,1) = [20,10,23,55] - minIndexPTile4d(:,2) = [11,11,31,61] - maxIndexPTile4d(:,2) = [20,20,34,64] - minIndexPTile4d(:,3) = [1 ,11,41,71] - maxIndexPTile4d(:,3) = [10,20,45,73] - - ! Tiles 1 and 3 each have 6 DEs (along different dimensions); tile 2 has 16 DEs - ! (2x2x2x2) - decompPTile4d(:,1) = [1,3,1,2] - decompPTile4d(:,2) = [2,2,2,2] - decompPTile4d(:,3) = [3,1,2,1] - - ! The DEs are scattered in a disorganized fashion across PETs. We have the following - ! number of DEs on each PET: - ! PET #: 0 1 2 3 4 5 6 7 - ! # DEs: 4 1 3 8 0 5 0 7 - delayout4d = ESMF_DELayoutCreate(petMap=[3,7,2,7,5,0,3,3,5,5,1,3,0,7,2,3,5,7,3,0,7,3,2,7,0,3,5,7], rc=rc) - if (rc /= ESMF_SUCCESS) return - - distgrid3tile4d = ESMF_DistGridCreate( & - minIndexPTile = minIndexPTile4d, & - maxIndexPTile = maxIndexPTile4d, & - regDecompPTile = decompPTile4d, & - delayout = delayout4d, & - rc=rc) - if (rc /= ESMF_SUCCESS) return - - !------------------------------------------------------------------------ - ! Create arrays on the 3-tile, 4-d distgrid - !------------------------------------------------------------------------ - - ! Create a 2-d array... so there are 2 replicated dimensions - arrayWithReplicatedDims = ESMF_ArrayCreate(distgrid3tile4d, arraySpec, name="arrayWithReplicatedDims", & - ! Dimensions 2 and 4 of the DistGrid will be replicated dimensions; note also - ! that we rearrange dimensions 1 and 3: - distgridToArrayMap = [2,0,1,0], & - rc=rc) - if (rc /= ESMF_SUCCESS) return - call ESMF_ArrayGet(arrayWithReplicatedDims, localDECount=arrayWithReplicatedDimsLdeCount, rc=rc) - if (rc /= ESMF_SUCCESS) return - do lde = 0, arrayWithReplicatedDimsLdeCount-1 - call ESMF_ArrayGet(array1, farrayPtr=arrayWithReplicatedDimsData, rc=rc) - if (rc /= ESMF_SUCCESS) return - call fillArray(arrayWithReplicatedDimsData, (10+lde)) - end do - - arrayWithReplicatedDimsRead = ESMF_ArrayCreate(distgrid3tile4d, arraySpec, name="arrayWithReplicatedDims", & - ! Dimensions 2 and 4 of the DistGrid will be replicated dimensions; note also - ! that we rearrange dimensions 1 and 3: - distgridToArrayMap = [2,0,1,0], & - rc=rc) - if (rc /= ESMF_SUCCESS) return - end subroutine createArrays subroutine fillArray(array, multiplier) From de864309f2418a2d9aa5b015e73fd8b4b31d387c Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 10 Oct 2023 20:17:20 -0600 Subject: [PATCH 08/10] Add a comment about issues with I/O with replicated dimensions --- src/Infrastructure/IO/src/ESMCI_IO.C | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index ceed5caf28..b9011f9778 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1262,8 +1262,15 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet std::vector maxIndexPDimPTileVec; std::vector distgridToArrayMapVec; if (replicatedDims>0){ - // FIXME(wjs, 2023-05-26) Still need to fix this loop to work with the multi-tile - // case; then remove the following error check + // TODO(wjs, 2023-05-26) Still need to fix this loop to work with the multi-tile + // case; then remove the following error check. + // + // See commit 28f951b333 (reverted in c5f596127e) for an attempt to get this working + // in the multi-tile case. However, this attempt was aborted due to finding what + // appear to be issues with I/O with Arrays with replicated dimensions even in the + // single-tile case - see https://github.com/esmf-org/esmf/issues/184. We have decided + // to leave this functionality in place in case anyone is relying on it, but we do not + // expect I/O with Arrays with replicated dimensions to work reliably or consistently. if (tileCount > 1) { ESMC_LogDefault.MsgFoundError(ESMF_RC_NOT_IMPL, "Multi-tile with > 1 DE per PET and replicated dims not yet implemented", ESMC_CONTEXT, rc); From c67016dc053323d3d8943302cd6f281056740404 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 12 Oct 2023 14:04:34 -0600 Subject: [PATCH 09/10] Remove a seemingly-unnecessary error check This error check was added in commit 7379b259f2d2d621e335204026b2e2644318772c (by Gerhard, Jan 12, 2018): "Add checks to prevent redist_arraycreate1de() from trying to redist an array where the index space is too small to span all PETs." Note that there is no error check like this in the case where we're not doing a redist, which makes me think that this may not be a fundamental PIO limitation. Furthermore, in a test program, it worked to remove this error check when writing and reading a 2x4 Field with 10 PETs (and going through the redist function). So it seems like this check isn't necessary. --- src/Infrastructure/IO/src/ESMCI_IO.C | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index b9011f9778..667c6dd888 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1297,15 +1297,6 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet distgridToArrayMap = &(distgridToArrayMapVec[0]); } - if (tileCount == 1) { - // FIXME(wjs, 2023-05-26) Generalize this error-check to work with the multi-tile case - if ((maxIndexPDimPTile[0]-minIndexPDimPTile[0]+1) minIndexInterface((int*)minIndexPDimPTile, ndims-replicatedDims); From 3d6678e6be9f4c7e46b7c93f84f7344040bcb885 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 13 Oct 2023 07:52:34 -0600 Subject: [PATCH 10/10] Multi-tile I/O: Ensure there are at least as many PETs as tiles Without this new error check, we get the following if trying to run with fewer PETs than tiles: 20231013 064715.921 WARNING PET1 ESMCI_PIO_Handler.C:1994 ESMCI::PIO_IODescHandler::constr I/O does not support multiple DEs per PET 20231013 064715.921 ERROR PET1 ESMCI_PIO_Handler.C:1730 ESMCI::PIO_Handler::getIODesc() Not found - Internal subroutine call returned Error This new error check gives a more explicit error message for this case. --- src/Infrastructure/IO/src/ESMCI_IO.C | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Infrastructure/IO/src/ESMCI_IO.C b/src/Infrastructure/IO/src/ESMCI_IO.C index d6f882430e..9ac5a9cecc 100644 --- a/src/Infrastructure/IO/src/ESMCI_IO.C +++ b/src/Infrastructure/IO/src/ESMCI_IO.C @@ -1254,6 +1254,15 @@ void IO::redist_arraycreate1de(Array *src_array_p, Array **dest_array_p, int pet int rank = src_array_p->getRank(); int tileCount = dg_orig->getTileCount(); + if (tileCount > petCount) { + // If tileCount > petCount, the decomposition created by the default DistGrid sets + // deCount = tileCount, which leads to having more DEs than PETs, which is exactly + // what we're trying to avoid in this function. + ESMC_LogDefault.MsgFoundError(ESMF_RC_INTNRL_BAD, + "Multi-tile I/O requires at least as many PETs as tiles", ESMC_CONTEXT, rc); + return; // bail out + } + int replicatedDims = src_array_p->getReplicatedDimCount(); std::vector minIndexPDimPTileVec;