Add replicatedDimCount to ESMF_ArrayGet, and improve some unit tests of replicated dimensions

src/Infrastructure/Array/examples/ESMF_ArrayEx.F90

@@ -1308,6 +1308,7 @@ program ESMF_ArrayEx
 
 !BOE
 ! \subsubsection{Arrays with replicated dimensions}
+! \label{sec:array:usage:replicated_dims}
 !
 ! Thus far most examples demonstrated cases where the DistGrid {\tt dimCount}
 ! was equal to the Array {\tt rank}. The previous section introduced the

src/Infrastructure/Array/include/ESMCI_Array.h

@@ -191,6 +191,8 @@ namespace ESMCI {
     int *totalUBound;                 // [redDimCount*ssiLocalDeCount]
     int tensorCount;                  // number of tensor dimensions
     int tensorElementCount;           // number of tensor elements per element
+    int replicatedDimCount;           // number of replicated dimensions
+                                      // (i.e., dimensions where distgridToArrayMap[i] == 0)
     int *undistLBound;                // [tensorCount]
     int *undistUBound;                // [tensorCount]
     int *distgridToArrayMap;          // [dimCount] - entries are basis 1
@@ -262,6 +264,7 @@ namespace ESMCI {
       totalLBound = NULL;
       totalUBound = NULL;
       tensorCount = 0;
+      replicatedDimCount = 0;
       undistLBound = NULL;
       undistUBound = NULL;
       distgridToArrayMap = NULL;
@@ -300,6 +303,7 @@ namespace ESMCI {
       totalLBound = NULL;
       totalUBound = NULL;
       tensorCount = 0;
+      replicatedDimCount = 0;
       undistLBound = NULL;
       undistUBound = NULL;
       distgridToArrayMap = NULL;
@@ -327,8 +331,8 @@ namespace ESMCI {
       int *localDeToDeMap, DistGrid *distgrid, bool distgridCreator,
       int *exclusiveLBound, int *exclusiveUBound, int *computationalLBound,
       int *computationalUBound, int *totalLBound, int *totalUBound,
-      int tensorCount, int tensorElementCount, int *undistLBoundArray,
-      int *undistUBoundArray, int *distgridToArrayMapArray,
+      int tensorCount, int tensorElementCount, int replicatedDimCount,
+      int *undistLBoundArray, int *undistUBoundArray, int *distgridToArrayMapArray,
       int *arrayToDistGridMapArray, int *distgridToPackedArrayMapArray,
       ESMC_IndexFlag indexflagArg, int *rc,
       VM *vm=NULL); // allow specific VM instead default
@@ -385,6 +389,7 @@ namespace ESMCI {
     const int *getTotalLBound()             const {return totalLBound;}
     const int *getTotalUBound()             const {return totalUBound;}
     int getTensorCount()                    const {return tensorCount;}
+    int getReplicatedDimCount()             const {return replicatedDimCount;}
     const int *getUndistLBound()            const {return undistLBound;}
     const int *getUndistUBound()            const {return undistUBound;}
     const int *getExclusiveElementCountPDe()const

src/Infrastructure/Array/interface/ESMCI_Array_F.C

@@ -193,7 +193,8 @@ extern "C" {
   }
 
   void FTN_X(c_esmc_arrayget)(ESMCI::Array **ptr, ESMC_TypeKind_Flag *typekind,
-    int *rank, int *ssiLocalDeCount, ESMCI::LocalArray **opt_localArrayList,
+    int *rank, int *replicatedDimCount,
+    int *ssiLocalDeCount, ESMCI::LocalArray **opt_localArrayList,
     int *len_localArrayList, ESMCI::InterArray<int> *localDeToDeMap,
     ESMCI::DistGrid **distgrid, ESMCI::DELayout **delayout,
     ESMC_IndexFlag *indexflag, 
@@ -227,6 +228,8 @@ extern "C" {
       *typekind = (*ptr)->getTypekind();
     if (ESMC_NOT_PRESENT_FILTER(rank) != ESMC_NULL_POINTER)
       *rank = (*ptr)->getRank();
+    if (ESMC_NOT_PRESENT_FILTER(replicatedDimCount) != ESMC_NULL_POINTER)
+      *replicatedDimCount = (*ptr)->getReplicatedDimCount();
     if (ESMC_NOT_PRESENT_FILTER(ssiLocalDeCount) != ESMC_NULL_POINTER)
       *ssiLocalDeCount = (*ptr)->getSsiLocalDeCount();
     if (ESMC_NOT_PRESENT_FILTER(indexflag) != ESMC_NULL_POINTER)

src/Infrastructure/Array/interface/ESMF_ArrayGet.cppF90

@@ -128,12 +128,13 @@ contains
 
 ! !INTERFACE:
   ! Private name; call using ESMF_ArrayGet()
-  subroutine ESMF_ArrayGetDefault(array, keywordEnforcer, arrayspec, typekind, &
+subroutine ESMF_ArrayGetDefault(array, keywordEnforcer, arrayspec, typekind, &
     rank, localarrayList, indexflag, distgridToArrayMap, &
     distgridToPackedArrayMap, arrayToDistGridMap, undistLBound, &
     undistUBound, exclusiveLBound, exclusiveUBound, computationalLBound, &
     computationalUBound, totalLBound, totalUBound, computationalLWidth, &
-    computationalUWidth, totalLWidth, totalUWidth, distgrid, dimCount, &
+    computationalUWidth, totalLWidth, totalUWidth, distgrid, &
+    dimCount, replicatedDimCount, &
     tileCount, minIndexPTile, maxIndexPTile, deToTileMap, indexCountPDe, &
     delayout, deCount, localDeCount, ssiLocalDeCount, localDeToDeMap, &
     localDeList, &    ! DEPRECATED ARGUMENT
@@ -164,6 +165,7 @@ type(ESMF_KeywordEnforcer), optional:: keywordEnforcer ! must use keywords below
     integer,               target, intent(out), optional :: totalUWidth(:,:)
     type(ESMF_DistGrid),           intent(out), optional :: distgrid
     integer,                       intent(out), optional :: dimCount
+    integer,                       intent(out), optional :: replicatedDimCount
     integer,                       intent(out), optional :: tileCount
     integer,                       intent(out), optional :: minIndexPTile(:,:)
     integer,                       intent(out), optional :: maxIndexPTile(:,:)
@@ -314,6 +316,10 @@ type(ESMF_KeywordEnforcer), optional:: keywordEnforcer ! must use keywords below
 !     Upon return this holds the associated {\tt ESMF\_DistGrid} object.
 !   \item[{[dimCount]}]
 !     Number of dimensions (rank) of {\tt distgrid}.
+!   \item[{[replicatedDimCount]}]
+!     Number of replicated dimensions in the Array. (See
+!     Section~\ref{sec:array:usage:replicated_dims} for an explanation of replicated
+!     dimensions.)
 !   \item[{[tileCount]}]
 !     Number of tiles in {\tt distgrid}.
 !   \item[{[minIndexPTile]}]
@@ -462,7 +468,8 @@ type(ESMF_KeywordEnforcer), optional:: keywordEnforcer ! must use keywords below
       ESMF_CONTEXT, rcToReturn=rc)) return
 
     ! Call into the C++ interface, which will sort out optional arguments
-    call c_ESMC_ArrayGet(array, opt_typekind, opt_rank, opt_ssiLocalDeCount, &
+    call c_ESMC_ArrayGet(array, opt_typekind, opt_rank, &
+      replicatedDimCount, opt_ssiLocalDeCount, &
       opt_localarrayPtrList, len_localarrayPtrList, localDeToDeMapArg, &
       opt_distgrid, opt_delayout, indexflag, &
       distgridToArrayMapArg, distgridToPackedArrayMapArg, &

src/Infrastructure/Array/src/ESMCI_Array.C

@@ -160,6 +160,7 @@ Array::Array(
   int *totalUBoundArg,                    // (in)
   int tensorCountArg,                     // (in)
   int tensorElementCountArg,              // (in)
+  int replicatedDimCountArg,              // (in)
   int *undistLBoundArray,                 // (in)
   int *undistUBoundArray,                 // (in)
   int *distgridToArrayMapArray,           // (in)
@@ -259,6 +260,8 @@ Array::Array(
     memcpy(undistLBound, undistLBoundArray, tensorCountArg * sizeof(int));
     memcpy(undistUBound, undistUBoundArray, tensorCountArg * sizeof(int));
   }
+  // replicated dimension count
+  replicatedDimCount = replicatedDimCountArg;
   // distgridToArrayMap, arrayToDistGridMap and distgridToPackedArrayMap
   int dimCount = distgrid->getDimCount();
   distgridToArrayMap = new int[dimCount];
@@ -511,6 +514,19 @@ int Array::constructContiguousFlag(int redDimCount){
 }
 //-----------------------------------------------------------------------------
 
+// Helper for create
+//-----------------------------------------------------------------------------
+#undef  ESMC_METHOD
+#define ESMC_METHOD "ESMCI::calcReplicatedDimCount()"
+  int calcReplicatedDimCount(int dimCount, int *distgridToArrayMap) {
+    int replicatorCount = 0;
+    for (int i=0; i<dimCount; i++) {
+      if (distgridToArrayMap[i] == 0) ++replicatorCount;
+    }
+
+    return replicatorCount;
+  }
+//-----------------------------------------------------------------------------
 
 //-----------------------------------------------------------------------------
 //
@@ -650,9 +666,7 @@ Array *Array::create(
     }
   }
   // determine replicatorCount
-  int replicatorCount = 0;  // initialize
-  for (int i=0; i<dimCount; i++)
-    if (distgridToArrayMapArray[i] == 0) ++replicatorCount;
+  int replicatorCount = calcReplicatedDimCount(dimCount, distgridToArrayMapArray);
   // determine reduced dimCount -> redDimCount
   int redDimCount = dimCount - replicatorCount;
   // determine tensorCount
@@ -1240,7 +1254,7 @@ Array *Array::create(
       ssiLocalDeCountArg, NULL, distgrid, false,
       exclusiveLBound, exclusiveUBound, computationalLBound,
       computationalUBound, totalLBound, totalUBound, tensorCount,
-      tensorElementCount, undistLBoundArray, undistUBoundArray,
+      tensorElementCount, replicatorCount, undistLBoundArray, undistUBoundArray,
       distgridToArrayMapArray, arrayToDistGridMapArray,
       distgridToPackedArrayMap, indexflag, &localrc);
     if (ESMC_LogDefault.MsgFoundError(localrc, ESMCI_ERR_PASSTHRU, ESMC_CONTEXT,
@@ -1400,9 +1414,7 @@ Array *Array::create(
     }
   }
   // determine replicatorCount
-  int replicatorCount = 0;  // initialize
-  for (int i=0; i<dimCount; i++)
-    if (distgridToArrayMapArray[i] == 0) ++replicatorCount;
+  int replicatorCount = calcReplicatedDimCount(dimCount, distgridToArrayMapArray);
   // determine reduced dimCount -> redDimCount
   int redDimCount = dimCount - replicatorCount;
   // determine tensorCount
@@ -2050,7 +2062,7 @@ Array *Array::create(
       ssiLocalDeCountArg, localDeToDeMapArg, distgrid, false,
       exclusiveLBound, exclusiveUBound, computationalLBound,
       computationalUBound, totalLBound, totalUBound, tensorCount,
-      tensorElementCount, undistLBoundArray, undistUBoundArray,
+      tensorElementCount, replicatorCount, undistLBoundArray, undistUBoundArray,
       distgridToArrayMapArray, arrayToDistGridMapArray,
       distgridToPackedArrayMap, indexflag, &localrc, vm);
     if (ESMC_LogDefault.MsgFoundError(localrc, ESMCI_ERR_PASSTHRU,
@@ -2155,6 +2167,7 @@ Array *Array::create(
     int tensorCount =
       arrayOut->tensorCount = arrayIn->tensorCount
         - rmLeadingTensors - rmTrailingTensors;
+    arrayOut->replicatedDimCount = arrayIn->replicatedDimCount;
     arrayOut->vasLocalDeCount = arrayIn->vasLocalDeCount;
     if (copyflag == DATACOPY_REFERENCE){
       // sharing reference means also sharing memhandle
@@ -2470,6 +2483,7 @@ Array *Array::create(
       arrayOut->tensorElementCount = 1;
     else
       arrayOut->tensorElementCount = arrayIn->tensorElementCount;
+    arrayOut->replicatedDimCount = arrayIn->replicatedDimCount;
     arrayOut->distgrid = arrayIn->distgrid; // copy reference
     arrayOut->distgridCreator = false;      // not a locally created object
     arrayOut->delayout = arrayIn->delayout; // copy reference
@@ -4447,10 +4461,11 @@ int Array::serialize(
     for (int i=0; i<distgrid->getDimCount(); i++)
       *ip++ = distgridToPackedArrayMap[i];
     *ip++ = tensorElementCount;
+    *ip++ = replicatedDimCount;
     for (int i=0; i<delayout->getDeCount(); i++)
       *ip++ = exclusiveElementCountPDe[i];
   } else
-    ip += 2 + 2*tensorCount + 2*distgrid->getDimCount () +
+    ip += 3 + 2*tensorCount + 2*distgrid->getDimCount () +
       rank + delayout->getDeCount ();
 
   // fix offset
@@ -4542,6 +4557,7 @@ int Array::deserialize(
   for (int i=0; i<distgrid->getDimCount(); i++)
     distgridToPackedArrayMap[i] = *ip++;
   tensorElementCount = *ip++;
+  replicatedDimCount = *ip++;
   exclusiveElementCountPDe = new int[delayout->getDeCount()];
   for (int i=0; i<delayout->getDeCount(); i++)
     exclusiveElementCountPDe[i] = *ip++;

src/Infrastructure/Array/tests/ESMF_ArrayCreateGetUTest.F90

@@ -54,6 +54,7 @@ program ESMF_ArrayCreateGetUTest
   !LOCAL VARIABLES:
   type(ESMF_VM):: vm
   integer:: i,j,k,l, next, rank
+  integer:: replicatedDimCount
   integer:: petCount, localPet, deCount, de, localDeCount, lde, ssiLocalDeCount
   integer, allocatable  :: regDecomp(:)
   type(ESMF_ArraySpec)  :: arrayspec, arrayspec2
@@ -316,6 +317,19 @@ program ESMF_ArrayCreateGetUTest
   write(failMsg, *) "Incorrect name"
   call ESMF_Test((trim(arrayName)=="MyArray with ArraySpec"), name, failMsg, result, ESMF_SRCLINE)
 
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "ArrayGet replicatedDimCount Test"
+  write(failMsg, *) "Did not return ESMF_SUCCESS"
+  call ESMF_ArrayGet(array, replicatedDimCount=replicatedDimCount, rc=rc)
+  call ESMF_Test((rc.eq.ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE)
+
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "Verify replicatedDimCount returned from Array with 0 replicated dims"
+  write(failMsg, *) "Incorrect replicatedDimCount"
+  call ESMF_Test(replicatedDimCount==0, name, failMsg, result, ESMF_SRCLINE)
+
   !------------------------------------------------------------------------
   !NEX_UTest_Multi_Proc_Only
   write(name, *) "ArrayGet Fortran array pointer, 2D ESMF_TYPEKIND_R8 Test"
@@ -2541,6 +2555,43 @@ program ESMF_ArrayCreateGetUTest
   call ESMF_ArrayDestroy(array, rc=rc)
   call ESMF_Test((rc.eq.ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE)
 
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "ArrayCreate Allocate 2D with 4D DistGrid Test"
+  write(failMsg, *) "Did not return ESMF_SUCCESS"
+  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
+    ! Dimensions 1 and 3 are replicated dimensions
+    distgridToArrayMap = [0,1,0,2], &
+    indexflag=ESMF_INDEX_GLOBAL, name="2D Array with 4D DistGrid", rc=rc)
+  call ESMF_Test((rc.eq.ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE)
+
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "ArrayGet replicatedDimCount Test"
+  write(failMsg, *) "Did not return ESMF_SUCCESS"
+  call ESMF_ArrayGet(array, replicatedDimCount=replicatedDimCount, rc=rc)
+  call ESMF_Test((rc.eq.ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE)
+
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "Verify replicatedDimCount returned from Array with 2 replicated dims"
+  write(failMsg, *) "Incorrect replicatedDimCount"
+  call ESMF_Test(replicatedDimCount==2, name, failMsg, result, ESMF_SRCLINE)
+
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "ArrayGet Fortran array pointer from Array with 2 replicated dims Test"
+  write(failMsg, *) "Did not return ESMF_SUCCESS"
+  call ESMF_ArrayGet(array, farrayPtr=farrayPtr2D, rc=rc)
+  call ESMF_Test((rc.eq.ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE)
+
+  !------------------------------------------------------------------------
+  !NEX_UTest_Multi_Proc_Only
+  write(name, *) "ArrayDestroy Test"
+  write(failMsg, *) "Did not return ESMF_SUCCESS"
+  call ESMF_ArrayDestroy(array, rc=rc)
+  call ESMF_Test((rc.eq.ESMF_SUCCESS), name, failMsg, result, ESMF_SRCLINE)
+
   !------------------------------------------------------------------------
   ! cleanup
   call ESMF_DistGridDestroy(distgrid, rc=rc)