Update kokkos min to 4.1

.github/workflows/CI.yml

@@ -27,7 +27,7 @@ jobs:
         cxx: ['g++', 'clang++']
         backend: ['OPENMP', 'SERIAL']
         cmake_build_type: ['Debug', 'Release']
-        kokkos_ver: ['3.7.02']
+        kokkos_ver: ['4.1.00']
         bounds_check: ['OFF']
         arborx: ['OFF']
         heffte: ['OFF' ]
@@ -42,7 +42,7 @@ jobs:
             cxx: 'g++'
             backend: 'THREADS'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -52,7 +52,7 @@ jobs:
             cxx: 'icpx'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -61,7 +61,7 @@ jobs:
             cxx: 'icpx'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -70,7 +70,7 @@ jobs:
             cxx: 'icpx'
             backend: 'SERIAL'
             cmake_build_type: 'Release'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -79,7 +79,7 @@ jobs:
             cxx: 'icpx'
             backend: 'SERIAL'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -88,7 +88,7 @@ jobs:
             cxx: 'icpx'
             backend: 'SERIAL'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'MKL'
             hypre: 'OFF'
@@ -98,7 +98,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '4.0.00'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -109,7 +109,7 @@ jobs:
             cxx: 'clang++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '4.0.00'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -118,7 +118,7 @@ jobs:
             cxx: 'g++'
             cmake_build_type: 'Release'
             backend: 'OPENMP'
-            kokkos_ver: '4.0.00'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -127,7 +127,7 @@ jobs:
             cxx: 'clang++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '4.0.00'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -137,7 +137,7 @@ jobs:
             cxx: 'g++'
             cmake_build_type: 'Release'
             backend: 'OPENMP'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -146,7 +146,7 @@ jobs:
             cxx: 'clang++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -155,7 +155,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '4.0.00'
+            kokkos_ver: '4.1.00'
             arborx: 'ArborX'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -164,7 +164,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'ArborX'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -173,7 +173,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'FFTW'
             hypre: 'OFF'
@@ -182,7 +182,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Release'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -192,7 +192,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'HYPRE'
@@ -201,7 +201,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -211,7 +211,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'OFF'
             heffte: 'OFF'
             hypre: 'OFF'
@@ -221,7 +221,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'ArborX'
             heffte: 'FFTW'
             hypre: 'HYPRE'
@@ -234,7 +234,7 @@ jobs:
             cxx: 'g++'
             backend: 'OPENMP'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.7.02'
+            kokkos_ver: '4.1.00'
             arborx: 'ArborX'
             heffte: 'FFTW'
             hypre: 'HYPRE'
@@ -568,7 +568,7 @@ jobs:
       matrix:
         cxx: ['nvcc']
         cmake_build_type: ['Release']
-        kokkos_ver: ['3.7.02']
+        kokkos_ver: ['4.1.00']
     runs-on: ubuntu-20.04
     container: ghcr.io/ecp-copa/ci-containers/cuda:12.2.0
     steps:

.jenkins

@@ -56,12 +56,12 @@ pipeline {
                         }
                     }
                 }
-                stage('ROCM-4.5-HIPCC-DEBUG') {
+                stage('ROCM-5.2-HIPCC-DEBUG') {
                     agent {
                         dockerfile {
                             filename 'Dockerfile.hipcc'
                             dir 'docker'
-                            additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-20.04:4.5-complete'
+                            additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-20.04:5.2-complete'
                             args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES'
                             label 'rocm-docker && vega && AMD_Radeon_Instinct_MI60'
                         }

CMakeLists.txt

@@ -36,7 +36,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 # Dependencies
 #------------------------------------------------------------------------------#
 # find kokkos
-find_package(Kokkos 3.7 REQUIRED)
+find_package(Kokkos 4.1 REQUIRED)
 
 # set supported kokkos devices
 set(CABANA_SUPPORTED_DEVICES SERIAL THREADS OPENMP CUDA HIP SYCL OPENMPTARGET)

docker/Dockerfile

@@ -61,7 +61,7 @@ RUN OPENMPI_VERSION=4.0.2 && \
 ENV PATH=${OPENMPI_DIR}/bin:$PATH
 
 # Install Kokkos
-ARG KOKKOS_VERSION=3.7.02
+ARG KOKKOS_VERSION=4.1.00
 ENV KOKKOS_DIR=/opt/kokkos
 RUN KOKKOS_URL=https://github.com/kokkos/kokkos/archive/${KOKKOS_VERSION}.tar.gz && \
     KOKKOS_ARCHIVE=kokkos-${KOKKOS_VERSION}.tar.gz && \
@@ -84,7 +84,7 @@ RUN KOKKOS_URL=https://github.com/kokkos/kokkos/archive/${KOKKOS_VERSION}.tar.gz
 
 # Install ArborX
 ENV ARBORX_DIR=/opt/arborx
-RUN ARBORX_VERSION=4834bff44c23c9510c6ed93366638dcdf85ab217 && \
+RUN ARBORX_VERSION=v1.4 && \
     ARBORX_URL=https://github.com/arborx/ArborX/archive/${ARBORX_VERSION}.tar.gz && \
     ARBORX_ARCHIVE=arborx.tar.gz && \
     wget --quiet ${ARBORX_URL} --output-document=${ARBORX_ARCHIVE} && \

docker/Dockerfile.hipcc

@@ -1,5 +1,5 @@
 # Use -complete to get both rocm and rocfft
-ARG BASE=rocm/dev-ubuntu-20.04:4.5-complete
+ARG BASE=rocm/dev-ubuntu-20.04:5.2-complete
 FROM $BASE
 
 ARG NPROCS=4
@@ -71,7 +71,7 @@ RUN OPENMPI_VERSION=4.0.4 && \
 ENV PATH=${OPENMPI_DIR}/bin:$PATH
 
 # Install Kokkos
-ARG KOKKOS_VERSION=3.7.02
+ARG KOKKOS_VERSION=4.1.00
 ARG KOKKOS_OPTIONS="-DKokkos_ENABLE_HIP=ON -DKokkos_ENABLE_LIBDL=OFF -DKokkos_ARCH_VEGA906=ON"
 ENV KOKKOS_DIR=/opt/kokkos
 RUN KOKKOS_URL=https://github.com/kokkos/kokkos/archive/${KOKKOS_VERSION}.tar.gz && \
@@ -88,7 +88,7 @@ RUN KOKKOS_URL=https://github.com/kokkos/kokkos/archive/${KOKKOS_VERSION}.tar.gz
 
 # Install ArborX
 ENV ARBORX_DIR=/opt/arborx
-RUN ARBORX_VERSION=4834bff44c23c9510c6ed93366638dcdf85ab217 && \
+RUN ARBORX_VERSION=v1.4 && \
     ARBORX_URL=https://github.com/arborx/ArborX/archive/${ARBORX_VERSION}.tar.gz && \
     ARBORX_ARCHIVE=arborx.tar.gz && \
     wget --quiet ${ARBORX_URL} --output-document=${ARBORX_ARCHIVE} && \
@@ -100,7 +100,6 @@ RUN ARBORX_VERSION=4834bff44c23c9510c6ed93366638dcdf85ab217 && \
       -D CMAKE_INSTALL_PREFIX=${ARBORX_DIR} \
       -D CMAKE_BUILD_TYPE=Debug \
       -D CMAKE_CXX_COMPILER=hipcc \
-      -D CMAKE_CXX_FLAGS=-amdgpu-target=gfx906 \
       -D CMAKE_CXX_EXTENSIONS=OFF \
       -D CMAKE_PREFIX_PATH=${KOKKOS_DIR} \
     .. && \

docker/Dockerfile.sycl

@@ -65,7 +65,7 @@ RUN wget https://cloud.cees.ornl.gov/download/oneapi-for-nvidia-gpus-${DPCPP_VER
     rm oneapi-for-nvidia-gpus-${DPCPP_VERSION}-linux.sh
 
 # Install Kokkos
-ARG KOKKOS_VERSION=3.7.02
+ARG KOKKOS_VERSION=4.1.00
 ARG KOKKOS_OPTIONS="-DKokkos_ENABLE_SYCL=ON -DCMAKE_CXX_FLAGS=-Wno-unknown-cuda-version -DKokkos_ENABLE_UNSUPPORTED_ARCHS=ON -DKokkos_ARCH_VOLTA70=ON -DCMAKE_CXX_STANDARD=17"
 ENV KOKKOS_DIR=/opt/kokkos
 RUN . /opt/intel/oneapi/setvars.sh --include-intel-llvm && \

example/core_tutorial/04_aosoa_advanced_unmanaged/advanced_aosoa_unmanaged.cpp

@@ -109,31 +109,32 @@ void unmanagedAoAoAExample()
         if ( slice_b( i ) != ( i % VectorLength ) )
         {
             // Unexpected Value
-            printf( "%d: Unexpected %d != %d \n", __LINE__, slice_b( i ),
-                    i % VectorLength );
+            std::cout << "Error: " << slice_b( i ) << " != " << i % VectorLength
+                      << std::endl;
         }
     }
 
     /*
-     * Use the view in an algorithm, like sort
+     * Use the view in an algorithm, like binning
      */
-    auto binning_data = Cabana::sortByKey( slice_b );
+    auto binning_data = Cabana::binByKey( slice_b, 64 );
     Cabana::permute( binning_data, aosoa );
 
     // Check it worked
-    for ( int i = 1; i < num_tuple; i++ )
+    for ( int i = 1; i < num_tuple; i += 2 )
     {
         // We expect it to be monotonically increasing
         if ( !( slice_b( i - 1 ) <= slice_b( i ) ) )
         {
             // Unexpected value
-            printf( "%d: Unexpected %d vs %d \n", __LINE__, slice_b( i - 1 ),
-                    slice_b( i ) );
+            std::cout << "Error: " << slice_b( i - 1 ) << " > " << slice_b( i )
+                      << std::endl;
         }
 
         // We can also set values
         slice_a( i, 0 ) = i / 2;
     }
+    std::cout << "Successful Cabana binning on unmanaged AoSoA." << std::endl;
 
     // Clean up local data
     delete[] local_data;